diff --git a/Cargo.lock b/Cargo.lock index ad177ae667..8f2bd37d66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -441,6 +441,7 @@ dependencies = [ "dunce", "rayon", "ruff_db", + "salsa", "serde", "serde_json", "sha2", @@ -3710,8 +3711,12 @@ dependencies = [ name = "ruff_python_edits" version = "0.0.12" dependencies = [ + "anyhow", + "ruff_diagnostics", "ruff_python_ast", + "ruff_python_index", "ruff_python_parser", + "ruff_python_trivia", "ruff_source_file", "ruff_text_size", ] @@ -5115,6 +5120,7 @@ dependencies = [ "ruff_memory_usage", "ruff_python_ast", "ruff_python_edits", + "ruff_python_index", "ruff_python_literal", "ruff_python_parser", "ruff_python_stdlib", diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index f492e74c20..fbbbff736d 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -2728,7 +2728,10 @@ dependencies = [ name = "ruff_python_edits" version = "0.0.12" dependencies = [ + "ruff_diagnostics", "ruff_python_ast", + "ruff_python_index", + "ruff_python_trivia", "ruff_source_file", "ruff_text_size", ] @@ -3862,6 +3865,7 @@ dependencies = [ "ruff_memory_usage", "ruff_python_ast", "ruff_python_edits", + "ruff_python_index", "ruff_python_literal", "ruff_python_parser", "ruff_python_stdlib", diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index 935b742fdc..af4ec152bd 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -390,9 +390,10 @@ fn finish( anyhow::anyhow!("the `{pass}` pass produced ill-formed IR:\n {detail}") })?; - // the default config on purpose, except for the target version: the - // lazy-import pass is what binds `JustFloat = float` locally instead of - // emitting a `from ty_extensions import ...` that has no module behind it. + // the default config on purpose, except for the target version and the + // deferral of `from` imports: the lazy-import pass is what binds + // `JustFloat = float` locally instead of emitting a + // `from ty_extensions import ...` that has no module behind it // // the version has to be the *interpreter's*, because this python runs inside // the extension at import time — emitting syntax the interpreter cannot parse @@ -402,6 +403,9 @@ fn finish( source.to_string() } else { let mut config = options.fallback.clone().unwrap_or_default(); + // the compiled functions read the module's globals by name, and a deferred + // `from` import is in them only once something has read it + config.defer_from_imports = false; if let Some((major, minor)) = toolchain.and_then(|toolchain| toolchain.version) && let Ok(parsed) = format!("{major}.{minor}").parse() { diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index ee882b2d37..17c82bc717 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -11561,6 +11561,24 @@ fn a_float_module_imports_without_any_extra_runtime_module() { assert_eq!(out, "12.0"); } +#[test] +fn a_compiled_function_reads_a_name_its_module_imports_with_from() { + // below 3.15 the lazy-import polyfill defers a `from` import by binding nothing where + // it is written, and a compiled function reads the module's globals by name. so the + // fallback a compiled module embeds keeps its `from` imports ordinary ones + agree( + "fromglobal", + "\ +from json import dumps +from os.path import basename as base + +def f(a: int) -> str: + return dumps([a]) + base('/x/y') +", + &["m.f(1)"], + ); +} + #[test] fn a_compiled_function_is_a_c_function_object() { // one of the few places the two builds are *meant* to differ: a natively @@ -22840,10 +22858,64 @@ def widened(value: object) -> str: // the class reports the module it was written in, not `builtins` "(m.Node.__module__, m.Keyed.__module__)", "(m.Node.__name__, m.Node.__qualname__)", - // an imported base reaches this module as a lazy proxy, so `__mro_entries__` - // is what turns it into the class — and python records what was written - "repr(m.Node.__orig_bases__)", + // an imported base is the class itself, with no `__mro_entries__` to + // record what was written in its place + "'__orig_bases__' in vars(m.Node)", + ], + ); +} + +#[test] +fn a_class_on_a_base_that_stands_for_others_agrees() { + // a base that is not a class may stand for a tuple of them through `__mro_entries__`, + // which python resolves before it works out a metaclass, recording what was written in + // `__orig_bases__`. the checker is shown the class the stand-in stands for, so the class + // compiles, and the metaclass here is `ABCMeta`, so the type is built by calling it + let Some(compiled) = agree_in( + "mroentries", + "\ +from abc import ABC +from typing import TYPE_CHECKING + + +class StandIn: + def __mro_entries__(self, bases: tuple[object, ...]) -> tuple[type, ...]: + return (ABC,) + + +if TYPE_CHECKING: + Base = ABC +else: + Base = StandIn() + + +class Node(Base): + def kind(self) -> str: + return \"node\" +", + &[ + "m.Node().kind()", + "[base.__name__ for base in m.Node.__mro__]", + "type(m.Node.__orig_bases__[0]).__name__", + "(type(m.Node).__name__, issubclass(m.Node, __import__('abc').ABC))", ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + // and the class is the one module init built, not the interpreted definition left + // standing because the build gave up on it + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_mroentries as m\nprint(type(m.Node.__dict__['kind']).__name__)\n", + ), + "method_descriptor" ); } @@ -29062,6 +29134,95 @@ class Joiner: ); } +/// `inspect.signature` reads a compiled function's defaults off its interpreted definition, +/// and in `.by` a default re-evaluated per call is a marker there. a tool that binds a call +/// from the signature and calls with `apply_defaults` hands the markers back, and the +/// compiled boundary hands such a call to the definition, whose guard reads each one as the +/// argument not given — the boundary was unboxing the marker as the argument instead +#[test] +fn a_default_marker_handed_back_agrees() { + const HANDED_BACK: &str = "(lambda f: (lambda b: (b.apply_defaults(), f(*b.args, **b.kwargs))[1])(__import__('inspect').signature(f).bind()))"; + let calls = [ + format!("[{HANDED_BACK}(m.fresh), {HANDED_BACK}(m.fresh)]"), + format!("[{HANDED_BACK}(m.limit), setattr(m, 'LIMIT', 5), {HANDED_BACK}(m.limit)]"), + format!("_warnings_from({HANDED_BACK}, m.relaxed)"), + format!("[{HANDED_BACK}(m.keyword), {HANDED_BACK}(m.keyword)]"), + "(lambda d: [m.Bag([1]).add(d), m.Bag([1]).add(d)])(__import__('inspect').signature(m.Bag.add).parameters['more'].default)".to_owned(), + ]; + agree( + "default_markers", + "\ +LIMIT = 2 + + +def fresh(xs: list[int] = []) -> int: + xs.append(1) + return len(xs) + + +def limit(x: int = LIMIT) -> int: + return x + + +def relaxed(x: int = 1, a: int) -> int: + return x + a + + +def keyword(*, xs: list[int] = []) -> int: + xs.append(1) + return len(xs) + + +class Bag: + def __init__(self, items: list[int]) -> None: + self.items = items + + def add(self, more: list[int] = [2]) -> int: + more.append(3) + return len(self.items) + len(more) +", + &calls.iter().map(String::as_str).collect::>(), + ); +} + +/// the marker a compiled method compares an argument against is read off its interpreted +/// definition at module init, through `__wrapped__` where a decorator wrapped it. a +/// `__wrapped__` that leads back round stops the walk, as `inspect.unwrap` stops, rather +/// than holding the import forever. the decorated method itself is left to the class the +/// interpreted definition decorated, which is the one reading its handle at init +#[test] +fn a_wrapped_chain_that_leads_back_round_agrees() { + agree_with_declines( + "wrapped_cycle", + "\ +class Looped: + def __init__(self, fn: (...) -> int): + self.fn = fn + self.__wrapped__ = self + + def __get__(self, instance: object, owner: object = None) -> object: + if instance === None: + return self + return lambda xs: self.fn(instance, xs) + + +def looped(fn: (...) -> int) -> Looped: + return Looped(fn) + + +class Bag: + def __init__(self): + self.n = 1 + + @looped + def fresh(self, xs: list[int] = []) -> int: + xs.append(1) + return len(xs) + self.n +", + &["[m.Bag().fresh([5]), m.Bag().fresh([5])]"], + ); +} + #[test] fn the_numeric_slots_agree() { // every binary numeric dunder fills a `nb_*` slot, and python never looks one up by @@ -44945,6 +45106,30 @@ def show() -> str: ); } +#[test] +fn a_plain_tuple_built_as_an_anonymous_named_tuple_agrees() { + // the checker reads each literal as an instance of the named tuple expected where it + // stands, and the fields are read by name: a plain tuple the lowering built has none + agree_with_declines( + "anonbuilt", + "\ +def take(p: (a: int, b: int)) -> int: + return p.a + p.b + + +def call() -> int: + return take((7, 8)) + + +def second() -> int: + items: list[(a: int, b: int)] = [] + items.append((1, 2)) + return items[0].b +", + &["m.call()", "m.second()"], + ); +} + #[test] fn a_checked_cast_agrees_on_the_value_it_refuses() { // the refusal is the point of the form, so the raise is part of the answer diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index d8b1c2362b..50f69bfc23 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -237,6 +237,17 @@ pub fn emit_module(module: &ModuleIr) -> String { "static PyObject *{} = NULL;", function.interpreted_symbol(module.name.dotted()) ); + let markers = function + .twin_defaults() + .into_iter() + .filter(|_| function.defaults_are_markers); + for index in markers { + let _ = writeln!( + out, + "static PyObject *{} = NULL;", + function.twin_default_symbol(module.name.dotted(), index) + ); + } } } if any_defers { @@ -1726,7 +1737,7 @@ fn emit_written_init(module: &ModuleIr, init: &Function) -> String { ); // the same boundary the method's own wrapper has, and the same hand-over: a slot // is handed a tuple and a dict rather than a vector, which is all that differs - let tests = defer_tests(init, true); + let tests = defer_tests(module, init, true); if !tests.is_empty() { let _ = writeln!( out, @@ -10578,7 +10589,7 @@ fn emit_terminator( /// /// `receiver` says whether the first parameter is a `self` that arrives outside the /// binding, which is what shifts every slot index -fn defer_tests(function: &Function, receiver: bool) -> Vec { +fn defer_tests(module: &ModuleIr, function: &Function, receiver: bool) -> Vec { let slot = |index: &usize| format!("by_bound[{}]", index - usize::from(receiver)); function .deferring @@ -10587,16 +10598,21 @@ fn defer_tests(function: &Function, receiver: bool) -> Vec { let slot = slot(index); format!("({slot} != NULL && !By_IsExactFloat({slot}))") }) - // an omitted parameter whose default is not an immediate: the interpreted - // definition holds the one object every such call has to share — unless the - // receiver holds it, in which case this boundary can fill it itself - .chain( - function - .computed_defaults - .iter() - .filter(|_| function.defaults_held_by == by_ir::function::DefaultsHeldBy::Twin) - .map(|index| format!("{} == NULL", slot(index))), - ) + // a parameter whose default only the interpreted definition holds, omitted: a + // computed default is the one object every such call has to share in python. in + // basedpython it is a marker the definition re-evaluates the default for, or + // raises for a required parameter after a defaulted one, so a call handed that + // very marker is the definition's too. a receiver that holds the default is not + // one of these, and this boundary fills it itself + .chain(function.twin_defaults().iter().map(|index| { + let slot = slot(index); + if function.defaults_are_markers { + let held = function.twin_default_symbol(module.name.dotted(), *index); + format!("({slot} == NULL || {slot} == {held})") + } else { + format!("{slot} == NULL") + } + })) .collect() } @@ -11062,7 +11078,7 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri ) }; - let tests = defer_tests(function, is_method); + let tests = defer_tests(module, function, is_method); if !tests.is_empty() { let _ = writeln!(out, " if ({}) goto by_wrap_defer;", tests.join(" || ")); } @@ -12613,6 +12629,31 @@ fn emit_module_init(module: &ModuleIr) -> String { ); } } + let markers = function + .twin_defaults() + .into_iter() + .filter(|_| function.defaults_are_markers); + for index in markers { + // a keyword-only parameter's default is held by name — the name the + // definition declares it under, which is the parameter's here — and a + // positional one's by its place among the definition's positional parameters + let position = if function.is_keyword_only(index) { + -1 + } else { + i64::try_from(index).unwrap_or(-1) + }; + let name = function + .params() + .get(index) + .and_then(|decl| decl.name.as_deref()) + .unwrap_or_default(); + let _ = writeln!( + interpreted_init, + " {} = By_TwinDefault({handle}, {position}, {});", + function.twin_default_symbol(module.name.dotted(), index), + c_string(name) + ); + } } // the body is run either way; what a class-level constant needs is what each `class` @@ -15898,11 +15939,18 @@ mod tests { /// a class whose `__init__` takes `value`, with a default only the interpreted /// definition holds fn holder() -> ModuleIr { + holder_holding(false) + } + + /// [`holder`], from a `.by` definition when `markers`, which holds a marker for the + /// computed default rather than its value + fn holder_holding(markers: bool) -> ModuleIr { let mut builder = FunctionBuilder::new("__init__", RType::NONE); let receiver = builder.param("self", RType::OBJECT); let value = builder.param("value", RType::OBJECT); builder.defaults(vec![None, None]); builder.computed_defaults(vec![1]); + builder.defaults_are_markers(markers); builder.push(Op::SetField { receiver: Value::Register(receiver), class: "Holder".to_string(), @@ -15971,7 +16019,8 @@ mod tests { "{slot}" ); // and the call that omits it goes to the definition holding the one object - // every such call has to share + // every such call has to share. one handed that object is the native's: python's + // default is a value, and handing the call over would only make it slower assert!( slot.contains( "if (by_bound[0] == NULL) return By_InitInterpreted(byi_app_Holder___init__, \ @@ -15979,6 +16028,7 @@ mod tests { ), "{slot}" ); + assert!(!c.contains("By_TwinDefault("), "{c}"); // which means the slot needs the handle the method wrapper already takes assert!( c.contains("static PyObject *byi_app_Holder___init__ = NULL;"), @@ -15986,6 +16036,29 @@ mod tests { ); } + /// a `.by` definition holds a marker for the default instead, which only its own guard + /// reads as the argument not given — so a call handed that very marker goes to it too + #[test] + fn a_constructor_handed_its_marker_hands_the_call_over() { + let c = emit_module(&holder_holding(true)); + let slot = body_of(&c, "static int By_app_Holder_Type_init("); + assert!( + slot.contains( + "if ((by_bound[0] == NULL || by_bound[0] == byi_app_Holder___init___d1)) \ + return By_InitInterpreted(byi_app_Holder___init__, \"Holder.__init__\", \ + selfobj, args, kwds);" + ), + "{slot}" + ); + // the marker is read off the definition once the module has run + assert!( + c.contains( + "byi_app_Holder___init___d1 = By_TwinDefault(byi_app_Holder___init__, 1, \"value\");" + ), + "{c}" + ); + } + #[test] fn a_nested_functions_computed_default_is_filled_from_its_environment() { // a nested function has no interpreted definition to hand the call to, so its diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index a1f3a0ad71..c011feb456 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -40,6 +40,7 @@ pub struct FunctionBuilder { deferring: Vec, computed_defaults: Vec, defaults_held_by: crate::function::DefaultsHeldBy, + defaults_are_markers: bool, posonly: usize, kwonly: usize, /// per-block `.by` spans, parallel to `blocks` @@ -85,6 +86,7 @@ impl FunctionBuilder { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, posonly: 0, kwonly: 0, block_ranges: vec![None], @@ -254,6 +256,13 @@ impl FunctionBuilder { self.computed_defaults = computed; } + /// that the interpreted definition holds basedpython's markers for those defaults + /// + /// see [`Function::defaults_are_markers`] + pub fn defaults_are_markers(&mut self, markers: bool) { + self.defaults_are_markers = markers; + } + /// that those defaults are held by the receiver rather than by an interpreted twin /// /// see [`crate::function::DefaultsHeldBy::Receiver`] @@ -441,6 +450,7 @@ impl FunctionBuilder { deferring: self.deferring, computed_defaults: self.computed_defaults, defaults_held_by: self.defaults_held_by, + defaults_are_markers: self.defaults_are_markers, binding: crate::function::Binding::Instance, coroutine_body: None, doc: self.doc, diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index 797d41d6b7..c101530cea 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -362,6 +362,12 @@ pub struct Function { pub computed_defaults: Vec, /// who keeps the values for [`Self::computed_defaults`] pub defaults_held_by: DefaultsHeldBy, + /// whether the defaults [`Self::twin_defaults`] names are basedpython's markers: a `.by` + /// definition holds, for a default it re-evaluates per call and for a required + /// parameter after a defaulted one, an object that stands for no argument at all. + /// python's own computed default is a value, and a call handed it is the native's + /// to answer + pub defaults_are_markers: bool, /// decorators to apply, outermost first, after the native function is /// installed in the module namespace pub decorators: Vec, @@ -501,8 +507,50 @@ impl Function { /// whether any call can be handed to the interpreted definition pub fn defers(&self) -> bool { - !self.deferring.is_empty() - || (!self.computed_defaults.is_empty() && self.defaults_held_by == DefaultsHeldBy::Twin) + !self.deferring.is_empty() || !self.twin_defaults().is_empty() + } + + /// the parameters whose default only the interpreted definition holds, by index + /// + /// a computed default, which python evaluated once where the definition stood, and a + /// required parameter after a defaulted one, which basedpython allows and python does + /// not: the definition declares a default for it that stands for no argument at all. + /// a call that omits one of these is the definition's to answer — and so is one that + /// hands it the very object the definition holds, where that is one of basedpython's + /// [markers](Self::defaults_are_markers), as `BoundArguments.apply_defaults` hands it + /// with a signature read off the definition + pub fn twin_defaults(&self) -> Vec { + if self.defaults_held_by != DefaultsHeldBy::Twin { + return Vec::new(); + } + let packed = usize::from(self.vararg) + usize::from(self.kwarg); + let named = self.param_count.saturating_sub(packed); + let positional = named.saturating_sub(self.kwonly); + let defaulted = |index: usize| self.defaults.get(index).is_some_and(Option::is_some); + let mut after_default = false; + let mut indices = Vec::new(); + for index in 0..named { + let computed = self.computed_defaults.contains(&index); + if computed || (index < positional && after_default && !defaulted(index)) { + indices.push(index); + } + after_default |= index < positional && (computed || defaulted(index)); + } + indices + } + + /// whether the parameter at `index` is keyword-only + pub fn is_keyword_only(&self, index: usize) -> bool { + let packed = usize::from(self.vararg) + usize::from(self.kwarg); + let named = self.param_count.saturating_sub(packed); + index < named && index >= named.saturating_sub(self.kwonly) + } + + /// the C identifier for the marker the interpreted definition holds for the parameter + /// at `index`, one of [`Self::twin_defaults`] of a function whose + /// [defaults are markers](Self::defaults_are_markers) + pub fn twin_default_symbol(&self, module: &str, index: usize) -> String { + format!("{}_d{index}", self.interpreted_symbol(module)) } /// the environment field holding the value for the parameter at `index`, for a @@ -1895,6 +1943,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: Binding::Instance, coroutine_body: None, doc: None, diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index f9be553047..16ac2d762e 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -1078,6 +1078,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -1155,6 +1156,7 @@ b2: deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 7ff963a97e..2badd88221 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -2311,6 +2311,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -2523,6 +2524,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -2577,6 +2579,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -2632,6 +2635,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -2739,6 +2743,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index f1de48a4ea..af91d0d8de 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -143,8 +143,9 @@ use ruff_python_stdlib::identifiers::is_identifier; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::ProgramEnvironment; use ty_python_semantic::types::soundness::{CheckKind, CheckTarget}; +use ty_python_semantic::types::soundness_sites::SiteCheck; use ty_python_semantic::types::{KnownClass, SpecialFormType, Type, TypeDefinition, TypeQualifier}; -use ty_python_semantic::{HasType, SemanticModel}; +use ty_python_semantic::{HasType, LoweringModel, SemanticModel}; /// lower every module-level function ty can represent natively /// @@ -154,7 +155,7 @@ use ty_python_semantic::{HasType, SemanticModel}; pub fn build_module( db: &dyn ty_python_semantic::Db, env: &ProgramEnvironment<'_>, - model: &SemanticModel<'_>, + model: &LoweringModel<'_>, suite: &[Stmt], module_name: impl Into, options: impl Into, @@ -1631,6 +1632,7 @@ fn lower_generator_constructor( db, model, layouts, + language, .. } = unit; let mut signed = signature( @@ -1668,6 +1670,7 @@ fn lower_generator_constructor( builder.binding_kinds(posonly, kwonly); builder.deferring(deferring); builder.computed_defaults(computed_defaults); + builder.defaults_are_markers(!language.defaults_evaluated_at_the_def()); let mut registers: HashMap = HashMap::new(); for (name, rtype) in ¶ms { registers.insert(name.clone(), builder.param(name.clone(), rtype.clone())); @@ -7939,6 +7942,7 @@ fn lower_function_with_receiver( builder.defaults_held_by_the_receiver(); } builder.computed_defaults(computed_defaults); + builder.defaults_are_markers(!unit.language.defaults_evaluated_at_the_def()); let mut locals: HashMap = HashMap::new(); for (name, rtype) in ¶ms { @@ -15645,8 +15649,8 @@ impl Lowering<'_, '_> { /// with: an unboxed `int` came through `PyLong_Check`, and an emitted class's /// instance through a type test. the narrowing that made the value raised the /// check's own `TypeError` if it failed, so nothing is left to ask - fn sound_target(&mut self, ty: &RType, plan: &CheckKind) -> Lowered> { - let CheckKind::Isinstance(target) = plan else { + fn sound_target(&mut self, ty: &RType, check: &SiteCheck) -> Lowered> { + let CheckKind::Isinstance(target) = &check.plan else { return Err(Decline::new( "a soundness check against a generic class's type arguments is not lowered", )); @@ -15654,6 +15658,14 @@ impl Lowering<'_, '_> { if self.proves(ty, target) { return Ok(None); } + // the interpreted build looks such a check's names up as it runs, and makes it only + // once they are bound + if !check.once_bound.is_empty() { + return Err(Decline::new( + "a soundness check on a class the module may not have bound yet when it runs \ + is not lowered", + )); + } self.resolve_sound_target(target).map(Some) } @@ -15662,26 +15674,29 @@ impl Lowering<'_, '_> { match target { CheckTarget::AnyOf(parts) => parts.iter().any(|part| self.proves(ty, part)), CheckTarget::NoneType => *ty == RType::NONE, - CheckTarget::Class(name) => match ty { - RType::Primitive(primitive) => { - let builtin = match primitive { - Primitive::Int | Primitive::Fixed(_) => "int", - Primitive::Float => "float", - Primitive::Bool | Primitive::Bit => "bool", - Primitive::Str => "str", - Primitive::List => "list", - Primitive::None - | Primitive::Object - | Primitive::Bytes - | Primitive::Dict - | Primitive::Tuple => return false, - }; - name == builtin + CheckTarget::Class(name) => { + let name = name.source(); + match ty { + RType::Primitive(primitive) => { + let builtin = match primitive { + Primitive::Int | Primitive::Fixed(_) => "int", + Primitive::Float => "float", + Primitive::Bool | Primitive::Bit => "bool", + Primitive::Str => "str", + Primitive::List => "list", + Primitive::None + | Primitive::Object + | Primitive::Bytes + | Primitive::Dict + | Primitive::Tuple => return false, + }; + name == builtin + } + RType::Instance { class, .. } => self.extends(class, &name), + RType::Tuple(_) => name == "tuple", + RType::Array(_) => false, } - RType::Instance { class, .. } => self.extends(class, name), - RType::Tuple(_) => name == "tuple", - RType::Array(_) => false, - }, + } } } @@ -15690,13 +15705,11 @@ impl Lowering<'_, '_> { fn resolve_sound_target(&mut self, target: &CheckTarget) -> Lowered { Ok(match target { CheckTarget::Class(name) => { - let class = match self.place(name) { + let name = name.source(); + let class = match self.place(&name) { Some(Place::Global { .. }) | None => { let dest = self.builder.temp(RType::OBJECT); - self.builder.push(Op::LoadGlobal { - dest, - name: name.clone(), - }); + self.builder.push(Op::LoadGlobal { dest, name }); Value::Register(dest) } Some(place) => { @@ -15875,6 +15888,19 @@ impl Lowering<'_, '_> { Expr::Attribute(node) => self.attribute(node), Expr::List(node) => self.display(&node.elts, Display::List), Expr::Set(node) => self.display(&node.elts, Display::Set), + // the checker reads a plain tuple standing where an anonymous named tuple is + // expected as an instance of that class, which the interpreted twin constructs. + // a tuple built here would have none of its fields + Expr::Tuple(node) + if !node.is_anon_named_tuple + && !node.is_anon_named_tuple_value + && self.model.anon_named_tuple_class(node).is_some() => + { + Err(Decline::new( + "a tuple literal standing where an anonymous named tuple is expected is \ + constructed as that class", + )) + } Expr::Tuple(node) => self.display(&node.elts, Display::Tuple), Expr::Dict(node) => self.dict_display(&node.items), Expr::Subscript(node) => { diff --git a/crates/by_irbuild/src/single_file.rs b/crates/by_irbuild/src/single_file.rs index 57885acecc..7474b9c231 100644 --- a/crates/by_irbuild/src/single_file.rs +++ b/crates/by_irbuild/src/single_file.rs @@ -11,7 +11,7 @@ use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; use ruff_python_ast::Stmt; use ty_project::{ProjectMetadata, TestDb}; -use ty_python_semantic::{ProgramEnvironment, SemanticModel}; +use ty_python_semantic::{LoweringModel, ProgramEnvironment}; use crate::Language; @@ -22,7 +22,7 @@ pub(crate) fn with_source( f: impl FnOnce( &dyn ty_python_semantic::Db, &ProgramEnvironment<'_>, - &SemanticModel<'_>, + &LoweringModel<'_>, &[Stmt], ) -> T, ) -> T { @@ -37,14 +37,14 @@ pub(crate) fn with_source_in( f: impl FnOnce( &dyn ty_python_semantic::Db, &ProgramEnvironment<'_>, - &SemanticModel<'_>, + &LoweringModel<'_>, &[Stmt], ) -> T, ) -> T { let (db, file) = make_db(source, language); let program_file = ty_python_semantic::Db::program_file(&db, file); let parsed = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); - let model = SemanticModel::new(&db, program_file); + let model = LoweringModel::new(&db, program_file); let env = ProgramEnvironment::from_file(program_file); f(&db, &env, &model, parsed.suite()) } diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index 5c5da52c50..8484b01126 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -451,6 +451,24 @@ def unwrap(v: Any) -> Cell[int]: assert!(reason.contains("type arguments"), "{reason}"); } +#[test] +fn a_soundness_check_on_a_class_declared_further_down_declines() { + // the function may run before the module binds `Later`, so the interpreted build looks + // it up as the check runs and makes the check only once it is bound, which a compiled + // module has no lowering for + let reason = decline( + "\ +from typing import Any + +def unwrap(v: Any) -> Later: + return v + +class Later: ... +", + ); + assert!(reason.contains("may not have bound yet"), "{reason}"); +} + #[test] fn a_gradual_parameter_is_an_object() { // `object` assumes nothing, so no check is needed and nothing declines @@ -12311,6 +12329,21 @@ def make() -> (name: str, age: int): assert!(reason.contains("named tuple"), "{reason}"); } +#[test] +fn a_plain_tuple_built_as_an_anonymous_named_tuple_is_declined() { + // the checker reads the literal as an instance of the named tuple expected where it + // stands, so its fields are read by name, and a plain tuple has none of them + let reason = decline( + "\ +def second() -> int: + items: list[(a: int, b: int)] = [] + items.append((1, 2)) + return items[0].b +", + ); + assert!(reason.contains("anonymous named tuple"), "{reason}"); +} + #[test] fn a_checked_cast_is_declined() { // `v cast! int` tests the value and raises when it does not hold; read as a call it diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index a35eaf9cc6..1613f2c36f 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -1491,6 +1491,78 @@ static inline PyObject *By_CallInterpreted(PyObject *fn, const char *name, return PyObject_Vectorcall(fn, args, (size_t)nargs, kwnames); } +/* the object the interpreted definition `twin` holds as a parameter's default: the + * positional parameter at `position`, counted from its first and receiver included, or + * the keyword-only parameter `name` where `position` is negative. a boundary that is + * handed this very object hands the call to the definition, as it does one that omits + * the argument. + * + * a definition a decorator wrapped is reached through `__wrapped__`, which is where + * `inspect.signature` reads the same defaults from. the walk stops where `inspect.unwrap` + * refuses to go on: at an object it has already passed, so a `__wrapped__` that leads + * back round cannot hold module init forever, and past the recursion limit. the objects + * passed are held until the walk ends, so none of their addresses can be reused by the + * next. NULL, and no error, where nothing is held: such a boundary then defers only a + * call that omits the argument */ +static PyObject *By_TwinDefault(PyObject *twin, Py_ssize_t position, const char *name) { + PyObject *function, *passed, *found = NULL; + if (twin == NULL) return NULL; + passed = PyList_New(0); + if (passed == NULL) { + PyErr_Clear(); + return NULL; + } + function = Py_NewRef(twin); + while (function != NULL && !PyFunction_Check(function) && !PyMethod_Check(function)) { + PyObject *wrapped; + Py_ssize_t at, count = PyList_GET_SIZE(passed); + for (at = 0; at < count; at++) { + if (PyList_GET_ITEM(passed, at) == function) break; + } + if (at < count || count >= Py_GetRecursionLimit() || PyList_Append(passed, function) < 0) { + Py_DECREF(function); + function = NULL; + break; + } + wrapped = PyObject_GetAttrString(function, "__wrapped__"); + Py_DECREF(function); + function = wrapped; + } + Py_DECREF(passed); + if (function != NULL && PyMethod_Check(function)) { + PyObject *inner = Py_NewRef(PyMethod_GET_FUNCTION(function)); + Py_DECREF(function); + function = inner; + if (!PyFunction_Check(function)) { + Py_DECREF(function); + function = NULL; + } + } + if (function == NULL) { + PyErr_Clear(); + return NULL; + } + if (position >= 0) { + PyObject *defaults = PyFunction_GetDefaults(function); + PyCodeObject *code = (PyCodeObject *)PyFunction_GetCode(function); + if (defaults != NULL && PyTuple_Check(defaults)) { + Py_ssize_t held = PyTuple_GET_SIZE(defaults); + Py_ssize_t first = code->co_argcount - held; + if (position >= first && position - first < held) { + found = Py_NewRef(PyTuple_GET_ITEM(defaults, position - first)); + } + } + } else { + PyObject *kwdefaults = PyFunction_GetKwDefaults(function); + if (kwdefaults != NULL && PyDict_Check(kwdefaults)) { + found = PyDict_GetItemString(kwdefaults, name); + Py_XINCREF(found); + } + } + Py_DECREF(function); + return found; +} + /* `float`'s narrowing as a test, for a caller that branches on it and reads the double * itself: the double a float holds may be any value at all, so a failure that is only an * error value would have to ask the thread whether an exception is set on every read */ @@ -3853,8 +3925,7 @@ static inline PyObject *By_SpecSubclass(PyObject *module_dict, const char *name, * * python resolves `__mro_entries__` before it works out a metaclass or calls one: a * base that is not a class may stand for a tuple of them. `typing.Generic[T]` is the - * familiar one, and a name bound to a proxy — which is what a lazily imported base is - * until it is first read — is another */ + * familiar one, which a generic class's lowering writes among its bases */ static inline PyObject *By_ResolveBases(PyObject *bases) { Py_ssize_t index, prior; PyObject *tuple; diff --git a/crates/by_stage/Cargo.toml b/crates/by_stage/Cargo.toml index 40775522f1..7074d09240 100644 --- a/crates/by_stage/Cargo.toml +++ b/crates/by_stage/Cargo.toml @@ -29,6 +29,7 @@ sha2 = { workspace = true } walkdir = { workspace = true } [dev-dependencies] +salsa = { workspace = true } tempfile = { workspace = true } [lints] diff --git a/crates/by_stage/src/emit.rs b/crates/by_stage/src/emit.rs index db3fa14bc4..1da5bbcee7 100644 --- a/crates/by_stage/src/emit.rs +++ b/crates/by_stage/src/emit.rs @@ -10,17 +10,22 @@ //! them to a terminal and a language server hands them to an editor, and a //! pipeline that wrote to stderr could only ever serve the first. +use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; use by_transforms::config::Config; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use rayon::iter::{ + IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, +}; use ruff_db::diagnostic::{Annotation, Diagnostic, DiagnosticId, Severity, Span}; +use ruff_db::panic::PanicActivity; use ruff_db::source::source_text; use ty_project::ProjectDatabase; use ty_project::parallel::ParallelIteratorExt; use crate::project::Rebuilder; -use crate::runtime::RuntimeLayout; +use crate::runtime::{RuntimeLayout, RuntimePackage}; use crate::staging::transpiled_destination; /// How much of the check outcome blocks emitting output. @@ -87,6 +92,34 @@ pub struct Emit<'a> { pub root: &'a Path, } +/// one module the transpile fans out over +struct Job<'a> { + by_path: &'a Path, + file: ruff_db::files::File, + /// the package whose runtime copy the module imports, when the build writes + /// one + runtime_package: Option, +} + +/// what one module's transpile came to +enum Outcome { + Transpiled { + python: String, + line_map: Vec>, + requirements: by_transforms::RuntimeRequirements, + }, + /// the transpiler declined to emit the module, for whatever reason it gave + Failed(by_transforms::TranspileError), + /// the transpiler panicked, reported the way a check reports a panic + Panicked(Diagnostic), +} + +impl Outcome { + fn is_failure(&self) -> bool { + !matches!(self, Outcome::Transpiled { .. }) + } +} + /// Check every file, then for each non-blocked file call `consume` with the /// transpiled Python. pub fn check_and_transpile( @@ -170,45 +203,86 @@ pub fn check_and_transpile( } let mut ok = unusable.is_empty(); - // the transpile stays serial. it is the expensive half and the files are - // independent, so it looks like it should fan out — but a type-aware pass - // builds databases of its own (a single-file db for the rewritten source, - // and the project rebuilt over it), and salsa refuses to attach one while a - // query on another is running. parallelising this needs the transpiler to - // stop making databases mid-query first - let rebuild = || Some(rebuilder.rebuild()); - for (bpy, file) in handles { - if unusable.contains(file) { - continue; - } - // the helpers this module calls come from a copy written beside it, and - // which copy depends on where it lands — so the config is settled per - // file rather than once for the build - let relative = transpiled_destination(roots, root, bpy); + // the helpers a module calls come from a copy written beside it, and which + // copy depends on where it lands, so the config is settled per file rather + // than once for the build + let system = ruff_db::Db::system(db); + let runtime_wanted = runtime.is_some(); + let jobs: Vec> = handles + .iter() + .filter(|(_, file)| !unusable.contains(file)) + .map(|(by_path, file)| Job { + by_path, + file: *file, + runtime_package: if runtime_wanted { + RuntimePackage::of( + &transpiled_destination(roots, root, by_path), + by_path, + system, + ) + } else { + None + }, + }) + .collect(); + let finished = fan_out(jobs, gate == CheckGate::AllErrors, db.clone(), |db, job| { let config = Config { - runtime_module: runtime - .as_deref_mut() - .and_then(|layout| layout.claim(&relative, bpy, ruff_db::Db::system(db))), + runtime_module: job.runtime_package.as_ref().map(RuntimePackage::module), ..config.clone() }; - match by_transforms::transpile_typed_with_report(db, *file, &config, Some(&rebuild)) { - Ok((out, line_map, needed)) => { + let rebuild = || Some(rebuilder.rebuild()); + // caught on the worker that ran it, because the hook that records a + // panic's location and backtrace is armed per thread + let transpiled = ty_project::catch_panic( + &*db, + job.file, + PanicActivity::Transpiling, + AssertUnwindSafe(|| { + by_transforms::transpile_typed_with_report(&*db, job.file, &config, Some(&rebuild)) + }), + ); + match transpiled { + Ok(Ok((python, line_map, requirements))) => Outcome::Transpiled { + python, + line_map, + requirements, + }, + Ok(Err(error)) => Outcome::Failed(error), + Err(panic) => Outcome::Panicked(panic), + } + }); + for (job, outcome) in finished { + // a job is skipped only after an earlier one failed, and the hand-over + // has already stopped there + let Some(outcome) = outcome else { + break; + }; + if let (Some(layout), Some(package)) = (runtime.as_deref_mut(), &job.runtime_package) { + layout.record(package); + } + let failure = match outcome { + Outcome::Transpiled { + python, + line_map, + requirements: needed, + } => { requirements.merge(needed); - let by_source = source_text(db, *file); + let by_source = source_text(db, job.file); consume(&Transpiled { - by_path: bpy, + by_path: job.by_path, by_source: by_source.as_str(), - python: &out, + python: &python, line_map: &line_map, })?; + continue; } - Err(e) => { - all_diagnostics.push(transpile_failure_diagnostic(*file, &e)); - ok = false; - if gate == CheckGate::AllErrors { - break; - } - } + Outcome::Failed(error) => transpile_failure_diagnostic(job.file, &error), + Outcome::Panicked(diagnostic) => diagnostic, + }; + all_diagnostics.push(failure); + ok = false; + if gate == CheckGate::AllErrors { + break; } } @@ -222,6 +296,46 @@ pub fn check_and_transpile( }) } +/// Run `transpile` over every job in parallel, and hand each job back with what +/// it came to, in the jobs' own order whatever order they finished in. +/// +/// With `stop_at_first_failure`, a job is not started once a job before it has +/// failed: the caller stops at the first failure in order, so nothing after it is +/// needed. It comes back with no outcome. +/// +/// Each job gets its own copy of `handle`, as the check's jobs each get their own +/// handle on the project db, and none is attached for the whole job: a type-aware +/// pass analyses rewritten source in databases of its own, and salsa refuses to +/// attach a second database on a thread that already has one attached. That holds +/// only because a rayon worker has nothing attached between jobs, so a rayon job +/// that runs attached must never block on nested parallelism, where its worker +/// could steal one of these jobs and run it under that attachment. +fn fan_out( + jobs: Vec, + stop_at_first_failure: bool, + handle: H, + transpile: impl Fn(&mut H, &J) -> Outcome + Sync + Send, +) -> Vec<(J, Option)> +where + J: Send, + H: Clone + Send, +{ + let first_failure = AtomicUsize::new(usize::MAX); + jobs.into_par_iter() + .enumerate() + .map_with(handle, |handle, (index, job)| { + if index > first_failure.load(Ordering::Relaxed) { + return (job, None); + } + let outcome = transpile(handle, &job); + if stop_at_first_failure && outcome.is_failure() { + first_failure.fetch_min(index, Ordering::Relaxed); + } + (job, Some(outcome)) + }) + .collect() +} + /// Whether this diagnostic says the file has no source to transpile: it could /// not be parsed, or it could not be read at all (an encoding ty does not speak, /// a permission error). Either way the transpiler sees an empty module, so @@ -254,3 +368,106 @@ pub fn transpile_failure_diagnostic( } diag } + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::mpsc; + use std::time::Duration; + + use ruff_db::diagnostic::{Diagnostic, DiagnosticId, Severity}; + + use super::{Outcome, fan_out}; + + fn transpiled() -> Outcome { + Outcome::Transpiled { + python: String::new(), + line_map: Vec::new(), + requirements: by_transforms::RuntimeRequirements::default(), + } + } + + fn panicked() -> Outcome { + Outcome::Panicked(Diagnostic::new( + DiagnosticId::Panic, + Severity::Fatal, + "the transpiler is broken", + )) + } + + fn pool(threads: usize) -> rayon::ThreadPool { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("a thread pool") + } + + /// the first job cannot finish until the second has, so the order they finish in is the + /// reverse of theirs, and the jobs still come back in their own + #[test] + fn jobs_come_back_in_their_own_order_whatever_order_they_finish_in() { + let (second_finished, wait_for_second) = mpsc::channel::<()>(); + let wait_for_second = Mutex::new(wait_for_second); + let second_finished = Mutex::new(second_finished); + let saw_the_second_finish = AtomicUsize::new(0); + + let finished = pool(2).install(|| { + fan_out(vec![0, 1], false, (), |(), job: &usize| { + if *job == 0 { + let waited = wait_for_second + .lock() + .expect("the receiver") + .recv_timeout(Duration::from_secs(30)); + if waited.is_ok() { + saw_the_second_finish.fetch_add(1, Ordering::Relaxed); + } + } else { + let _ = second_finished.lock().expect("the sender").send(()); + } + transpiled() + }) + }); + + assert_eq!( + saw_the_second_finish.load(Ordering::Relaxed), + 1, + "the two jobs did not run at the same time" + ); + let order: Vec = finished.iter().map(|(job, _)| *job).collect(); + assert_eq!(order, [0, 1]); + assert!(finished.iter().all(|(_, outcome)| outcome.is_some())); + } + + /// one thread runs the jobs in their order, so every job after the failed first one is + /// one that had not started when it failed + #[test] + fn a_job_after_a_failure_is_not_started_when_the_caller_stops_there() { + let started = AtomicUsize::new(0); + let finished = pool(1).install(|| { + fan_out(vec![0, 1, 2, 3], true, (), |(), job: &usize| { + started.fetch_add(1, Ordering::Relaxed); + if *job == 0 { panicked() } else { transpiled() } + }) + }); + + assert_eq!(started.load(Ordering::Relaxed), 1); + let ran: Vec = finished + .iter() + .map(|(_, outcome)| outcome.is_some()) + .collect(); + assert_eq!(ran, [true, false, false, false]); + } + + /// a build reports every module it could not emit, so a failure stops nothing there + #[test] + fn every_job_runs_when_the_caller_goes_on_past_a_failure() { + let finished = pool(1).install(|| { + fan_out(vec![0, 1, 2], false, (), |(), job: &usize| { + if *job == 0 { panicked() } else { transpiled() } + }) + }); + + assert!(finished.iter().all(|(_, outcome)| outcome.is_some())); + } +} diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index 50cc973ec4..9654ad8a49 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -17,6 +17,7 @@ //! to be one implementation of each. use std::ffi::OsStr; +use std::ops::Deref; use std::path::{Path, PathBuf}; use anyhow::Context; @@ -112,12 +113,27 @@ impl Rebuilder { /// returns the `\\?\` verbatim form and the second does not — so a caller that /// re-derived it would find none of the db's paths under it. pub type ProjectBuild = ( - ProjectDatabase, + FrozenProjectDatabase, Vec<(PathBuf, ruff_db::files::File)>, Rebuilder, PathBuf, ); +/// a project db for a command that runs once, whose inputs are frozen +/// +/// it hands out only a shared [`ProjectDatabase`], because salsa panics on a write to a frozen +/// input and a write needs the db by `&mut`. a clone of that handle can still be written +/// through, and panics the same way +pub struct FrozenProjectDatabase(ProjectDatabase); + +impl Deref for FrozenProjectDatabase { + type Target = ProjectDatabase; + + fn deref(&self) -> &ProjectDatabase { + &self.0 + } +} + /// the sources `build`, `run` and `transpile` claim: a `.py` beside a `.by` /// is that file's own output, so writing beside it again would be circular pub const BY_SOURCES: &[&str] = &["by", "byi"]; @@ -276,7 +292,15 @@ pub fn build_project_db( .with_context(|| format!("non-utf8 path: {}", canonical_root.display()))?; let metadata = project_metadata.clone(); - let db = ProjectDatabase::use_defaults(project_metadata, system); + let mut db = ProjectDatabase::use_defaults(project_metadata, system); + // what `by check` does to its db, for the same reasons: the command runs once and changes + // no input, so salsa need not record a dependency on each one, and every parsed syntax + // tree is kept until it exits rather than evicted, a trade only a short-lived db should + // make. the freeze reaches only files indexed after it, so it comes before the sources + // are read + ruff_db::disable_lru(&mut db); + db.freeze_open_files(); + db.freeze(); let mut sources = project_sources(&db, extensions, &canonical_root, output); // the walk is over a hash set, so order is arbitrary; emit deterministically @@ -291,7 +315,12 @@ pub fn build_project_db( root: sys_root.to_path_buf(), included, }; - Ok((db, sources, rebuilder, canonical_root)) + Ok(( + FrozenProjectDatabase(db), + sources, + rebuilder, + canonical_root, + )) } /// The sources a build claims out of a db that already exists. @@ -356,8 +385,60 @@ pub fn module_roots(db: &ProjectDatabase, cwd: &Path) -> Vec { #[cfg(test)] mod tests { - use super::is_hidden_within; + use super::{BY_SOURCES, build_project_db, is_hidden_within}; + use ruff_db::files::File; + use salsa::Setter as _; + use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::Path; + use ty_project::{Db, ProjectDatabase}; + + /// whether salsa refuses `write` because the input it writes is frozen + /// + /// each write gets a db of its own, built outside the panic it is expected to raise, so a + /// build that panics on its own fails the test rather than passing it + fn refuses_as_frozen(write: impl FnOnce(&mut ProjectDatabase, File)) -> bool { + let project = tempfile::tempdir().expect("tempdir"); + std::fs::write(project.path().join("main.by"), "x = 1\n").expect("a source"); + let (db, sources, _, _) = + build_project_db(project.path(), BY_SOURCES, None).expect("a project db"); + assert_eq!(sources.len(), 1); + let file = sources[0].1; + let mut db = db.0; + let Err(panic) = catch_unwind(AssertUnwindSafe(|| write(&mut db, file))) else { + return false; + }; + let message = panic + .downcast_ref::() + .map(String::as_str) + .or_else(|| panic.downcast_ref::<&str>().copied()) + .unwrap_or_default(); + message.contains("never-changing inputs cannot be mutated") + } + + /// the sources are frozen as `by check` freezes them, which only holds for a file indexed + /// after the freeze: one indexed before it stays mutable, and checking it costs what an + /// incremental db does + #[test] + fn the_sources_a_build_reads_never_change() { + assert!(refuses_as_frozen(|db, file| { + file.set_source_text_override(db).to(None); + })); + } + + #[test] + fn the_project_a_build_reads_never_changes() { + assert!(refuses_as_frozen(|db, _| { + db.project().set_included_paths(db, Vec::new()); + })); + } + + #[test] + fn a_build_never_has_open_files() { + assert!(refuses_as_frozen(|db, _| { + db.project() + .set_open_files(db, std::iter::empty().collect()); + })); + } #[test] fn a_hidden_directory_is_not_project_source() { diff --git a/crates/by_stage/src/restage.rs b/crates/by_stage/src/restage.rs index c7af1e95bb..a075ff8587 100644 --- a/crates/by_stage/src/restage.rs +++ b/crates/by_stage/src/restage.rs @@ -375,7 +375,7 @@ fn restage_transpiled( rebuilder: &tree.rebuilder, requirements: &mut by_transforms::RuntimeRequirements::default(), // a re-stage writes modules back into a tree the build already laid - // out, so what it claims is thrown away: whatever copy of the runtime + // out, so what it records is thrown away: whatever copy of the runtime // a module imports is already sitting where the build put it runtime: Some(&mut RuntimeLayout::default()), roots: &tree.record.module_roots, diff --git a/crates/by_stage/src/runtime.rs b/crates/by_stage/src/runtime.rs index 07d418fed3..91672dd869 100644 --- a/crates/by_stage/src/runtime.rs +++ b/crates/by_stage/src/runtime.rs @@ -28,7 +28,7 @@ use ruff_db::system::{System, SystemPath}; /// the copies a build writes /// -/// claimed for every module rather than only those that turned out to call a +/// recorded for every module rather than only those that turned out to call a /// helper: `by restage` recomputes modules against a tree an earlier build /// wrote, and an edit that newly reaches for a helper has to find the copy there #[derive(Debug, Default)] @@ -38,15 +38,29 @@ pub struct RuntimeLayout { } impl RuntimeLayout { - /// record that `source`, landing at `relative` in the module tree, needs the - /// runtime, and answer the module it imports it from — or `None` for a module - /// in no package, which gets the definitions pasted in - pub(crate) fn claim( - &mut self, - relative: &Path, - source: &Path, - system: &dyn System, - ) -> Option { + /// record that a module importing `package`'s copy was transpiled, whether or + /// not the transpile succeeded + pub(crate) fn record(&mut self, package: &RuntimePackage) { + self.packages.insert(package.0.clone()); + } + + /// where each copy goes within the module tree + pub fn files(&self) -> impl Iterator + '_ { + self.packages + .iter() + .map(|package| Path::new(package).join(by_transforms::runtime::FILE_NAME)) + } +} + +/// the package whose copy of the runtime a module imports +#[derive(Debug)] +pub(crate) struct RuntimePackage(String); + +impl RuntimePackage { + /// the package `source`, landing at `relative` in the module tree, imports the + /// runtime from — or `None` for a module in no package, which gets the + /// definitions pasted in + pub(crate) fn of(relative: &Path, source: &Path, system: &dyn System) -> Option { let mut components = relative.components(); let first = components.next()?; // a `relative` of one component is the module's own file name: it sits at @@ -59,15 +73,12 @@ impl RuntimeLayout { if !is_package(&module_tree_base(relative, source).join(package), system) { return None; } - self.packages.insert(package.to_owned()); - Some(format!("{package}.{}", by_transforms::runtime::MODULE_NAME)) + Some(Self(package.to_owned())) } - /// where each copy goes within the module tree - pub fn files(&self) -> impl Iterator + '_ { - self.packages - .iter() - .map(|package| Path::new(package).join(by_transforms::runtime::FILE_NAME)) + /// the module the copy is imported as + pub(crate) fn module(&self) -> String { + format!("{}.{}", self.0, by_transforms::runtime::MODULE_NAME) } } @@ -117,14 +128,16 @@ mod tests { Self { root, system } } - fn claim(&self, layout: &mut RuntimeLayout, relative: &str) -> Option { + fn record(&self, layout: &mut RuntimeLayout, relative: &str) -> Option { let source = self .root .path() .join("src") .join(relative) .with_extension("by"); - layout.claim(Path::new(relative), &source, &self.system) + let package = RuntimePackage::of(Path::new(relative), &source, &self.system)?; + layout.record(&package); + Some(package.module()) } } @@ -133,7 +146,7 @@ mod tests { fn a_root_module_gets_the_definitions_pasted_in() { let project = Project::with(&["src/main.by"]); let mut layout = RuntimeLayout::default(); - assert_eq!(project.claim(&mut layout, "main.py"), None); + assert_eq!(project.record(&mut layout, "main.py"), None); assert_eq!(layout.files().count(), 0); } @@ -142,7 +155,7 @@ mod tests { let project = Project::with(&["src/pkg/__init__.py", "src/pkg/a.by"]); let mut layout = RuntimeLayout::default(); assert_eq!( - project.claim(&mut layout, "pkg/a.py").as_deref(), + project.record(&mut layout, "pkg/a.py").as_deref(), Some("pkg._by_runtime") ); assert_eq!( @@ -157,7 +170,7 @@ mod tests { let project = Project::with(&["src/pkg/__init__.by", "src/pkg/a.by"]); let mut layout = RuntimeLayout::default(); assert_eq!( - project.claim(&mut layout, "pkg/a.py").as_deref(), + project.record(&mut layout, "pkg/a.py").as_deref(), Some("pkg._by_runtime") ); } @@ -168,7 +181,7 @@ mod tests { fn a_module_in_no_package_gets_the_definitions_pasted_in() { let project = Project::with(&["src/scripts/tool.by"]); let mut layout = RuntimeLayout::default(); - assert_eq!(project.claim(&mut layout, "scripts/tool.py"), None); + assert_eq!(project.record(&mut layout, "scripts/tool.py"), None); assert_eq!(layout.files().count(), 0); } @@ -178,7 +191,7 @@ mod tests { let project = Project::with(&["src/pkg/__init__.py", "src/pkg/sub/deep/a.by"]); let mut layout = RuntimeLayout::default(); assert_eq!( - project.claim(&mut layout, "pkg/sub/deep/a.py").as_deref(), + project.record(&mut layout, "pkg/sub/deep/a.py").as_deref(), Some("pkg._by_runtime") ); assert_eq!( @@ -191,9 +204,9 @@ mod tests { fn each_package_carries_its_own() { let project = Project::with(&["src/one/__init__.py", "src/two/__init__.py"]); let mut layout = RuntimeLayout::default(); - project.claim(&mut layout, "one/a.py"); - project.claim(&mut layout, "two/b.py"); - project.claim(&mut layout, "one/c.py"); + project.record(&mut layout, "one/a.py"); + project.record(&mut layout, "two/b.py"); + project.record(&mut layout, "one/c.py"); assert_eq!( layout.files().collect::>(), vec![ @@ -204,7 +217,7 @@ mod tests { } #[test] - fn an_unclaimed_layout_writes_nothing() { + fn an_unrecorded_layout_writes_nothing() { assert_eq!(RuntimeLayout::default().files().count(), 0); } } diff --git a/crates/by_transforms/src/config.rs b/crates/by_transforms/src/config.rs index c3fa6db20e..ce9d5142a8 100644 --- a/crates/by_transforms/src/config.rs +++ b/crates/by_transforms/src/config.rs @@ -1,5 +1,7 @@ use std::collections::BTreeMap; +pub use ty_python_semantic::types::soundness_sites::SoundnessPositions; + pub use ruff_python_ast::PythonVersion; #[derive(Debug, Clone)] @@ -21,6 +23,14 @@ pub struct Config { /// Tests that compare exact transpile output should set this to `false` /// to keep their expected strings free of the lazy preamble pub lazy_imports: bool, + /// when true (the default), the polyfill defers a `from` import below 3.15 by + /// binding nothing where it is written and reading the name through the + /// module's `_LazyFrom` on first use. off, a `from` import stays an ordinary + /// import, and only module imports are deferred. the interpreted fallback a + /// natively compiled module embeds turns it off: the compiled functions read + /// the module's globals by name, where a deferred name is not until something + /// has read it + pub defer_from_imports: bool, /// when true, every transpiled file is prefixed with /// `from __future__ import annotations`, deferring all annotation /// evaluation. off by default: forward references are handled surgically @@ -101,89 +111,6 @@ pub enum FloatLiteralLowering { Literal, } -/// Per-position toggles for the runtime type-soundness checks. Each field -/// enables validation at one syntactic position where ty accepts a value on -/// an annotation-level claim it can't verify at runtime. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[expect( - clippy::struct_excessive_bools, - reason = "independent per-position toggles, not a state machine" -)] -pub struct SoundnessPositions { - /// results of calls whose type is typevar-derived — a generic function's - /// return (`t[T]() -> T`) or a method bound to a specialized generic - /// instance (`dict[str, int].get`) - pub generic_calls: bool, - /// element reads out of a specialized container (`a[0]` on `list[str]`) - pub projections: bool, - /// loop / comprehension elements drawn from a specialized iterable - pub iterations: bool, - /// explicit `Any` (or a context-solved generic result) flowing into an - /// annotated assignment target (`a: str = any_val`) - pub assignments: bool, - /// a returned value validated against the enclosing function's declared - /// return type (`def g() -> str: return any_val`) - pub returns: bool, - /// a call argument validated against its matched parameter's annotation - /// (`takes(any_val)` where `takes(s: str)`) - pub arguments: bool, - /// a function's own parameters validated against their annotations at - /// entry, defending its contract against callers the checker never saw - /// (untyped or third-party code). off in the default set — it runs on - /// every call, so it's opt-in — but included in [`Self::all`] - pub parameters: bool, -} - -impl SoundnessPositions { - /// every position enabled, including the opt-in defensive `parameters` - /// entry checks (the CLI `--soundness all`) - pub fn all() -> Self { - Self { - parameters: true, - ..Self::defaults() - } - } - - /// the positions enabled without an explicit request: every inference-gap - /// check, but *not* the defensive `parameters` entry checks (those add a - /// check on every call, so they stay opt-in) - pub fn defaults() -> Self { - Self { - generic_calls: true, - projections: true, - iterations: true, - assignments: true, - returns: true, - arguments: true, - parameters: false, - } - } - - /// no position enabled — the pass is a no-op - pub fn none() -> Self { - Self { - generic_calls: false, - projections: false, - iterations: false, - assignments: false, - returns: false, - arguments: false, - parameters: false, - } - } - - /// whether any position is enabled - pub(crate) fn any(self) -> bool { - self.generic_calls - || self.projections - || self.iterations - || self.assignments - || self.returns - || self.arguments - || self.parameters - } -} - impl Default for Config { fn default() -> Self { Self { @@ -191,6 +118,7 @@ impl Default for Config { is_python: false, is_stub: false, lazy_imports: true, + defer_from_imports: true, inject_future_annotations: false, prune_unused_imports_after_reverse: true, soundness: SoundnessPositions::defaults(), diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index b40128332f..01c6ae8b26 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -1,6 +1,7 @@ pub mod config; mod reverse_transforms; pub mod runtime; +mod semantic_syntax; pub(crate) mod source_map; mod transforms; pub(crate) mod type_info; @@ -21,6 +22,7 @@ pub mod entry_point { }; } +use transforms::quotes::WrittenQuotes; use transforms::repeated_underscore::{CodeNames, PrivateNames, SynthesizedClasses}; use std::collections::{BTreeSet, HashSet}; @@ -28,8 +30,8 @@ use std::collections::{BTreeSet, HashSet}; use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::{DbWithWritableSystem as _, SystemPath, SystemPathBuf}; use ruff_diagnostics::{Edit, Fix, IsolationLevel, SourceMap}; -use ruff_python_ast::Stmt; -use ruff_python_ast::visitor::Visitor; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{Expr, Stmt}; use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; use salsa::Setter as _; @@ -43,6 +45,21 @@ pub(crate) fn make_in_memory_db(source: &str) -> (TestDb, File) { make_in_memory_db_at(source, PythonVersion::latest_ty()) } +/// a single-file db over `source` read as the python it is: transpiled output, which +/// python runs, and which is to be understood the way python understands it +fn make_in_memory_python_db(source: &str) -> (TestDb, File) { + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program_with_python_version(PythonVersion::latest_ty()) + .expect("program init failed"); + db.write_file("/output.py", source) + .expect("write file failed"); + let file = system_path_to_file(&db, "/output.py").expect("file not in db"); + (db, file) +} + /// [`make_in_memory_db`], analysing the source at `python_version` fn make_in_memory_db_at(source: &str, python_version: PythonVersion) -> (TestDb, File) { let mut db = TestDb::new(ProjectMetadata::new( @@ -115,7 +132,8 @@ fn run_context_sensitive_phase<'a>( if !parsed.errors().is_empty() { return std::borrow::Cow::Borrowed(source); } - let model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); + let model = ty_python_semantic::LoweringModel::new(db, db.program_file(file)); + model.infer_in_check_order(); transforms::context_sensitive::qualify(source, parsed.suite(), &model) } @@ -142,7 +160,8 @@ fn run_erased_union_phase<'a>( if !parsed.errors().is_empty() { return std::borrow::Cow::Borrowed(source); } - let model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); + let model = ty_python_semantic::LoweringModel::new(db, db.program_file(file)); + model.infer_in_check_order(); transforms::erased_union::reify(source, parsed.suite(), &model, config.min_version) } @@ -234,6 +253,7 @@ fn transpile_with_report( ty_python_semantic::extension_backing_functions(&local_db, local_file), ); let synthesized_classes = SynthesizedClasses::default(); + let quotes = WrittenQuotes::default(); let written_names = written_names .with_private(&private_names) .with_synthesized(&synthesized_classes); @@ -245,6 +265,7 @@ fn transpile_with_report( let (source, ast_errors, _phase0_map) = transforms::ast_driver::run_against_source( source, written_names, + "es, config, if unchanged { transforms::ast_driver::SemanticSource::Project(&local_db, local_file) @@ -266,7 +287,7 @@ fn transpile_with_report( ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); - let model = ty_python_semantic::SemanticModel::new( + let model = ty_python_semantic::LoweringModel::new( &db, ty_python_semantic::Db::program_file(&db, file), ); @@ -280,13 +301,14 @@ fn transpile_with_report( // --- Phase 2: import-redirect, surface-syntax cleanup, lazy-import marking --- let (final_output, requirements) = run_import_redirect_phase(output, config); - let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; + let final_output = run_anon_named_tuple_cleanup(final_output, written_names, "es, config)?; let final_output = run_lazy_import_phase( final_output, config, &model.eagerly_imported_modules(), &model.eagerly_imported_names(), - ); + "es.texts(), + )?; let final_output = run_version_polyfill_phase(final_output, config); // --- Phase 3: syntax verification --- @@ -455,6 +477,7 @@ pub fn transpile_typed_with_report( ty_python_semantic::extension_backing_functions(db, file), ); let synthesized_classes = SynthesizedClasses::default(); + let quotes = WrittenQuotes::default(); let written_names = written_names .with_private(&private_names) .with_synthesized(&synthesized_classes); @@ -488,11 +511,16 @@ pub fn transpile_typed_with_report( }; // which imports must stay eager, computed against the *project* db: a // single-file db cannot resolve the modules that declare the conformances - let eager_model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); + let eager_model = ty_python_semantic::LoweringModel::new(db, db.program_file(file)); let eager_imports = eager_model.eagerly_imported_modules(); let eager_names = eager_model.eagerly_imported_names(); - let (spliced, ast_errors, phase0_map) = - transforms::ast_driver::run_against_source(working_source, written_names, config, project); + let (spliced, ast_errors, phase0_map) = transforms::ast_driver::run_against_source( + working_source, + written_names, + "es, + config, + project, + ); if let Some(first) = ast_errors.first() { return Err(first.clone().into()); } @@ -533,8 +561,14 @@ pub fn transpile_typed_with_report( } let (final_output, requirements) = run_import_redirect_phase(output, config); - let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; - let final_output = run_lazy_import_phase(final_output, config, &eager_imports, &eager_names); + let final_output = run_anon_named_tuple_cleanup(final_output, written_names, "es, config)?; + let final_output = run_lazy_import_phase( + final_output, + config, + &eager_imports, + &eager_names, + "es.texts(), + )?; let final_output = run_version_polyfill_phase(final_output, config); // phases 1-2c only prepend preambles at the top and edit within lines, so @@ -609,95 +643,190 @@ fn line_count(s: &str) -> usize { fn run_anon_named_tuple_cleanup( mut source: String, written: WrittenNames, + quotes: &WrittenQuotes, config: &Config, ) -> Result { - use ruff_python_ast::visitor::Visitor; - for _ in 0..4 { - let (db, file) = make_in_memory_db(&source); - let source_ref = ruff_db::source::source_text(&db, file); - let src = source_ref.as_str(); - let module = ruff_db::parsed::parsed_module( - &db, - ty_python_semantic::Db::program_file(&db, file).python_file(&db), - ) - .load(&db); - let model = ty_python_semantic::SemanticModel::new( - &db, - ty_python_semantic::Db::program_file(&db, file), - ); + match run_cleanup_round(&source, written, quotes, config)? { + CleanupRound::Skipped | CleanupRound::Unchanged => return Ok(source), + CleanupRound::Lowered(lowered) => source = lowered, + } + } + Ok(source) +} + +/// what one round of [`run_anon_named_tuple_cleanup`] did +#[derive(Debug, PartialEq, Eq)] +enum CleanupRound { + /// the output spells nothing the cleanup lowers, so the round built no db and asked the + /// checker nothing + Skipped, + /// the output was analysed and nothing in it needed lowering + Unchanged, + /// the output with what was left lowered, which the next round reads again + Lowered(String), +} + +/// one round of [`run_anon_named_tuple_cleanup`] over `source` +/// +/// the only syntax the round lowers is an anonymous named tuple form and an inline protocol, +/// and the only other thing it does is construct each plain tuple literal the checker reads as +/// an anonymous named tuple. the checker makes that type only from a form the parser flags +/// (`DynamicNamedTupleAnchor::Anonymous`), and the round's db reads nothing but `source` and +/// the vendored stubs, none of which spells one (`vendored_stubs_spell_no_anonymous_named_tuple` +/// checks that). so a `source` that spells neither has nothing to lower or construct, and the +/// round leaves it before building a db +fn run_cleanup_round( + source: &str, + written: WrittenNames, + quotes: &WrittenQuotes, + config: &Config, +) -> Result { + // read as basedpython, the way the round's db reads it + let parsed = ruff_python_parser::parse_unchecked_source( + source, + ruff_python_ast::PySourceType::BasedPython, + ); + if SurfaceSyntax::find(parsed.suite(), SurfaceSyntax::CLEANED_UP).is_none() { + return Ok(CleanupRound::Skipped); + } + let (db, file) = make_in_memory_db(source); + let source_ref = ruff_db::source::source_text(&db, file); + let src = source_ref.as_str(); + let module = ruff_db::parsed::parsed_module( + &db, + ty_python_semantic::Db::program_file(&db, file).python_file(&db), + ) + .load(&db); + let model = ty_python_semantic::LoweringModel::new( + &db, + ty_python_semantic::Db::program_file(&db, file), + ); - let mut anon = transforms::anon_named_tuple::AnonNamedTuple::new( - src, - written, - &model, - config.clone(), - module.suite(), + let mut anon = transforms::anon_named_tuple::AnonNamedTuple::new( + src, + written, + quotes, + &model, + config.clone(), + module.suite(), + ); + for stmt in module.suite() { + anon.visit_stmt(stmt); + } + anon.finish(); + if let Some(err) = anon.errors.first() { + return Err(err.clone()); + } + let protocol = + transforms::protocol_type::cleanup(src, written, quotes, &model, module.suite(), config)?; + + if anon.edits.is_empty() && !anon.needs_import && protocol.is_none() { + return Ok(CleanupRound::Unchanged); + } + let mut preamble = String::new(); + if anon.needs_import { + // an earlier run over the pre-splice source already emitted a class + // (and its imports) for an anon-NT whose edit an AST pass then + // re-rendered away, so only what the spliced output is actually + // missing gets prepended. matched as a run of whole lines, the way + // the inline-protocol cleanup does + let source_lines: Vec<&str> = source.lines().collect(); + let push_missing = |preamble: &mut String, entry: &str| { + let entry_lines: Vec<&str> = entry.lines().collect(); + let present = !entry_lines.is_empty() + && source_lines + .windows(entry_lines.len()) + .any(|window| window == entry_lines.as_slice()); + if !present { + preamble.push_str(entry); + preamble.push('\n'); + } + }; + push_missing( + &mut preamble, + &written.import_from("typing", &["NamedTuple"]), ); - for stmt in module.suite() { - anon.visit_stmt(stmt); + let (imports, helpers) = anon.callable.take_requirements(); + for line in imports + .into_iter() + .chain(runtime_entries(config, &helpers, source)) + { + push_missing(&mut preamble, line.trim_end_matches('\n')); + } + for defs in [anon.callable.class_defs().to_owned(), anon.class_defs()] { + for class_def in defs.split_inclusive("\n\n") { + push_missing(&mut preamble, class_def.trim_end_matches('\n')); + } } - if let Some(err) = anon.errors.first() { - return Err(err.clone()); + } + let mut edits = anon.edits; + if let Some((protocol_edits, protocol_preamble)) = protocol { + preamble.push_str(&protocol_preamble); + edits.extend(protocol_edits); + } + + let (body, _) = apply_transforms_once(src, edits); + Ok(CleanupRound::Lowered(if preamble.is_empty() { + body + } else { + splice_preamble(&body, &preamble) + })) +} + +/// basedpython-only syntax [`run_anon_named_tuple_cleanup`] lowers when an earlier pass left it +/// in the output +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SurfaceSyntax { + /// an anonymous named tuple, as a type `(name: T, ...)` or as a value `(name=v, ...)` + AnonymousNamedTuple, + /// `protocol(...)` + InlineProtocol, +} + +impl SurfaceSyntax { + /// what [`run_anon_named_tuple_cleanup`] lowers + const CLEANED_UP: &[Self] = &[Self::AnonymousNamedTuple, Self::InlineProtocol]; + + fn of(expr: &Expr) -> Option { + match expr { + Expr::Tuple(tuple) if tuple.is_anon_named_tuple || tuple.is_anon_named_tuple_value => { + Some(Self::AnonymousNamedTuple) + } + Expr::ProtocolType(_) => Some(Self::InlineProtocol), + _ => None, } - let protocol = - transforms::protocol_type::cleanup(src, written, &model, module.suite(), config)?; + } - if anon.edits.is_empty() && !anon.needs_import && protocol.is_none() { - return Ok(source); + /// the first expression in `stmts`, in source order, that spells one of `kinds` + fn find(stmts: &[Stmt], kinds: &[Self]) -> Option<(Self, TextRange)> { + struct Finder<'a> { + kinds: &'a [SurfaceSyntax], + found: Option<(SurfaceSyntax, TextRange)>, } - let mut preamble = String::new(); - if anon.needs_import { - // an earlier run over the pre-splice source already emitted a class - // (and its imports) for an anon-NT whose edit an AST pass then - // re-rendered away, so only what the spliced output is actually - // missing gets prepended. matched as a run of whole lines, the way - // the inline-protocol cleanup does - let source_lines: Vec<&str> = source.lines().collect(); - let push_missing = |preamble: &mut String, entry: &str| { - let entry_lines: Vec<&str> = entry.lines().collect(); - let present = !entry_lines.is_empty() - && source_lines - .windows(entry_lines.len()) - .any(|window| window == entry_lines.as_slice()); - if !present { - preamble.push_str(entry); - preamble.push('\n'); + impl<'ast> Visitor<'ast> for Finder<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + if self.found.is_none() { + walk_stmt(self, stmt); } - }; - push_missing( - &mut preamble, - &written.import_from("typing", &["NamedTuple"]), - ); - let (imports, helpers) = anon.callable.take_requirements(); - for line in imports - .into_iter() - .chain(runtime_entries(config, &helpers, &source)) - { - push_missing(&mut preamble, line.trim_end_matches('\n')); } - for defs in [anon.callable.class_defs().to_owned(), anon.class_defs()] { - for class_def in defs.split_inclusive("\n\n") { - push_missing(&mut preamble, class_def.trim_end_matches('\n')); + fn visit_expr(&mut self, expr: &'ast Expr) { + if self.found.is_some() { + return; } + if let Some(kind) = SurfaceSyntax::of(expr) + && self.kinds.contains(&kind) + { + self.found = Some((kind, expr.range())); + return; + } + walk_expr(self, expr); } } - let mut edits = anon.edits; - if let Some((protocol_edits, protocol_preamble)) = protocol { - preamble.push_str(&protocol_preamble); - edits.extend(protocol_edits); - } - - let (body, _) = apply_transforms_once(src, edits); - source = if preamble.is_empty() { - body - } else { - splice_preamble(&body, &preamble) - }; - - let _ = config; + let mut finder = Finder { kinds, found: None }; + finder.visit_body(stmts); + finder.found } - Ok(source) } /// Rewrite stdlib imports to `typing_extensions` where the imported name is @@ -736,19 +865,23 @@ fn run_import_redirect_phase(source: String, config: &Config) -> (String, Runtim /// prepends `lazy ` (PEP 810, Python 3.15+) to every `import` and /// `from import` statement. Skips `from __future__` and star imports. /// -/// Gated on `min_version >= 3.15`: PEP 810 syntax doesn't parse on older -/// Python, so we leave imports eager when the target version can't handle -/// the keyword. A redundant `lazy` keyword written in source is stripped -/// in that case so the output stays valid +/// Below 3.15 the keyword does not parse, so the imports are lowered onto the +/// runtime's polyfill instead: a module import binds a lazily loaded module, and +/// a `from` import binds nothing at all, every read of the name being written as +/// a read through the module's `_LazyFrom`, which resolves it on first use. A +/// redundant `lazy` keyword written in source is stripped so the output stays +/// valid /// /// `eager` names the modules that must not be deferred whatever the target: a /// module declaring a conformance exists at runtime only because its /// registration ran, so deferring it defers the conformance out of existence /// -/// `eager_names` names the *bindings* that must be bound to the real object: a -/// lazy proxy cannot stand where cpython checks for a real class, which is what -/// `except` does, nor where a special form is told apart by identity, nor in an -/// annotation something reads back +/// `eager_names` names the *bindings* that must be bound to the real object where +/// the import is written: what `except` catches, a special form, and a name an +/// annotation, a decorator or a forward reference written as a string reads, whose +/// readers look them up in the module's namespace or tell them apart by identity as +/// the module is executing. `quotes` are the type expressions inside the forward +/// references the lowering wrote as strings, recorded as it wrote them /// /// A stub defers nothing: it is never executed, and deferring its imports would /// hand a checker a call result where the stub declares a module or a class @@ -757,12 +890,13 @@ fn run_lazy_import_phase( config: &Config, eager: &[String], eager_names: &[String], -) -> String { + quotes: &[String], +) -> Result { if !config.lazy_imports { - return source; + return Ok(source); } - let (db, file) = make_in_memory_db(&source); + let (db, file) = make_in_memory_python_db(&source); let source_ref = ruff_db::source::source_text(&db, file); let src = source_ref.as_str(); let module = ruff_db::parsed::parsed_module( @@ -770,21 +904,20 @@ fn run_lazy_import_phase( ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); + let model = ty_python_semantic::LoweringModel::new( + &db, + ty_python_semantic::Db::program_file(&db, file), + ); // the lowering writes imports of its own — an enum's `ClassVar` among them — which // the source the transpile began from never held, so the output is asked too. the // annotations are read off the output for the same reason: a lowering writes those let mut eager_names = eager_names.to_vec(); - eager_names.extend( - ty_python_semantic::SemanticModel::new( - &db, - ty_python_semantic::Db::program_file(&db, file), - ) - .eagerly_imported_names(), - ); + eager_names.extend(model.eagerly_imported_names()); eager_names.extend(transforms::lazy_import::annotation_names( module.suite(), src, )); + eager_names.extend(transforms::lazy_import::forward_reference_names(quotes)); eager_names.extend(transforms::lazy_import::decorator_names(module.suite())); let deferral = if config.is_stub { @@ -794,30 +927,56 @@ fn run_lazy_import_phase( } else { transforms::lazy_import::Deferral::Polyfill }; - // the runtime is what the polyfill itself runs on, and a helper reached - // through a proxy would be a proxy call on every use + let deferrable = if deferral == transforms::lazy_import::Deferral::Polyfill + && config.defer_from_imports + { + transforms::lazy_import::deferrable_reads(module.suite(), model.deferrable_from_imports()) + } else { + std::collections::HashMap::default() + }; + let code = CodeNames::of(module.suite()); + let written = transforms::repeated_underscore::WrittenNames::new(src).with_code(&code); + let holder = written.fresh("_by_lazy"); + // the runtime is what the polyfill itself runs on let mut eager = eager.to_vec(); eager.extend(config.runtime_module.clone()); - let mut lazy = transforms::lazy_import::LazyImport::new(src, deferral, &eager, &eager_names); + let mut lazy = transforms::lazy_import::LazyImport::new( + src, + deferral, + &eager, + &eager_names, + deferrable.keys().copied().collect(), + &holder, + ); for stmt in module.suite() { lazy.visit_stmt(stmt); } - let needs_module = lazy.needs_module_helper; - let needs_attr = lazy.needs_attr_helper; - let needs_ty_ext = lazy.needs_ty_ext_marker; - let needs_character_class = lazy.needs_character_class; + let mut edits = std::mem::take(&mut lazy.edits); + for (alias, bound) in &lazy.deferred { + for read in deferrable.get(alias).into_iter().flatten() { + edits.push(Fix::safe_edit(Edit::range_replacement( + format!("{holder}.{bound}"), + *read, + ))); + } + } + let deferred_names: Vec = lazy + .deferred + .iter() + .map(|(_, bound)| bound.clone()) + .collect(); let helpers = transforms::lazy_import::polyfill_helpers( - needs_module, - needs_attr, - needs_ty_ext, - needs_character_class, + lazy.needs_module_helper, + !deferred_names.is_empty(), + lazy.needs_ty_ext_marker, + lazy.needs_character_class, ); - if lazy.edits.is_empty() && helpers.is_empty() { - return source; + if edits.is_empty() && helpers.is_empty() { + return Ok(source); } - let (mut body, _) = apply_transforms_once(src, lazy.edits); + let (mut body, _) = apply_transforms_once(src, edits); // phase 0 already gave the module the helpers *it* needed. where that was an import of the // runtime written beside it, the helpers this phase needs belong on that same line rather // than on a second one naming the same module @@ -826,17 +985,58 @@ fn run_lazy_import_phase( { let line = runtime::import_line(module, helpers.iter().copied()); let left_over = transforms::ast_driver::merge_into_own_imports(&mut body, vec![line]); - return match left_over.first() { - Some(line) => splice_preamble(&body, &format!("{line}\n")), - None => body, - }; - } - let preamble = runtime_preamble(config, &helpers, &body); - if preamble.is_empty() { - body + if let Some(line) = left_over.first() { + body = splice_preamble(&body, &format!("{line}\n")); + } } else { - splice_preamble(&body, &preamble) + let preamble = runtime_preamble(config, &helpers, &body); + if !preamble.is_empty() { + body = splice_preamble(&body, &preamble); + } + } + if deferred_names.is_empty() { + return Ok(body); } + let globals = written.builtin("globals"); + let lines = transforms::lazy_import::holder_lines(&holder, &deferred_names, &globals); + let mut block = String::new(); + for import in written.builtin_imports(&lines, config.min_version.minor) { + block.push_str(&import); + block.push('\n'); + } + block.push_str(&lines); + insert_after_binding(&body, runtime::LAZY_FROM.name(), &block) +} + +/// `text` inserted on the line after the top-level statement of `module` that binds `name`, +/// a runtime helper the preamble provides. a module that does not bind it was not given it, +/// and the lines that need it have nowhere to go +fn insert_after_binding(module: &str, name: &str, text: &str) -> Result { + let parsed = + ruff_python_parser::parse_unchecked_source(module, ruff_python_ast::PySourceType::Python); + let Some(statement) = parsed + .suite() + .iter() + .find(|stmt| runtime::bindings(stmt).contains_key(name)) + else { + return Err(format!( + "the lazy-import polyfill needs `{name}`, and the module was not given it" + )); + }; + let end = usize::from(statement.end()); + let at = module[end..] + .find('\n') + .map_or(module.len(), |newline| end + newline + 1); + let separator = if at == module.len() && !module.ends_with('\n') { + "\n" + } else { + "" + }; + Ok(format!( + "{}{separator}{text}{}", + &module[..at], + &module[at..] + )) } /// The lines that give `module`, python, the runtime helpers it calls: an import of @@ -890,27 +1090,46 @@ fn run_version_polyfill_phase(source: String, config: &Config) -> String { } /// Re-parse the transpiled output *as the target python version* and report any -/// construct that version cannot parse. +/// construct that version cannot parse, or that python refuses to compile on any +/// version from it on /// /// [`verify_syntax`] asks whether the output is python at all; this asks whether /// it is python the file's declared floor can run. Without it, syntax no /// polyfill covers — `except*`, t-strings, a PEP 701 f-string — is emitted -/// verbatim and fails at import time in generated code the author never wrote. +/// verbatim and fails at import time in generated code the author never wrote, +/// and so does python that parses but does not compile: an assignment +/// expression in a comprehension's iterable, a capture pattern that makes the +/// remaining cases unreachable fn verify_target_syntax(source: &str, config: &Config) -> Result<(), TranspileError> { let options = ruff_python_parser::ParseOptions::from(ruff_python_ast::PySourceType::Python) .with_target_version(config.min_version); let parsed = ruff_python_parser::parse_unchecked(source, options); - let Some(first) = parsed.unsupported_syntax_errors().first() else { - return Ok(()); - }; - // the parser's own wording already names both versions: "Cannot use - // `except*` on Python 3.9 (syntax was added in Python 3.11)" - Err(TranspileError { - kind: TranspileErrorKind::InvalidSyntax, - message: first.to_string(), - output_range: Some(first.range), - by_range: None, - }) + if let Some(first) = parsed.unsupported_syntax_errors().first() { + // the parser's own wording already names both versions: "Cannot use + // `except*` on Python 3.9 (syntax was added in Python 3.11)" + return Err(TranspileError { + kind: TranspileErrorKind::InvalidSyntax, + message: first.to_string(), + output_range: Some(first.range), + by_range: None, + }); + } + // a `.by` source holds back python's two `match` checks, because a bare + // `case A:` name may be an enum member of the subject rather than the + // capture it looks like. by here every name is spelled out, so they apply + // with the rest + if let Some(module) = parsed.syntax().as_module() + && let Some(error) = + semantic_syntax::first_error(&module.body, source, config.min_version, config.is_stub) + { + return Err(TranspileError { + kind: TranspileErrorKind::InvalidSyntax, + message: format!("transpiler produced invalid Python: {error}"), + output_range: Some(error.range), + by_range: None, + }); + } + Ok(()) } /// Splice `preamble` into `body` where generated lines belong: after the module @@ -1018,7 +1237,7 @@ impl From for TranspileError { } fn verify_syntax(source: &str, author: &AuthorNames) -> Result<(), TranspileError> { - use ruff_python_ast::{PySourceType, visitor::Visitor}; + use ruff_python_ast::PySourceType; let parsed = ruff_python_parser::parse_unchecked_source(source, PySourceType::Python); let parse_errors = parsed.errors(); @@ -1039,76 +1258,6 @@ fn verify_syntax(source: &str, author: &AuthorNames) -> Result<(), TranspileErro }); } - #[expect(clippy::items_after_statements, reason = "scanner colocated with use")] - struct AnonNamedTupleScanner { - leftover_range: Option, - leftover_typeof: Option, - } - #[expect(clippy::items_after_statements, reason = "scanner colocated with use")] - impl<'ast> Visitor<'ast> for AnonNamedTupleScanner { - fn visit_expr(&mut self, expr: &'ast ruff_python_ast::Expr) { - if self.leftover_range.is_some() || self.leftover_typeof.is_some() { - return; - } - if let ruff_python_ast::Expr::Tuple(t) = expr { - if t.is_anon_named_tuple || t.is_anon_named_tuple_value { - self.leftover_range = Some(::range(t)); - return; - } - } - if let ruff_python_ast::Expr::Subscript(s) = expr { - if s.is_typeof { - self.leftover_typeof = - Some(::range(s)); - return; - } - } - ruff_python_ast::visitor::walk_expr(self, expr); - } - } - - let mut scanner = AnonNamedTupleScanner { - leftover_range: None, - leftover_typeof: None, - }; - for stmt in parsed.suite() { - scanner.visit_stmt(stmt); - if scanner.leftover_range.is_some() || scanner.leftover_typeof.is_some() { - break; - } - } - if let Some(range) = scanner.leftover_range { - let snippet = &source[usize::from(range.start())..usize::from(range.end())]; - return Err(TranspileError { - kind: TranspileErrorKind::Bug, - message: format!("transpiler failed to lower anonymous named tuple syntax `{snippet}`"), - output_range: Some(range), - by_range: None, - }); - } - if let Some(range) = scanner.leftover_typeof { - let snippet = &source[usize::from(range.start())..usize::from(range.end())]; - return Err(TranspileError { - kind: TranspileErrorKind::Bug, - message: format!("transpiler failed to lower `typeof` syntax `{snippet}`"), - output_range: Some(range), - by_range: None, - }); - } - - // a `.by` source holds back python's two `match` checks, because a bare - // `case A:` name may be an enum member of the subject rather than the - // capture it looks like. by here every name is spelled out, so the checks - // apply again — and a `match` that fails them is python that will not parse - if let Some(error) = first_invalid_match_statement(parsed.suite()) { - return Err(TranspileError { - kind: TranspileErrorKind::InvalidSyntax, - message: format!("transpiler produced invalid Python: {error}"), - output_range: Some(error.range), - by_range: None, - }); - } - verify_runtime_helpers(parsed.suite(), &author.reads)?; verify_no_helper_name_clash(parsed.suite(), author)?; verify_no_shadowed_helper(parsed.suite(), author) @@ -1418,48 +1567,6 @@ fn verify_runtime_helpers(suite: &[Stmt], written: &HashSet) -> Result<( } } -/// The first `match` in `suite` that python's own parse-time checks reject. -fn first_invalid_match_statement( - suite: &[ruff_python_ast::Stmt], -) -> Option { - use ruff_python_ast::visitor::{Visitor, walk_stmt}; - use ruff_python_parser::semantic_errors::{SemanticSyntaxChecker, SemanticSyntaxError}; - - struct Scanner { - error: Option, - } - impl<'ast> Visitor<'ast> for Scanner { - fn visit_stmt(&mut self, stmt: &'ast ruff_python_ast::Stmt) { - if self.error.is_some() { - return; - } - if let ruff_python_ast::Stmt::Match(match_stmt) = stmt { - self.error = SemanticSyntaxChecker::python_match_statement_errors( - match_stmt, - // the checks are version-independent; `match` itself needs - // 3.10, which the parse above has already settled - ruff_python_ast::PythonVersion::latest(), - ) - .into_iter() - .next(); - if self.error.is_some() { - return; - } - } - walk_stmt(self, stmt); - } - } - - let mut scanner = Scanner { error: None }; - for stmt in suite { - scanner.visit_stmt(stmt); - if scanner.error.is_some() { - break; - } - } - scanner.error -} - /// Map a byte offset in the generated python to a byte range in the original /// `.by` source, via the line table. Returns the full `.by` line's range (line /// granularity is all the table provides today; see the sourcemap plan). @@ -1596,7 +1703,7 @@ pub fn optional_marker_edits(source: &str) -> Vec<(TextRange, String)> { ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); - let model = ty_python_semantic::SemanticModel::new( + let model = ty_python_semantic::LoweringModel::new( &db, ty_python_semantic::Db::program_file(&db, file), ); @@ -1627,7 +1734,7 @@ pub fn reverse_transpile(source: &str, config: &Config) -> Result Option { + let parsed = ruff_python_parser::parse_unchecked_source(source, PySourceType::BasedPython); + SurfaceSyntax::find(parsed.suite(), SurfaceSyntax::CLEANED_UP).map(|(kind, _)| kind) + } + + #[test] + fn an_anonymous_named_tuple_type_is_left_over() { + assert_eq!( + spells("def f(p: list[(a: int, b: str)]) -> None: ...\n"), + Some(SurfaceSyntax::AnonymousNamedTuple) + ); + } + + #[test] + fn an_anonymous_named_tuple_value_is_left_over() { + assert_eq!( + spells("def f() -> object:\n return [(a=1, b=2)]\n"), + Some(SurfaceSyntax::AnonymousNamedTuple) + ); + } + + #[test] + fn an_inline_protocol_type_is_left_over() { + assert_eq!( + spells("def f(p: int | protocol(a: int)) -> None: ...\n"), + Some(SurfaceSyntax::InlineProtocol) + ); + } + + #[test] + fn an_inline_protocol_value_is_left_over() { + assert_eq!( + spells("class A:\n P = protocol(a: int)\n"), + Some(SurfaceSyntax::InlineProtocol) + ); + } + + /// phase 3 reads the output as python, which rejects every basedpython-only form, so one + /// an earlier pass left behind fails the transpile rather than reaching python + #[test] + fn phase_3_rejects_a_leftover_basedpython_form() { + for output in [ + "a = b = 1\nprint(a === b)\n", + "a = b = 1\nprint(a !== b)\n", + "x: (a: int)\n", + "x = (a=1)\n", + "P = protocol(a: int)\n", + "y = 1\nx: typeof y = y\n", + ] { + let error = super::verify_syntax(output, &super::AuthorNames::none()) + .expect_err("a leftover basedpython form must not pass phase 3"); + assert_eq!( + error.kind, + super::TranspileErrorKind::InvalidSyntax, + "{output:?}" + ); + assert!( + error.message.contains("is not valid in .py files"), + "{output:?}: {}", + error.message + ); + } + } + + /// python that names its own named tuple and protocol, and builds plain tuples + const PYTHON: &str = concat!( + "from typing import NamedTuple, Protocol\n", + "\n", + "class P(NamedTuple):\n", + " a: int\n", + "\n", + "class Q(Protocol):\n", + " a: int\n", + "\n", + "def f(p: P = P(1), q: tuple[int, int] = (1, 2)) -> P:\n", + " return P(*q[:1])\n", + ); + + #[test] + fn python_spells_none_of_it() { + assert_eq!(spells(PYTHON), None); + } + + fn round(source: &str) -> CleanupRound { + let quotes = WrittenQuotes::default(); + run_cleanup_round( + source, + WrittenNames::new(source), + "es, + &Config::test_default(), + ) + .expect("the round should succeed") + } + + /// the speedup: an output with nothing left to lower is passed over before any db is built + #[test] + fn a_round_over_python_builds_no_db() { + assert_eq!(round(PYTHON), CleanupRound::Skipped); + } + + #[test] + fn a_round_over_a_leftover_lowers_it() { + let CleanupRound::Lowered(lowered) = round("def f(p: (a: int)) -> int:\n return p.a\n") + else { + panic!("the leftover anonymous named tuple was not lowered"); + }; + assert_eq!(spells(&lowered), None, "{lowered}"); + } + + /// whether the db reads `file` as spelling an anonymous named tuple anywhere + fn spells_anonymous_named_tuple(db: &ty_project::TestDb, file: File) -> bool { + let module = ruff_db::parsed::parsed_module( + db, + ty_python_semantic::Db::program_file(db, file).python_file(db), + ) + .load(db); + SurfaceSyntax::find(module.suite(), &[SurfaceSyntax::AnonymousNamedTuple]).is_some() + } + + /// the cleanup passes over an output that spells no anonymous named tuple without asking + /// the checker which of its plain tuples it reads as one, which holds only while nothing + /// else its db reads spells one either + #[test] + fn vendored_stubs_spell_no_anonymous_named_tuple() { + let (mut db, _) = make_in_memory_db(""); + db.write_file("/control.byi", "def f(p: (a: int, b: str)) -> None: ...\n") + .expect("write file failed"); + let control = system_path_to_file(&db, "/control.byi").expect("file not in db"); + assert!( + spells_anonymous_named_tuple(&db, control), + "the scan must see a stub that does spell one" + ); + + let mut scanned = 0; + let mut spelling = Vec::new(); + let mut directories = vec![VendoredPathBuf::from("")]; + while let Some(directory) = directories.pop() { + let entries: Vec<_> = ruff_db::Db::vendored(&db) + .read_directory(&directory) + .collect(); + for entry in entries { + match entry.file_type() { + FileType::Directory => directories.push(entry.into_path()), + FileType::File => { + if !matches!(entry.path().extension(), Some("pyi" | "byi")) { + continue; + } + let file = vendored_path_to_file(&db, entry.path()) + .expect("a listed vendored file exists"); + scanned += 1; + if spells_anonymous_named_tuple(&db, file) { + spelling.push(entry.path().to_string()); + } + } + } + } + } + assert!(scanned > 0, "no vendored stub was scanned"); + assert!( + spelling.is_empty(), + "these vendored stubs spell an anonymous named tuple, so the cleanup in \ + `run_cleanup_round` can no longer pass over an output that spells none without \ + asking the checker: {spelling:?}" + ); + } +} diff --git a/crates/by_transforms/src/reverse_transforms/auto_quote.rs b/crates/by_transforms/src/reverse_transforms/auto_quote.rs index a7056c46f8..b1ce518639 100644 --- a/crates/by_transforms/src/reverse_transforms/auto_quote.rs +++ b/crates/by_transforms/src/reverse_transforms/auto_quote.rs @@ -137,9 +137,11 @@ mod tests { ); } + /// a string in a base's type argument is a forward reference in python and a literal type + /// in basedpython, so it comes back as the reference it is, whatever it names #[test] - fn other_strings_unchanged() { - check("class A(list[\"B\"]): ...\n", "class A(list[\"B\"])\n"); + fn a_forward_reference_to_another_name_is_unquoted_too() { + check("class A(list[\"B\"]): ...\n", "class A(list[B])\n"); } #[test] diff --git a/crates/by_transforms/src/reverse_transforms/forward_references.rs b/crates/by_transforms/src/reverse_transforms/forward_references.rs index 0c10f7cdd6..89d8c26a45 100644 --- a/crates/by_transforms/src/reverse_transforms/forward_references.rs +++ b/crates/by_transforms/src/reverse_transforms/forward_references.rs @@ -13,33 +13,56 @@ //! inside the unquoted text is a forward reference too, and is unquoted the same //! way //! -//! the positions are the annotation positions: parameters, returns, variables -//! (local ones included — the reader of a `.by` file reads them all the same), -//! `TypeAlias` values and pep 695 bounds and defaults. inside one, the -//! arguments of `Literal[…]` and the metadata of `Annotated[…]` are values, and -//! stay strings. a string whose text is not a single expression, or is a bare -//! tuple — which means something else once it is not a string — is left as it -//! is +//! the positions are the ones where basedpython reads a string as a literal type, which +//! the forward transpile writes as `Literal[…]`: the very strings that lowering wraps +//! ([`literal_type_strings`]), so that a forward reference python wrote comes back from a +//! round trip as the reference it was. a string left a string here would come back a +//! `Literal`. inside the unquoted text a string is a forward reference in every type +//! position but the arguments of `Literal[…]` and the metadata of `Annotated[…]`. a +//! string whose text is not a single expression, or is a bare tuple — which means +//! something else once it is not a string — is left as it is use ruff_python_ast::Expr; -use ruff_python_parser::{parse_expression, parse_module}; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; +use ruff_python_parser::parse_expression; use ruff_text_size::{Ranged, TextRange}; -use crate::transforms::source_util::for_each_annotation_in_stmt; +use crate::transforms::literal_types::literal_type_strings; use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr}; -/// `source` with each string annotation replaced by the expression it spells +/// `source` with each string the forward transpile would write as a literal type replaced by +/// the expression it spells pub(crate) fn unquote_forward_references(source: &str) -> String { - let Ok(parsed) = parse_module(source) else { + let (db, file) = crate::make_in_memory_db(source); + let program_file = ty_python_semantic::Db::program_file(&db, file); + let module = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); + if !module.errors().is_empty() { return source.to_owned(); + } + let model = ty_python_semantic::LoweringModel::new(&db, program_file); + let ranges = literal_type_strings(source, module.suite(), &model); + let mut strings = Strings { + ranges: &ranges, + collector: Collector { edits: Vec::new() }, }; - let mut collector = Collector { edits: Vec::new() }; - for stmt in parsed.suite() { - for_each_annotation_in_stmt(stmt, |annotation| { - walk_one_type_expr(annotation, &mut collector); - }); + strings.visit_body(module.suite()); + splice(source, strings.collector.edits) +} + +/// the strings at `ranges`, each unquoted as a forward reference +struct Strings<'a> { + ranges: &'a [TextRange], + collector: Collector, +} + +impl<'ast> SourceOrderVisitor<'ast> for Strings<'_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if expr.is_string_literal_expr() && self.ranges.contains(&expr.range()) { + self.collector.visit(expr, TypePos::Root); + } else { + walk_expr(self, expr); + } } - splice(source, collector.edits) } struct Collector { @@ -229,6 +252,74 @@ mod tests { ); } + /// every string the forward transpile would write as a literal type is a forward reference + /// in python, wherever it stands: a type argument in a value expression, a base's, a typing + /// construct's and a type variable's bound among them + #[test] + fn a_string_the_forward_transpile_reads_as_a_literal_type_is_unquoted() { + check( + indoc! {" + from typing import Callable, Generic, List, Optional, TypeVar, Union + + T = TypeVar(\"T\") + + class Foo(Generic[T]): + pass + + class Later: + pass + + Alias = List[\"Later\"] + UAlias = Union[\"Later\", int] + CAlias = Callable[[\"Later\"], \"Later\"] + MaybeLater = Optional[\"Later\"] + B = TypeVar(\"B\", bound=\"Later\") + + class Bar(Foo[\"Later\"]): + pass + "}, + indoc! {" + from typing import Callable, Generic, List, Optional, TypeVar, Union + + T = TypeVar(\"T\") + + class Foo(Generic[T]): + pass + + class Later: + pass + + Alias = List[Later] + UAlias = Union[Later, int] + CAlias = Callable[[Later], Later] + MaybeLater = Optional[Later] + B = TypeVar(\"B\", bound=Later) + + class Bar(Foo[Later]): + pass + "}, + ); + } + + /// the value of a `TypeAlias` is a type expression + #[test] + fn a_type_alias_value_is_unquoted() { + check( + "from typing import TypeAlias\n\nclass Later: ...\n\nX: TypeAlias = \"Later\"\n", + "from typing import TypeAlias\n\nclass Later\n\ntype X = Later\n", + ); + } + + /// a string that is no type expression — a subscript on a class that is not generic, an + /// enum's member lookup — is a value, and stays one + #[test] + fn a_member_lookup_stays_a_string() { + let source = + "from enum import Enum\n\nclass Colour(Enum):\n Red = 1\n\nred = Colour[\"Red\"]\n"; + let out = reverse_transpile(source, &Config::test_default()).unwrap(); + assert!(out.contains("red = Colour[\"Red\"]"), "{out}"); + } + /// a bare tuple means something else once it is not a string #[test] fn a_bare_tuple_stays_a_string() { diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs index df1d8397af..12540b519b 100644 --- a/crates/by_transforms/src/runtime.rs +++ b/crates/by_transforms/src/runtime.rs @@ -20,12 +20,12 @@ //! a helper resolves to the top-level statement that binds its name, as our own //! parser reads the file. a comment written between two definitions falls outside //! both and never reaches the output; one inside a body is part of it. a -//! top-level statement that binds nothing, such as -//! `_by_forward_operators(_LazyAttr)`, travels with the definition above it +//! top-level statement that binds nothing, such as a call that finishes setting up +//! a class, travels with the definition above it //! //! # dependencies //! -//! helpers call each other: `_lazy_attr` resolves through `_lazy_module`, +//! helpers call each other: `_LazyFrom` resolves through `_lazy_module`, //! `_soundness_parametric` through `_parametric_is`. what a helper needs is read //! out of the calls in its body, so asking for one yields a set that runs @@ -73,6 +73,7 @@ helpers! { SOUNDNESS_PARAMETRIC = "_soundness_parametric", SOUNDNESS_ITER_P = "_soundness_iter_p", SOUNDNESS_AITER_P = "_soundness_aiter_p", + SOUNDNESS_ONCE_BOUND = "_soundness_once_bound", CHECKED_CAST = "_checked_cast", TRY_CAST = "_try_cast", CHECKED_CAST_PRED = "_checked_cast_pred", @@ -102,9 +103,11 @@ helpers! { OPTIONAL = "Optional", FORCE_UNWRAP = "_force_unwrap", MAIN_ARGS = "_by_main_args", + DEFAULT = "_by_default", RAISES = "_by_raises", LAZY_MODULE = "_lazy_module", - LAZY_ATTR = "_lazy_attr", + LAZY_FROM_HOLDER = "_LazyFrom", + LAZY_FROM = "_lazy_from", TY_EXT_MARKER = "_TyExtMarker", CHARACTER = "Character", MATCH_MISS = "_by_match_miss", @@ -1194,48 +1197,6 @@ mod tests { assert_eq!(DISCARD.name(), ty_python_semantic::DISCARD_ADAPTER); } - /// the proxy's operator forwarding is read by nothing; it has to travel with - /// the class it patches, or the proxy is left with no operators at all - #[test] - fn set_up_code_travels_with_its_definition() { - let pasted = inline( - [LAZY_ATTR], - &[], - crate::transforms::repeated_underscore::WrittenNames::new(""), - ) - .concat(); - assert!(pasted.contains("class _LazyAttr:"), "{pasted}"); - assert!( - pasted.contains("_by_forward_operators(_LazyAttr)"), - "{pasted}" - ); - assert!(pasted.contains("def _lazy_module("), "{pasted}"); - } - - /// the proxy's three overrides of `object`'s own members are reported by - /// `missing-override-decorator`, and the emitted python is read by this project's own - /// checker. `@override` is `typing.override`, which arrived in 3.12, and the polyfill - /// runs on 3.9 — so each carries the suppression the decorator would have been - #[test] - fn the_proxies_overrides_do_not_report_in_the_emitted_python() { - let pasted = inline( - [LAZY_ATTR], - &[], - crate::transforms::repeated_underscore::WrittenNames::new(""), - ) - .concat(); - for member in ["__class__", "__setattr__", "__delattr__"] { - let line = pasted - .lines() - .find(|line| line.trim_start().starts_with(&format!("def {member}("))) - .unwrap_or_else(|| panic!("`{member}` is no longer defined:\n{pasted}")); - assert!( - line.contains("# ty: ignore[missing-override-decorator]"), - "`{member}` lost its suppression: {line}" - ); - } - } - /// the reads a pasted definition renames are the names python itself looks up in the /// module's globals: its compiler's symbol table answers the same question, so the two /// are held to agree over the whole runtime. with no python to ask, there is nothing @@ -1287,7 +1248,7 @@ print(" ".join(sorted(found))) /// in a function of its own leaves them reading it as written #[test] fn a_pasted_definition_reads_a_builtin_the_module_binds_under_its_own_name() { - let module = "def getattr(o, n): return 42\ndef f(format): return format\n"; + let module = "def getattr(o, n): return 42\ndef f(sorted): return sorted\n"; let parsed = ruff_python_parser::parse_unchecked_source( module, ruff_python_ast::PySourceType::Python, @@ -1295,15 +1256,12 @@ print(" ".join(sorted(found))) let code = crate::transforms::repeated_underscore::CodeNames::of(parsed.suite()); let written = crate::transforms::repeated_underscore::WrittenNames::new(module).with_code(&code); - let pasted = inline([LAZY_ATTR], parsed.suite(), written).concat(); + let pasted = inline([LAZY_FROM_HOLDER], parsed.suite(), written).concat(); assert!( - pasted.contains("v = getattr2(m, self._by_attr)") && !pasted.contains("getattr(m,"), - "{pasted}" - ); - assert!( - pasted.contains("lambda s, f: format(s._by_resolve(), f)"), + pasted.contains("return getattr2(m, attr)") && !pasted.contains("getattr(m,"), "{pasted}" ); + assert!(pasted.contains("return sorted(names)"), "{pasted}"); } /// a function or class that declares a builtin's name `global` and binds it rebinds the @@ -1318,7 +1276,7 @@ print(" ".join(sorted(found))) " for getattr in [len]:\n", " pass\n", "def local():\n", - " format = 1\n", + " sorted = 1\n", ); let parsed = ruff_python_parser::parse_unchecked_source( module, @@ -1327,15 +1285,12 @@ print(" ".join(sorted(found))) let code = crate::transforms::repeated_underscore::CodeNames::of(parsed.suite()); let written = crate::transforms::repeated_underscore::WrittenNames::new(module).with_code(&code); - let pasted = inline([LAZY_ATTR], parsed.suite(), written).concat(); + let pasted = inline([LAZY_FROM_HOLDER], parsed.suite(), written).concat(); assert!( - pasted.contains("v = getattr2(m, self._by_attr)") && !pasted.contains("getattr(m,"), + pasted.contains("return getattr2(m, attr)") && !pasted.contains("getattr(m,"), "{pasted}" ); // a local binding leaves the module's alone - assert!( - pasted.contains("lambda s, f: format(s._by_resolve(), f)"), - "{pasted}" - ); + assert!(pasted.contains("return sorted(names)"), "{pasted}"); } } diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py index f671a46f46..a4ed952803 100644 --- a/crates/by_transforms/src/runtime/_by_runtime.py +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -32,6 +32,26 @@ async def _soundness_aiter(_it, _t): yield _soundness_check(_x, _t) +# a check in a function on a class the module binds further down, or only on some +# paths. every binding the module makes of each of `_names` binds the class the check +# names, so once they are all bound `_target` builds the target the checker named, and +# `_check` is made against it; before then the value passes unchecked. the names are +# looked up in the globals of the module the check is written in, which the lambda +# `_target` carries +def _soundness_once_bound(_names, _v, _check, _target, *_rest): + _globals = _target.__globals__ + # a deferred `from` import is bound once its statement has run, which its + # module's `_LazyFrom` knows, and is in the namespace only once something has + # read it. the `_LazyFrom` is the module's `__getattr__`'s receiver, asked by + # what it holds rather than by its class: this check is often pasted in by + # another pass than the import's, and each paste has a class of its own + _deferred = getattr(getattr(_globals.get("__getattr__"), "__self__", None), "__by_from__", {}) + for _name in _names: + if _name not in _globals and _deferred.get(_name) is None: + return _v + return _check(_v, _target(), *_rest) + + # --- checked casts --------------------------------------------------------- # `value cast! T` verifies at run time and raises on a mismatch; `value cast? T` # yields None instead. the predicate forms serve any target the parametric @@ -90,6 +110,36 @@ def _force_unwrap(_v): return _v +# --- parameter defaults ---------------------------------------------------- +# a default re-evaluated per call has no value until a call evaluates it, so the +# signature holds a marker in its place and the body tests for it. each default has +# a marker of its own, bound at module level under a name of the transpiler's, and +# the body's guard tests for that one object. its `repr` is the default as +# basedpython reads it, which is what `inspect.signature` shows; `` stands +# for a parameter python cannot declare required where it stands. a marker handed +# back as an argument, as `apply_defaults` hands it, is the argument not given +# +# a marker reduces to the name it is bound under, so a pickled one is read back from +# that name, and a copy of one — which `copy` makes through the same reduction — is +# the marker itself. a tool that copies or pickles a signature's defaults hands back +# the object the guard tests for + + +class _by_default: + def __init__(self, source, module, name): + self._by_source = source + self.__module__ = module + self._by_name = name + + def __repr__(self): + if self._by_source is None: + return "" + return f"" + + def __reduce__(self): + return self._by_name + + # --- effects, entry points and loop capture -------------------------------- @@ -122,11 +172,16 @@ def _target(_generic, _args): _instance = None if _generic is None else _generic.instance _owner = _args[0] if _instance is None else _instance _values.extend(_type_argument(_owner, _param) for _param in _receiver) - return _resolve(*_values) - except Exception: + except (TypeError, KeyError, IndexError): # a parameter nothing answered for: what the declaration says without - # it is the ceiling + # it is the ceiling. binding the type arguments and reading one off a + # receiver raise `TypeError`, a parameter the specialization left + # unbound `KeyError`, and a method called with no receiver `IndexError` return _allowed + # the lambda only reads the values and the names the module binds, so + # anything it raises is a lowering that names something wrong, and is not + # mistaken for a value nothing answered for + return _resolve(*_values) def _check(_exc, _generic, _args): if not _by_isinstance(_exc, _target(_generic, _args), _allowed): @@ -924,6 +979,27 @@ def _by_method_matches(klass, name, params, ret): hints = typing.get_type_hints(method) except Exception: return False + import sys + + # a marker is the object its module binds under the name it carries. no attribute + # of its own says so — a class of the program's may define any of them — and a + # value is read without assuming it has any of them + def marker_source(value): + try: + module = sys.modules.get(value.__module__) + if module is None or getattr(module, value._by_name) is not value: + return False, None + return True, value._by_source + except Exception: + return False, None + + # a required parameter after a default carries a marker in place of a default + def has_default(p): + if p.default is inspect.Parameter.empty: + return False + marker, source = marker_source(p.default) + return not (marker and source is None) + positional = [ p for p in signature.parameters.values() if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) @@ -935,7 +1011,7 @@ def _by_method_matches(klass, name, params, ret): # extra positional parameters the protocol doesn't supply must be optional, # else a caller matching the protocol would fail to provide them for p in positional[len(params):]: - if p.default is inspect.Parameter.empty: + if not has_default(p): return False # likewise any required keyword-only parameter would break a protocol call for p in signature.parameters.values(): @@ -944,11 +1020,23 @@ def _by_method_matches(klass, name, params, ret): for (expected, variance), p in zip(params, positional): if p.name in hints: actual = hints[p.name] - elif p.default is not inspect.Parameter.empty: + elif not has_default(p): + return False + elif marker_source(p.default)[0]: + if variance == 3: + continue + # the default is evaluated per call, so the signature holds a marker + # and nothing at runtime records the type the checker read off the + # default. the marker's own type would answer for it, wrongly + raise TypeError( + "cannot check `" + klass.__qualname__ + "." + name + "` against a " + + "parameterized protocol: parameter `" + p.name + "` has no " + + "annotation, and its default is evaluated per call, so nothing " + + "records its type at runtime. annotate it" + ) + else: # a reified default gives the parameter's inferred type at runtime actual = type(p.default) - else: - return False if not _by_variance_ok(actual, expected, variance): return False if ret is not None: @@ -1179,18 +1267,25 @@ def _type_argument(owner, name): # --- lazy imports ---------------------------------------------------------- # below python 3.15 there is no `lazy` keyword, so an import is rewritten to a -# call: a module import becomes `_lazy_module`, and a `from` import a proxy that -# resolves the attribute the first time anything touches it +# call. a module import becomes `_lazy_module`, which binds the real module and +# runs its body on first attribute access +# +# a `from` import binds nothing where it is written. the module holds one +# `_LazyFrom` whose `__dict__` is the module's own namespace, the statement +# registers its names with it (`_lazy_from`), and every read of an imported +# name is written as a read off it. a name already in the namespace is a plain +# dict hit; a missing one is resolved from its module, stored in the namespace +# and returned, so from its first use on the name is the real object, exactly +# as PEP 810 makes it. sharing the namespace is also what keeps patching +# working: `mock.patch("app.svc.fetch")` writes `app.svc`'s namespace, which is +# what every read in `app.svc` looks in first # -# `__class__` makes `isinstance(proxy, C)` work, and `__instancecheck__` makes -# `isinstance(x, proxy)` work for a lazily-imported class (`isinstance` looks -# `__instancecheck__` up on `type(classinfo)`, which is `_LazyAttr`). -# `type(proxy)` and `proxy is x` cannot be fixed by any proxy — that is exactly -# why PEP 810 is a language feature — and are documented limits of this -# polyfill +# the module's `__getattr__` and `__dir__` serve the names nothing has read yet +# to other modules: `from app.svc import fetch`, `app.svc.fetch`, `dir(app.svc)` -import importlib as _by_il, importlib.util as _by_iu, sys as _by_sys +import builtins as _by_builtins, importlib as _by_il, importlib.util as _by_iu, sys as _by_sys +import _thread as _by_thread # a relative import names its module against the package doing the importing, # which only that module knows, so it hands `__package__` in def _lazy_module(name, package=None): @@ -1203,97 +1298,141 @@ def _lazy_module(name, package=None): return _by_il.import_module(name) spec = _by_iu.find_spec(name) if spec is None or spec.loader is None: - raise ImportError(f"No module named {name!r}", name=name) + raise ModuleNotFoundError(f"No module named {name!r}", name=name) spec.loader = _by_iu.LazyLoader(spec.loader) mod = _by_iu.module_from_spec(spec) spec.loader.exec_module(mod) return _by_sys.modules.setdefault(name, mod) -def _by_forward_operators(proxy): - import operator as op - def one(f): return lambda s: f(s._by_resolve()) - def two(f): return lambda s, o: f(s._by_resolve(), o) - def rtwo(f): return lambda s, o: f(o, s._by_resolve()) - for n, f in (("add", op.add), ("sub", op.sub), ("mul", op.mul), ("matmul", op.matmul), - ("truediv", op.truediv), ("floordiv", op.floordiv), ("mod", op.mod), - ("divmod", divmod), ("pow", op.pow), ("lshift", op.lshift), - ("rshift", op.rshift), ("and", op.and_), ("xor", op.xor), ("or", op.or_)): - setattr(proxy, "__" + n + "__", two(f)) - setattr(proxy, "__r" + n + "__", rtwo(f)) - for n in ("lt", "le", "eq", "ne", "gt", "ge"): - setattr(proxy, "__" + n + "__", two(getattr(op, n))) - for n, f in (("neg", op.neg), ("pos", op.pos), ("abs", abs), ("invert", op.inv), - ("len", len), ("iter", iter), ("next", next), ("bool", bool), - ("str", str), ("repr", repr), ("bytes", bytes), ("int", int), - ("float", float), ("complex", complex), ("index", op.index), - ("hash", hash), ("reversed", reversed)): - setattr(proxy, "__" + n + "__", one(f)) - setattr(proxy, "__getitem__", two(op.getitem)) - setattr(proxy, "__contains__", two(op.contains)) - setattr(proxy, "__delitem__", two(op.delitem)) - setattr(proxy, "__setitem__", lambda s, k, v: op.setitem(s._by_resolve(), k, v)) - setattr(proxy, "__format__", lambda s, f: format(s._by_resolve(), f)) - setattr(proxy, "__round__", lambda s, *a: round(s._by_resolve(), *a)) - setattr(proxy, "__enter__", lambda s: s._by_resolve().__enter__()) - setattr(proxy, "__exit__", lambda s, *a: s._by_resolve().__exit__(*a)) - -class _LazyAttr: - __slots__ = ("_by_mod", "_by_attr", "_by_val", "_by_has") - def __init__(self, mod, attr): - object.__setattr__(self, "_by_mod", mod) - object.__setattr__(self, "_by_attr", attr) - object.__setattr__(self, "_by_val", None) - object.__setattr__(self, "_by_has", False) - def _by_resolve(self): - if not self._by_has: - m = _lazy_module(self._by_mod) - try: - v = getattr(m, self._by_attr) - except AttributeError: - # a submodule rather than an attribute: `urllib/__init__.py` never - # imports `parse`, and cpython binds it only because `__import__` is - # handed a fromlist. reading the attribute alone never triggers that - try: - v = _by_il.import_module(self._by_mod + "." + self._by_attr) - except ImportError: - # worded as the import machinery words it, down to the module's - # file: a `from x import y` that fails is something programs catch - # and report, so the report must not say where the import was - # written. `name_from` is left off — cpython's own constructor - # only took it from 3.12, and this polyfill runs on 3.9 - p = getattr(m, "__file__", None) - raise ImportError("cannot import name " + repr(self._by_attr) + - " from " + repr(self._by_mod) + - ("" if p is None else " (" + p + ")"), - name=self._by_mod, path=p) from None - object.__setattr__(self, "_by_val", v) - object.__setattr__(self, "_by_has", True) - return self._by_val - # the three below really do override `object`'s, and `@override` is what says - # so — but it is `typing.override`, which arrived in 3.12, and this polyfill - # runs on 3.9. the suppression is what the decorator would have been - @property - def __class__(self): return self._by_resolve().__class__ # ty: ignore[missing-override-decorator] - def __getattr__(self, k): return getattr(self._by_resolve(), k) - def __setattr__(self, k, v): setattr(self._by_resolve(), k, v) # ty: ignore[missing-override-decorator] - def __delattr__(self, k): delattr(self._by_resolve(), k) # ty: ignore[missing-override-decorator] - def __call__(self, *a, **k): return self._by_resolve()(*a, **k) - def __class_getitem__(cls, k): return cls - def __instancecheck__(self, o): return isinstance(o, self._by_resolve()) - def __subclasscheck__(self, o): return issubclass(o, self._by_resolve()) - def __mro_entries__(self, bases): - r = self._by_resolve() - m = getattr(r, "__mro_entries__", None) - if m is None: return (r,) - return m(tuple(r if b is self else b for b in bases)) +# the names a module's deferred `from` imports bind. `names` is every one of them, +# so a read of one whose statement has not run yet is what python's global lookup +# gives — the builtin of that name, or a `NameError` — while any other attribute is +# an `AttributeError`, as tools that walk a module's namespace probing what they +# find expect. no deferred name begins with two underscores, so none can be +# mistaken for an attribute of the holder itself +class _LazyFrom: + __slots__ = ("__dict__", "__by_from__", "__by_resolving__") + + def __init__(self, namespace, names): + self.__dict__ = namespace + self.__by_from__ = dict.fromkeys(names) + self.__by_resolving__ = set() + + def __getattr__(self, name): + source = self.__by_from__.get(name) + if source is None: + if name in self.__by_from__: + # `if PY2: from itertools import izip as zip` leaves `zip` the + # builtin where the import did not run. the builtins are the ones + # the module runs with, a module or its dict + b = self.__dict__.get("__builtins__", _by_builtins) + b = b if isinstance(b, dict) else b.__dict__ + if name in b: + return b[name] + error = NameError(f"name {name!r} is not defined") + # `NameError.name` arrived in 3.10, and this polyfill runs on 3.9 + error.name = name + raise error + raise AttributeError(name) + # while a name resolves, the module's own `__getattr__` does not answer for + # it: `from . import helper` in a package's `__init__` reads `helper` off the + # package, which is this module, and has to fall through to importing the + # submodule rather than resolve itself again + key = (name, _by_thread.get_ident()) + self.__by_resolving__.add(key) + try: + value = _lazy_from_value(*source) + finally: + self.__by_resolving__.discard(key) + self.__dict__[name] = value + return value + + # the module's `__getattr__`: a deferred name another module reads before + # this one has + def __by_getattr__(self, name): + if ( + self.__by_from__.get(name) is None + or (name, _by_thread.get_ident()) in self.__by_resolving__ + ): + raise AttributeError( + f"module {self.__dict__.get('__name__')!r} has no attribute {name!r}" + ) + return getattr(self, name) + + # the module's `__dir__` + def __by_dir__(self): + names = set(self.__dict__) + names.update(name for name, source in self.__by_from__.items() if source is not None) + return sorted(names) + + +# what `from mod import attr` binds, read as python's import machinery reads it +def _lazy_from_value(mod, attr): + # the module is needed now, so it is imported as an ordinary import would, and + # whatever its body raises is the import's error, an `AttributeError` included + m = _by_sys.modules.get(mod) + if m is None: + m = _by_il.import_module(mod) + else: + # one another import made lazily runs its body on its first attribute read + m.__dict__ + try: + return getattr(m, attr) + except AttributeError: + pass + name = getattr(m, "__name__", mod) + # a package's submodule nothing has imported yet: `urllib/__init__.py` never + # imports `parse`, and cpython binds it because `__import__` is handed a + # fromlist, which imports it. a submodule that does not exist is the name + # missing; a module its body cannot import is the error that body raised + if hasattr(m, "__path__"): + submodule = name + "." + attr + try: + _by_il.import_module(submodule) + except ModuleNotFoundError as error: + if error.name != submodule or ( + submodule in _by_sys.modules and _by_sys.modules[submodule] is None + ): + raise + try: + return getattr(m, attr) + except AttributeError: + pass + value = _by_sys.modules.get(submodule) + if value is not None: + return value + # worded as the import machinery words it, down to the module's file: a + # `from x import y` that fails is something programs catch and report, so the + # report must not say where the import was written + path = getattr(m, "__file__", None) + where = path if isinstance(path, str) else "unknown location" + if getattr(getattr(m, "__spec__", None), "_initializing", False): + message = ( + f"cannot import name {attr!r} from partially initialized module {name!r} " + f"(most likely due to a circular import) ({where})" + ) + else: + message = f"cannot import name {attr!r} from {name!r} ({where})" + error = ImportError(message, name=name, path=path) + # `name_from` arrived in 3.12, and this polyfill runs on 3.9 + if hasattr(error, "name_from"): + error.name_from = attr + raise error -_by_forward_operators(_LazyAttr) -def _lazy_attr(mod, attr, package=None): +# a `from` import: registers each name it binds, as `{bound: imported}`, with the +# module's `_LazyFrom`. a relative import names its module against the package +# doing the importing, which only that module knows, so it hands `__package__` in +# +# a value the name already holds is from before the statement ran — a reload runs +# the module again over the namespace it had — and the statement replaces it +def _lazy_from(lazy, mod, names, package=None): if package is not None: mod = _by_iu.resolve_name(mod, package) - return _LazyAttr(mod, attr) + for bound, attr in names.items(): + lazy.__dict__.pop(bound, None) + lazy.__by_from__[bound] = (mod, attr) # a type-only marker for `ty_extensions` names, which have no runtime import diff --git a/crates/by_transforms/src/semantic_syntax.rs b/crates/by_transforms/src/semantic_syntax.rs new file mode 100644 index 0000000000..e68622d10c --- /dev/null +++ b/crates/by_transforms/src/semantic_syntax.rs @@ -0,0 +1,313 @@ +//! python's compile-time checks on the emitted module +//! +//! a module python parses can still be one it refuses to compile: an assignment +//! expression in a comprehension's iterable, a parameter named twice, a `break` outside +//! a loop, a capture pattern that makes the remaining cases unreachable. the parser +//! reports these apart from its parse errors, through a checker that asks its caller +//! about the scope each node sits in. this answers it for the transpiled output, so a +//! module python would refuse is a transpile error instead of a `SyntaxError` at import +//! +//! the questions that need a binding analysis — what a `global` or `nonlocal` names, +//! whether a name is a parameter — are answered so that the checker reports nothing +//! about them. what those checks find is written by the author, not by a lowering, and +//! `by check` has already reported it on the source + +use std::cell::RefCell; + +use ruff_python_ast::{Expr, PythonVersion, Stmt}; +use ruff_python_parser::semantic_errors::{ + LazyImportContext, SemanticSyntaxChecker, SemanticSyntaxContext, SemanticSyntaxError, + SemanticSyntaxErrorKind, YieldOutsideFunctionKind, +}; +use ruff_text_size::TextRange; + +use crate::transforms::python_scopes::{self, Scope, ScopeHooks, ScopeState}; + +/// the first error python's compile-time checks find in `suite`, on any version from +/// `min_version` on. the checks changed between versions in both directions — an async +/// comprehension inside a sync one compiles from 3.11, an assignment expression in an +/// annotation stopped compiling in 3.14 — so each version is checked +pub(crate) fn first_error( + suite: &[Stmt], + source: &str, + min_version: PythonVersion, + is_stub: bool, +) -> Option { + let future_annotations_or_stub = is_stub || has_future_annotations(suite); + PythonVersion::iter() + .filter(|version| *version >= min_version) + .find_map(|python_version| { + let mut checker = Checker { + syntax: SemanticSyntaxChecker::new(), + errors: RefCell::default(), + source, + python_version, + future_annotations_or_stub, + }; + python_scopes::walk(suite, &mut checker); + checker.errors.into_inner().into_iter().next() + }) +} + +fn has_future_annotations(suite: &[Stmt]) -> bool { + suite.iter().any(|stmt| { + matches!(stmt, Stmt::ImportFrom(import) + if import.module.as_deref() == Some("__future__") + && import.names.iter().any(|alias| alias.name.as_str() == "annotations")) + }) +} + +struct Checker<'src> { + syntax: SemanticSyntaxChecker, + errors: RefCell>, + source: &'src str, + python_version: PythonVersion, + future_annotations_or_stub: bool, +} + +impl ScopeHooks<'_> for Checker<'_> { + fn stmt(&mut self, stmt: &Stmt, state: &ScopeState) { + let mut syntax = std::mem::take(&mut self.syntax); + syntax.visit_stmt( + stmt, + &Context { + checker: self, + state, + }, + ); + self.syntax = syntax; + } + + fn expr(&mut self, expr: &Expr, state: &ScopeState) { + let mut syntax = std::mem::take(&mut self.syntax); + syntax.visit_expr( + expr, + &Context { + checker: self, + state, + }, + ); + self.syntax = syntax; + } +} + +/// the checker's questions, answered at one node +struct Context<'a, 'src> { + checker: &'a Checker<'src>, + state: &'a ScopeState, +} + +impl Context<'_, '_> { + /// the scopes around the node, innermost first, past the annotation and type + /// parameter scopes, which the checker's questions look through + fn scopes(&self) -> impl Iterator { + self.state + .scopes() + .iter() + .rev() + .filter(|scope| !matches!(scope, Scope::Annotation | Scope::TypeParams)) + } +} + +impl SemanticSyntaxContext for Context<'_, '_> { + fn future_annotations_or_stub(&self) -> bool { + self.checker.future_annotations_or_stub + } + + fn lazy_import_context(&self) -> Option { + match self.scopes().next() { + Some(Scope::Function { .. }) => return Some(LazyImportContext::Function), + Some(Scope::Class) => return Some(LazyImportContext::Class), + _ => {} + } + self.state + .in_try() + .then_some(LazyImportContext::TryExceptBlocks) + } + + fn python_version(&self) -> PythonVersion { + self.checker.python_version + } + + fn source(&self) -> &str { + self.checker.source + } + + fn global(&self, _name: &str) -> Option { + None + } + + fn has_nonlocal_binding(&self, _name: &str) -> bool { + true + } + + fn in_async_context(&self) -> bool { + for scope in self.scopes() { + match scope { + Scope::Class | Scope::Lambda => return false, + Scope::Function { is_async } => return *is_async, + _ => {} + } + } + false + } + + fn in_await_allowed_context(&self) -> bool { + for scope in self.scopes() { + match scope { + Scope::Class => return false, + Scope::Function { .. } + | Scope::Lambda + | Scope::Comprehension { + generator: true, .. + } => return true, + _ => {} + } + } + false + } + + fn in_yield_allowed_context(&self) -> bool { + matches!( + self.scopes().next(), + Some(Scope::Function { .. } | Scope::Lambda) + ) + } + + fn in_sync_comprehension(&self) -> bool { + self.scopes().any(|scope| { + matches!( + scope, + Scope::Comprehension { + generator: false, + is_async: false + } + ) + }) + } + + fn in_class_body_comprehension(&self) -> bool { + for scope in self.scopes() { + match scope { + Scope::Comprehension { .. } => {} + Scope::Class => return true, + _ => return false, + } + } + false + } + + fn in_module_scope(&self) -> bool { + self.state.scopes().is_empty() + } + + fn in_function_scope(&self) -> bool { + matches!( + self.scopes().next(), + Some(Scope::Function { .. } | Scope::Lambda) + ) + } + + fn def_is_method(&self) -> bool { + // a definition is checked before the scope it opens is entered + matches!(self.state.scopes().last(), Some(Scope::Class)) + } + + fn in_generator_context(&self) -> bool { + self.scopes().any(|scope| { + matches!( + scope, + Scope::Comprehension { + generator: true, + .. + } + ) + }) + } + + fn in_notebook(&self) -> bool { + false + } + + fn report_semantic_error(&self, error: SemanticSyntaxError) { + // an `await` outside a function is reported as outside an async function too, + // which says the same thing + if matches!( + error.kind, + SemanticSyntaxErrorKind::YieldOutsideFunction(YieldOutsideFunctionKind::Await) + ) { + return; + } + self.checker.errors.borrow_mut().push(error); + } + + fn in_loop_context(&self) -> bool { + self.state.in_loop() + } + + fn is_bound_parameter(&self, _name: &str) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use ruff_python_ast::{PySourceType, PythonVersion}; + use ruff_python_parser::parse_unchecked_source; + + fn first_error(source: &str, min_version: PythonVersion) -> Option { + let parsed = parse_unchecked_source(source, PySourceType::Python); + super::first_error(parsed.suite(), source, min_version, false) + .map(|error| error.to_string()) + } + + /// what python refuses to compile, whatever scope it is written in + #[test] + fn refuses_what_python_does_not_compile() { + for source in [ + "[x for x in (t := xs)]\n", + "def f():\n return [x for y in z for x in (lambda: (t := 1))()]\n", + "class C:\n ys = [(t := x) for x in xs]\n", + "class C:\n ys = [x for x in xs if [(t := 1) for _ in z]]\n", + "def f(a, a):\n pass\n", + "def f():\n return [(yield) for x in xs]\n", + "class C:\n yield 1\n", + "break\n", + "for x in xs:\n pass\nelse:\n continue\n", + "while x:\n def f():\n break\n", + "for x in xs:\n class C:\n continue\n", + "class C[T]((t := 1)):\n pass\n", + ] { + assert!( + first_error(source, PythonVersion::PY313).is_some(), + "accepted: {source}" + ); + } + } + + /// and what it compiles + #[test] + fn accepts_what_python_compiles() { + for source in [ + "def f():\n return [(t := x) for x in xs]\n", + "class C:\n y = (t := 1)\n ys = [(lambda: (t := x))() for x in xs]\n", + "class C:\n def f(self, y=(t := 1)):\n return [(u := x) for x in xs]\n", + "def f():\n yield 1\n g = lambda: (yield)\n", + "async def f():\n return [await x for x in xs], (await x for x in xs)\n", + "def f():\n return (await x for x in xs)\n", + "x = [x for x in ([t for t in (f(),) if t is not None] or [[]])[0]]\n", + "for x in xs:\n if x:\n break\n try:\n continue\n finally:\n pass\n", + "while x:\n with y:\n break\nelse:\n pass\n", + "class C[T](b[([t for t in (f(),) if t is not None] or [0])[0]]):\n pass\n", + ] { + assert_eq!(first_error(source, PythonVersion::PY313), None, "{source}"); + } + } + + /// a check that changed between versions applies on every version the output runs on + #[test] + fn checks_every_version_the_output_runs_on() { + let source = "def f(x: (t := int)):\n pass\n"; + assert!(first_error(source, PythonVersion::PY313).is_some()); + } +} diff --git a/crates/by_transforms/src/source_map.rs b/crates/by_transforms/src/source_map.rs index 40ef4fb861..03a85e61bb 100644 --- a/crates/by_transforms/src/source_map.rs +++ b/crates/by_transforms/src/source_map.rs @@ -65,13 +65,28 @@ impl Replacement { if start >= end { return; } + self.push_copied(&source[start..end], start); + } + + /// Append `text`, copied from the source at `start`. + fn push_copied(&mut self, text: &str, start: usize) { let continues = self.runs.last().is_some_and(|&(at, origin)| { matches!(origin, Origin::Copied(from) if from + (self.text.len() - at) == start) }); if !continues { self.runs.push((self.text.len(), Origin::Copied(start))); } - self.text.push_str(&source[start..end]); + self.text.push_str(text); + } + + /// Append `other`, run by run, so each keeps where it came from. + pub(crate) fn append(&mut self, other: &Replacement) { + for (text, origin) in other.runs() { + match origin { + Origin::Generated(anchor) => self.push_generated(text, anchor), + Origin::Copied(start) => self.push_copied(text, start), + } + } } /// The replacement text. diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index fe19648202..77d759a584 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -1,16 +1,17 @@ use std::cell::{Cell, RefCell}; use ruff_diagnostics::{Edit, Fix}; -use ruff_python_ast::{Expr, PythonVersion, Stmt}; +use ruff_python_ast::{Expr, ExprSubscript, ExprTuple, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::Config; -use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; +use crate::transforms::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::transforms::callable::CallableSyntax; use crate::transforms::optional_type; +use crate::transforms::quotes::WrittenQuotes; use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{ - Recurse, RootKind, TypeExprVisitor, TypePos, walk_type_positions, + Recurse, RootKind, TypeExprVisitor, TypePos, callable_parameter_type, walk_type_positions, }; use crate::type_info::TypeInfo; @@ -49,17 +50,19 @@ pub(crate) struct TupleLiteralType<'src> { /// has to carry the fold rather than re-emit the operation symbolic_substitutions: &'src [(TextRange, String)], edits: Vec, + templates: Vec<(TextRange, Vec)>, } impl<'src> TupleLiteralType<'src> { fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: &Config, symbolic_substitutions: &'src [(TextRange, String)], ) -> Self { - let mut leaves = CallableSyntax::new(source, written, config).with_types(types); + let mut leaves = CallableSyntax::new(source, written, quotes, config).with_types(types); for (range, rendered) in symbolic_substitutions { leaves.add_substitution(*range, rendered.clone()); } @@ -72,6 +75,7 @@ impl<'src> TupleLiteralType<'src> { leaves: RefCell::new(leaves), symbolic_substitutions, edits: Vec::new(), + templates: Vec::new(), } } @@ -196,19 +200,7 @@ impl<'src> TupleLiteralType<'src> { // parameter list, not a tuple type. Lower to `Callable[[int, // str], R]` (list form) so the runtime / type-checker reads // it as parameters - let is_callable_subscript = matches!(s.value.as_ref(), - Expr::Name(n) if n.id.as_str() == "Callable" - ) || matches!(s.value.as_ref(), - Expr::Attribute(a) if a.attr.id.as_str() == "Callable" - ); - if is_callable_subscript - && let Expr::Tuple(slice_tuple) = s.slice.as_ref() - && !slice_tuple.parenthesized - && slice_tuple.elts.len() == 2 - && let (Some(params_expr), Some(returns_expr)) = - (slice_tuple.elts.first(), slice_tuple.elts.get(1)) - && let Expr::Tuple(params_tuple) = params_expr - && params_tuple.parenthesized + if let Some((params_tuple, returns_expr)) = callable_tuple_parameters(self.types, s) { let params_str = params_tuple .elts @@ -275,19 +267,12 @@ impl<'src> TupleLiteralType<'src> { /// - `*: T` / `*name: T` → dropped (Callable list has no variadic) /// - `**: T` / `**name: T` → dropped fn lower_callable_param(&self, elt: &Expr) -> String { - match elt { - Expr::Named(named) => { - if matches!(named.target.as_ref(), Expr::Starred(_)) { - return String::new(); - } - self.transform_annotation(&named.value) - .unwrap_or_else(|| self.fallback_src(&named.value)) - } - Expr::Starred(_) => String::new(), - _ => self - .transform_annotation(elt) - .unwrap_or_else(|| self.fallback_src(elt)), - } + callable_parameter_type(elt) + .map(|ty| { + self.transform_annotation(ty) + .unwrap_or_else(|| self.fallback_src(ty)) + }) + .unwrap_or_default() } /// Lowers one element of a parameter-shape tuple to a Python type @@ -382,6 +367,26 @@ impl TypeExprVisitor for TupleLiteralType<'_> { if matches!(expr, Expr::Starred(_)) { return Recurse::Descend; } + // basedpython: `Callable[(int, name: str), R]` writes the parameter list as a tuple. it + // becomes python's list, with each parameter's type passed through where it stands, so + // whatever lowers a parameter — a tuple type of its own, a forward reference quoted — + // composes inside the list + if let Expr::Subscript(s) = expr + && let Some((parameters, _)) = callable_tuple_parameters(self.types, s) + { + self.templates + .push((parameters.range(), callable_parameter_list(parameters))); + return Recurse::Descend; + } + // a generic's type arguments are lowered one by one, each where it stands, so a + // pass that writes one of them whole, a forward reference quoted where python reads + // it lazily, wraps that argument's lowering rather than landing inside a rewrite of + // the whole subscript. `X[()]` is python's spelling already + if let Expr::Subscript(s) = expr + && !matches!(s.slice.as_ref(), Expr::Tuple(t) if t.parenthesized && t.elts.is_empty()) + { + return Recurse::Descend; + } // `transform_annotation` is a deep recursive rewriter that produces // a single replacement string for the whole expression. emit the // edit at the expression's range and tell the walker to stop — @@ -405,14 +410,21 @@ impl TypeExprVisitor for TupleLiteralType<'_> { pub(crate) struct TupleLiteralTypePass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, } impl<'src> TupleLiteralTypePass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + config: Config, + ) -> Self { Self { source, written, + quotes, config, } } @@ -434,11 +446,13 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { let mut inner = TupleLiteralType::new( self.source, self.written, + self.quotes, types, &self.config, &symbolic_substitutions, ); walk_type_positions(stmts, Some(types), &mut inner); + ctx.template_edits.append(&mut inner.templates); for fix in inner.edits { for edit in fix.edits() { let range = edit.range(); @@ -466,6 +480,44 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { } } +/// `(int, name: str)`, a `Callable` parameter list written as a tuple, as python's list +/// `[int, str]`, each parameter's type passed through where it stands +fn callable_parameter_list(parameters: &ExprTuple) -> Vec { + let mut fragments = vec![Fragment::Lit("[".to_owned())]; + for (index, ty) in parameters + .elts + .iter() + .filter_map(callable_parameter_type) + .enumerate() + { + if index > 0 { + fragments.push(Fragment::Lit(", ".to_owned())); + } + fragments.push(Fragment::Src(ty.range())); + } + fragments.push(Fragment::Lit("]".to_owned())); + fragments +} + +/// the parameters and return of `Callable[(int, str), R]`, where basedpython writes the +/// parameter list as a tuple. which value is `Callable` is the checker's answer, however it is +/// spelled +fn callable_tuple_parameters<'a>( + types: &dyn TypeInfo, + subscript: &'a ExprSubscript, +) -> Option<(&'a ExprTuple, &'a Expr)> { + if types.is_callable_form(&subscript.value) + && let Expr::Tuple(slice_tuple) = subscript.slice.as_ref() + && !slice_tuple.parenthesized + && let [Expr::Tuple(params_tuple), returns_expr] = slice_tuple.elts.as_slice() + && params_tuple.parenthesized + { + Some((params_tuple, returns_expr)) + } else { + None + } +} + #[cfg(test)] mod tests { use crate::python_passthrough::unchanged; @@ -751,6 +803,55 @@ mod tests { ); } + /// `Callable` is recognised by what the name is bound to, so an alias and a + /// reassignment read their tuple as the parameter list too + #[test] + fn callable_under_another_name_reads_its_tuple_as_parameters() { + check( + indoc! {" + import typing + from typing import Callable as Fn + C = typing.Callable + a: Fn[(int, str), bool] + b: C[(int, str), bool] + "}, + indoc! {" + import typing + from typing import Callable as Fn + C = typing.Callable + a: Fn[[int, str], bool] + b: C[[int, str], bool] + "}, + ); + } + + #[test] + fn collections_abc_callable_reads_its_tuple_as_parameters() { + check( + "from collections.abc import Callable\na: Callable[(int, str), bool]\n", + "from collections.abc import Callable\na: Callable[[int, str], bool]\n", + ); + } + + /// a class of the program's own that happens to be named `Callable` is an + /// ordinary generic, whose tuple argument is a tuple type + #[test] + fn a_class_named_callable_keeps_its_tuple_argument() { + check( + indoc! {" + class Callable[A, B]: ... + a: Callable[(int, str), bool] + "}, + indoc! {" + from typing import TypeVar, Generic + _A = TypeVar(\"_A\") + _B = TypeVar(\"_B\") + class Callable(Generic[_A, _B]): ... + a: Callable[tuple[int, str], bool] + "}, + ); + } + #[test] fn callable_with_variadic_tuple_params() { // `*: T` in callable params is dropped — Callable's list form has diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index 7320570491..76c99f6252 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -6,6 +6,9 @@ //! across the module collapse to one class so structural equivalence is //! preserved at the type level. //! +//! a plain tuple literal the checker reads as an instance of an anonymous named +//! tuple — one standing wherever one is expected — is constructed as its class +//! //! Example: //! ```by //! def foo(x: (name: str, age: int)) -> (name: str, age: int): @@ -23,7 +26,7 @@ //! age: int //! //! def foo(x: _AnonNamedTuple_) -> _AnonNamedTuple_: -//! return ("asdf", 1) +//! return _AnonNamedTuple_("asdf", 1) //! //! a = _AnonNamedTuple_ //! ``` @@ -34,13 +37,15 @@ use std::hash::Hash; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt, StmtAnnAssign, StmtFunctionDef, StmtReturn}; +use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::Ranged; +use ty_python_semantic::AnonNamedTupleId; use crate::transforms::callable::CallableSyntax; use crate::transforms::generics::PolyfilledTypeParams; +use crate::transforms::quotes::WrittenQuotes; use crate::transforms::repeated_underscore::WrittenNames; -use crate::type_info::TypeInfo; +use crate::type_info::{AnonNamedTupleConstruction, AnonNamedTupleField, TypeInfo}; /// Synthetic field name for the i-th positional element of a mixed /// anonymous named tuple. `NamedTuple` disallows leading-underscore field @@ -94,6 +99,8 @@ impl Shape { pub(crate) struct AnonNamedTuple<'src> { source: &'src str, written: WrittenNames<'src>, + /// where a field type naming something the module binds is written as a string + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: crate::Config, /// One lowerer drives every field type in the run, so its imports and the @@ -115,7 +122,7 @@ pub(crate) struct AnonNamedTuple<'src> { /// emitted `NamedTuple` definition is valid Python. /// /// Stored as a `Vec` (`TextRange` isn't `Ord`) and walked linearly when - /// rendering. Inserts append; `render_subbed` skips entries that don't + /// rendering. Inserts append; `render_subbed_value` skips entries that don't /// fall within the requested range. range_to_class: Vec<(ruff_text_size::TextRange, String)>, /// Source range of every anonymous-named-tuple *value* form (`(name=v, @@ -136,10 +143,6 @@ pub(crate) struct AnonNamedTuple<'src> { typing_imports: std::cell::RefCell>, /// whether python evaluates the synthesized class's field annotations as it is made annotations_evaluated: bool, - /// Active function-scope return-annotation shape stack. Empty when not - /// inside a function. The innermost (last) entry governs how a `return` - /// statement inside the current function is coerced. - return_shape_stack: Vec>, /// Stack of typevar rename maps (`T` → `_T`) accumulated as the visitor /// descends into PEP 695 generics. Synthesized `NamedTuple` class bodies /// are hoisted to module scope, but the original source text references @@ -152,12 +155,15 @@ pub(crate) struct AnonNamedTuple<'src> { typevar_rename_stack: Vec>, /// the name the generics polyfill declares each type parameter under polyfilled: PolyfilledTypeParams, - /// Names bound to an anonymous named tuple by an alias — `P = (a: int)` or - /// `type P = (a: int)`. Coercion asks what the annotation *means*, not how - /// it is spelled, so a plain tuple literal under `x: P` is wrapped exactly - /// as one under `x: (a: int)` is. Populated in source order, so an alias - /// only applies below its own declaration. - aliases: HashMap, + /// the class emitted for each anonymous named tuple the checker identifies, so + /// that every type form it reads as one type, and every plain tuple literal it + /// reads as an instance of it, names one class + identity_classes: HashMap, + /// the plain tuple literals the checker reads as an anonymous named tuple, in + /// source order with an enclosing literal ahead of the ones inside it. each is + /// built as its class once the whole module has been visited, because the + /// type form that names the class may come after the literal + constructions: Vec, /// Hard transpile errors — each one aborts the whole transpilation so we /// never emit syntactically invalid Python. The transform aborts on the /// *first* failure during the visit pass; subsequent visits are no-ops. @@ -168,6 +174,7 @@ impl<'src> AnonNamedTuple<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: crate::Config, stmts: &[Stmt], @@ -176,8 +183,9 @@ impl<'src> AnonNamedTuple<'src> { polyfilled: PolyfilledTypeParams::decide(source, written, stmts, &config), source, written, + quotes, types, - callable: CallableSyntax::new(source, written, &config).with_types(types), + callable: CallableSyntax::new(source, written, quotes, &config).with_types(types), config, edits: Vec::new(), shapes: indexmap::IndexMap::new(), @@ -187,9 +195,9 @@ impl<'src> AnonNamedTuple<'src> { type_only_imports: std::cell::RefCell::default(), typing_imports: std::cell::RefCell::default(), annotations_evaluated: false, - return_shape_stack: Vec::new(), typevar_rename_stack: Vec::new(), - aliases: HashMap::new(), + identity_classes: HashMap::new(), + constructions: Vec::new(), errors: Vec::new(), } } @@ -213,26 +221,6 @@ impl<'src> AnonNamedTuple<'src> { &self.source[usize::from(range.start())..usize::from(range.end())] } - /// Render `range`'s source, substituting any anonymous-named-tuple - /// expression contained within `range` with its synthesized class name. - /// Used wherever a stretch of original source ends up in the transpiled - /// output (class field type, constructor argument, coerced literal) so - /// nested anon-NT regions don't leak through verbatim. - fn render_subbed(&self, range: ruff_text_size::TextRange) -> String { - let raw = self.render_subbed_with(range, &self.range_to_class); - let renamed = self.apply_typevar_renames(&raw); - // Synthesized NamedTuple classes are hoisted to module scope, but - // the renamed typevars (`_T`) get declared *after* them by the - // generics polyfill. Wrap typevar references in PEP 484 forward-ref - // quotes so the annotation is stored as a string and only resolved - // by type checkers, not at class-construction time. - if renamed != raw && !raw.is_empty() { - format!("\"{renamed}\"") - } else { - renamed - } - } - /// Lower a field's type expression to python source. /// /// The synthesized class body is hoisted out of the statement the field type @@ -243,15 +231,41 @@ impl<'src> AnonNamedTuple<'src> { fn render_field_type(&mut self, expr: &Expr) -> String { let lowered = self.callable.lower_type_expr(expr); let renamed = self.apply_typevar_renames(&lowered); - // same forward-reference quoting as `render_subbed`: a renamed typevar is - // declared below the hoisted class, so the annotation must stay a string - if renamed != lowered && !lowered.is_empty() { - format!("\"{renamed}\"") + // the class is hoisted ahead of everything the module binds, a renamed typevar + // among them, so a field naming one is a forward reference there and is quoted + // as one + let forward = renamed != lowered || self.reads_a_module_binding(expr); + if forward && !lowered.is_empty() { + self.quotes.quote(&renamed) } else { renamed } } + /// whether `expr` reads a name the module binds + fn reads_a_module_binding(&self, expr: &Expr) -> bool { + struct Finder<'a> { + types: &'a dyn TypeInfo, + found: bool, + } + impl<'ast> Visitor<'ast> for Finder<'_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && self.types.is_bound_globally(&name.id) + { + self.found = true; + } + walk_expr(self, expr); + } + } + let mut finder = Finder { + types: self.types, + found: false, + }; + finder.visit_expr(expr); + finder.found + } + /// Active typevar rename map, flattened from the stack with inner scopes /// shadowing outer ones. Empty when not inside a generic scope or when /// `min_version` is 3.12+ (where PEP 695 stays native and no renames @@ -277,10 +291,11 @@ impl<'src> AnonNamedTuple<'src> { super::source_util::rename_identifiers(text, &borrowed) } - /// Like `render_subbed` but substitutes value-form constructor calls - /// instead of bare class names. Used when an outer anon-NT *value* form - /// is rendering its argument list — a nested value form there must call - /// its inner constructor, not just reference the inner class. + /// Render `range`'s source with every anonymous named tuple inside it + /// substituted: a value form by its constructor call, a type form by its + /// class name. Used when an outer anon-NT *value* form is rendering its + /// argument list — a nested value form there must call its inner + /// constructor, not just reference the inner class. fn render_subbed_value(&self, range: ruff_text_size::TextRange) -> String { // Combine value-form constructor calls (preferred) with class-name // subs for any nested *type* form in the same range. Value renders @@ -301,20 +316,6 @@ impl<'src> AnonNamedTuple<'src> { self.sweep(range, subs) } - fn render_subbed_with( - &self, - range: ruff_text_size::TextRange, - table: &[(ruff_text_size::TextRange, String)], - ) -> String { - let mut subs: Vec<(ruff_text_size::TextRange, &str)> = table - .iter() - .filter(|(r, _)| r.start() >= range.start() && r.end() <= range.end()) - .map(|(r, n)| (*r, n.as_str())) - .collect(); - subs.sort_by_key(|(r, _)| r.start()); - self.sweep(range, subs) - } - fn sweep( &self, range: ruff_text_size::TextRange, @@ -463,13 +464,20 @@ impl<'src> AnonNamedTuple<'src> { Ok(Some(Shape { fields })) } - fn class_name_for(&mut self, shape: Shape, source_range: ruff_text_size::TextRange) -> String { - let name = if let Some(existing) = self.shapes.get(&shape) { + fn class_name_for( + &mut self, + shape: Shape, + source_range: ruff_text_size::TextRange, + id: Option, + ) -> String { + let name = if let Some(existing) = id.and_then(|id| self.identity_classes.get(&id)) { existing.clone() } else { - let n = shape.class_name(self.written); - self.shapes.insert(shape, n.clone()); - n + let name = self.shape_class_name(shape); + if let Some(id) = id { + self.identity_classes.insert(id, name.clone()); + } + name }; // Track this anon-NT's source range → class name so any *outer* anon-NT // that contains this one can substitute its source span when rendering @@ -490,7 +498,8 @@ impl<'src> AnonNamedTuple<'src> { return; } }; - let class_name = self.class_name_for(shape, tuple.range()); + let id = self.types.anon_named_tuple_id(tuple); + let class_name = self.class_name_for(shape, tuple.range(), id); self.needs_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( class_name, @@ -508,7 +517,7 @@ impl<'src> AnonNamedTuple<'src> { } }; // Render constructor call: `ClassName(v1, v2, ...)` using positional - // args sourced from each field's value expression. We `render_subbed` + // args sourced from each field's value expression. We `render_subbed_value` // so a nested anon-NT inside any value gets substituted with its own // class-constructor call rather than emitted verbatim. let mut value_args: Vec = Vec::with_capacity(tuple.elts.len()); @@ -519,7 +528,9 @@ impl<'src> AnonNamedTuple<'src> { }; value_args.push(self.render_subbed_value(value_expr.range())); } - let class_name = self.class_name_for(shape, tuple.range()); + // a value form is left out of the checker's identities: its class is the one + // its own field values make, which a type form's need not be + let class_name = self.class_name_for(shape, tuple.range(), None); self.needs_import = true; let constructor_call = format!("{class_name}({})", value_args.join(", ")); self.range_to_value_render @@ -530,171 +541,6 @@ impl<'src> AnonNamedTuple<'src> { ))); } - /// Rewrite a `return ...` statement inside a function annotated with an - /// anonymous-named-tuple return type, when the returned value is a plain - /// tuple literal of matching arity. The plain tuple is wrapped as a - /// constructor call so runtime field access works. - fn rewrite_return_coercion(&mut self, ret: &StmtReturn) { - let Some(Some(shape)) = self.return_shape_stack.last().cloned() else { - return; - }; - let Some(value) = ret.value.as_ref() else { - return; - }; - if let Expr::Tuple(tuple) = value.as_ref() { - self.wrap_plain_tuple(tuple, shape); - } - } - - /// Walk an `x: T = ...` statement and coerce plain tuple literals on the - /// RHS when the annotation is an anonymous named tuple (directly or - /// inside a recognized container generic). - fn rewrite_ann_assign_coercion(&mut self, stmt: &StmtAnnAssign) { - let Some(value) = stmt.value.as_deref() else { - return; - }; - let Some(target) = self.coercion_target(&stmt.annotation) else { - return; - }; - self.apply_coercion(&target, value); - } - - fn return_shape_for(&mut self, func: &StmtFunctionDef) -> Option { - // Errors in shape extraction surface through the visitor's own pass - // over this annotation tuple — silently treat them as "no coercion" - // here so we don't double-report. - self.annotated_shape(func.returns.as_deref()?) - } - - /// Extract a coercion target from an annotation expression. Returns the - /// expected shape and the surrounding container so the caller knows - /// whether to wrap the value directly or to wrap each element of an - /// outer collection literal. - fn coercion_target(&mut self, annotation: &Expr) -> Option { - // a declaration modifier keeps the declared type in a marker subscript - // (`let x: T` parses as `x: __let__[T]`), and the annotation this reads - // is the one the source wrote - if let Some(declared) = ruff_python_ast::helpers::declaration_annotation_type(annotation) { - return self.coercion_target(declared); - } - // Direct: `x: (name: T, ...)` or `x: Alias` - if let Some(shape) = self.annotated_shape(annotation) { - return Some(CoercionTarget { - shape, - container: ContainerKind::Direct, - }); - } - // Containers: `x: list[(name: T, ...)]` and friends. The annotation - // is `Subscript(value=Name("list"), slice=Tuple{is_anon_named_tuple=true})`. - if let Expr::Subscript(sub) = annotation { - let Expr::Name(name) = sub.value.as_ref() else { - return None; - }; - let kind = match name.id.as_str() { - "list" | "List" => ContainerKind::ListLike, - "set" | "Set" | "frozenset" | "FrozenSet" => ContainerKind::SetLike, - _ => return None, - }; - let shape = self.annotated_shape(sub.slice.as_ref())?; - return Some(CoercionTarget { - shape, - container: kind, - }); - } - None - } - - /// The anonymous-named-tuple shape an annotation denotes — written out, or - /// reached through an alias declared above it. - fn annotated_shape(&mut self, annotation: &Expr) -> Option { - match annotation { - Expr::Tuple(t) => self.extract_type_shape(t).ok().flatten(), - Expr::Name(name) => self.aliases.get(name.id.as_str()).cloned(), - _ => None, - } - } - - /// Record `P = (name: T, ...)` / `type P = (name: T, ...)` so a later - /// annotation spelling `P` coerces the same way the written-out form does. - fn record_alias(&mut self, name: &Expr, value: &Expr) { - let (Expr::Name(target), Expr::Tuple(t)) = (name, value) else { - return; - }; - if !t.is_anon_named_tuple { - return; - } - // the tuple has already been visited, so a shape error is reported there - if let Ok(Some(shape)) = self.extract_type_shape(t) { - self.aliases.insert(target.id.to_string(), shape); - } - } - - /// Wrap a plain tuple literal `(v1, v2, ...)` as a constructor call to the - /// synthesized `NamedTuple` class for `shape`. No-op if the tuple is - /// already a basedpython anon form, has wrong arity, or contains starred - /// elements. - fn wrap_plain_tuple(&mut self, tuple: &ruff_python_ast::ExprTuple, shape: Shape) { - if tuple.is_anon_named_tuple || tuple.is_anon_named_tuple_value { - return; - } - if tuple.elts.len() != shape.fields.len() { - return; - } - if tuple.elts.iter().any(Expr::is_starred_expr) { - return; - } - // Build the constructor call. Each argument is rendered against the - // shape's corresponding field type so that a plain tuple element - // whose target field is itself a synthesized `NamedTuple` class is - // recursively wrapped — otherwise `record.point.x` would fail at - // runtime because the inner plain tuple doesn't carry field names. - self.needs_import = true; - let args: Vec = tuple - .elts - .iter() - .zip(shape.fields.iter()) - .map(|(elt, (_, field_type))| self.render_arg_against_field_type(elt, field_type)) - .collect(); - let name = if let Some(existing) = self.shapes.get(&shape) { - existing.clone() - } else { - let n = shape.class_name(self.written); - self.shapes.insert(shape, n.clone()); - n - }; - let constructor_call = format!("{name}({})", args.join(", ")); - self.edits.push(Fix::safe_edit(Edit::range_replacement( - constructor_call, - tuple.range(), - ))); - } - - /// Render a constructor argument. If `field_type` is the name of a - /// synthesized `NamedTuple` class and the argument is a plain tuple - /// matching that shape, recursively wrap it so nested field-name access - /// (`outer.inner.x`) keeps working at runtime. Otherwise fall back to - /// `render_subbed` which substitutes any inner anon-NT spans. - fn render_arg_against_field_type(&self, elt: &Expr, field_type: &str) -> String { - if let Some(inner_shape) = self.shape_for_class_name(field_type) { - if let Expr::Tuple(t) = elt { - if !t.is_anon_named_tuple - && !t.is_anon_named_tuple_value - && t.elts.len() == inner_shape.fields.len() - && !t.elts.iter().any(Expr::is_starred_expr) - { - let inner_args: Vec = t - .elts - .iter() - .zip(inner_shape.fields.iter()) - .map(|(e, (_, ft))| self.render_arg_against_field_type(e, ft)) - .collect(); - return format!("{}({})", field_type, inner_args.join(", ")); - } - } - } - self.render_subbed(elt.range()) - } - /// Push the typevar rename frame the generics polyfill gives a `[T, U, ...]` /// type-parameter list. Returns whether a frame was pushed so the caller /// knows to pop. No-op (returns false) for a list kept as native syntax @@ -720,137 +566,103 @@ impl<'src> AnonNamedTuple<'src> { .map(|(_, n)| n.clone()) } - fn shape_for_class_name(&self, name: &str) -> Option<&Shape> { - if !name.starts_with("_AnonNamedTuple_") { - return None; + /// the name of the class declaring `shape`, one class per shape + fn shape_class_name(&mut self, shape: Shape) -> String { + if let Some(existing) = self.shapes.get(&shape) { + return existing.clone(); } - self.shapes - .iter() - .find_map(|(shape, n)| (n == name).then_some(shape)) + let name = shape.class_name(self.written); + self.shapes.insert(shape, name.clone()); + name } - /// Apply the coercion target to the right-hand side of an assignment. - fn apply_coercion(&mut self, target: &CoercionTarget, value: &Expr) { - match target.container { - ContainerKind::Direct => { - if let Expr::Tuple(tuple) = value { - self.wrap_plain_tuple(tuple, target.shape.clone()); - } - } - ContainerKind::ListLike => { - let elts = match value { - Expr::List(list) => &list.elts[..], - // `[a, b]` is the common case; a parenthesized - // generator/expr would not match `list[X]` typewise. - _ => return, - }; - for elt in elts { - if let Expr::Tuple(tuple) = elt { - self.wrap_plain_tuple(tuple, target.shape.clone()); - } - } - } - ContainerKind::SetLike => { - let elts = match value { - Expr::Set(set) => &set.elts[..], - _ => return, - }; - for elt in elts { - if let Expr::Tuple(tuple) = elt { - self.wrap_plain_tuple(tuple, target.shape.clone()); - } + /// build each plain tuple literal the checker reads as an anonymous named tuple + /// as that class. called once the module has been visited + pub(crate) fn finish(&mut self) { + for construction in std::mem::take(&mut self.constructions) { + let class_name = match self.construction_class(&construction.target) { + Ok(name) => name, + Err(field) => { + self.errors.push(format!( + "the tuple at offset {} is built as an anonymous named tuple whose \ + field `{field}` has no type the emitted python can name; construct \ + it explicitly, or give it a `NamedTuple` class of its own", + u32::from(construction.range.start()) + )); + continue; } + }; + self.needs_import = true; + // a parenthesized literal is already the argument list of the call + if construction.parenthesized { + self.edits.push(Fix::safe_edit(Edit::insertion( + class_name, + construction.range.start(), + ))); + } else { + self.edits.push(Fix::safe_edit(Edit::insertion( + format!("{class_name}("), + construction.range.start(), + ))); + self.edits.push(Fix::safe_edit(Edit::insertion( + ")".to_owned(), + construction.range.end(), + ))); } } } -} -/// What surrounding shape a plain tuple literal at this position should be -/// coerced into. -#[derive(Clone, Debug)] -enum ContainerKind { - /// Annotation is the anon-NT directly: `x: (name: T, ...)`. - Direct, - /// Annotation wraps in a list-like generic: `list[X]`, `List[X]`. - ListLike, - /// Annotation wraps in a set-like generic: `set[X]`, `frozenset[X]`, - /// `Set[X]`, `FrozenSet[X]`. - SetLike, + /// the class a plain tuple literal is built as: the one emitted for a type form + /// the checker reads as the same type, or else one declared from the checker's + /// own field types. `Err` names a field whose type the output cannot spell + fn construction_class( + &mut self, + target: &AnonNamedTupleConstruction, + ) -> Result { + if let Some(existing) = self.identity_classes.get(&target.id) { + return Ok(existing.clone()); + } + let mut fields = Vec::with_capacity(target.fields.len()); + for (name, field) in &target.fields { + let field_type = match field { + AnonNamedTupleField::Spelled(spelled) => { + self.type_only_imports + .borrow_mut() + .extend(spelled.modules.iter().cloned()); + self.typing_imports + .borrow_mut() + .extend(spelled.typing_imports_in(self.written)); + spelled.annotation_in(self.written, self.annotations_evaluated) + } + AnonNamedTupleField::NamedTuple(nested) => self.construction_class(nested)?, + AnonNamedTupleField::Unspellable => return Err(name.clone()), + }; + fields.push((name.clone(), field_type)); + } + let class_name = self.shape_class_name(Shape { fields }); + self.identity_classes.insert(target.id, class_name.clone()); + Ok(class_name) + } } -#[derive(Clone, Debug)] -struct CoercionTarget { - shape: Shape, - container: ContainerKind, +/// a plain tuple literal to build as an anonymous named tuple +struct PendingConstruction { + range: ruff_text_size::TextRange, + parenthesized: bool, + target: AnonNamedTupleConstruction, } impl<'ast> Visitor<'ast> for AnonNamedTuple<'_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { - match stmt { - Stmt::FunctionDef(func) => { - // Visit non-body parts first so their anonymous-named-tuple - // expressions register synthesized classes before any body - // `return` statement is coerced. Walking via `walk_stmt` - // would visit the body before we've pushed the return shape - // onto the stack, so we manually drive the visit order. - let pushed = self.push_typevar_scope_from(func.type_params.as_deref()); - if let Some(tp) = func.type_params.as_deref() { - self.visit_type_params(tp); - } - self.visit_parameters(&func.parameters); - if let Some(returns) = func.returns.as_deref() { - self.visit_expr(returns); - } - let shape = self.return_shape_for(func); - self.return_shape_stack.push(shape); - for s in &func.body { - self.visit_stmt(s); - } - self.return_shape_stack.pop(); - for d in &func.decorator_list { - self.visit_decorator(d); - } - if pushed { - self.typevar_rename_stack.pop(); - } - } - Stmt::ClassDef(cls) => { - let pushed = self.push_typevar_scope_from(cls.type_params.as_deref()); - ruff_python_ast::visitor::walk_stmt(self, stmt); - if pushed { - self.typevar_rename_stack.pop(); - } - } - Stmt::Return(ret) => { - if let Some(v) = ret.value.as_deref() { - self.visit_expr(v); - } - self.rewrite_return_coercion(ret); - } - Stmt::AnnAssign(ann) => { - // Visit annotation first so any anon-NT inside it (including - // nested ones) registers its synthesized class. Then visit - // the RHS — any anon-NT children there register too. THEN - // coerce: the shape extracted from the annotation now uses - // the fully populated `range_to_class` so emitted class - // bodies don't embed raw inner anon-NT source. - self.visit_expr(&ann.annotation); - if let Some(v) = ann.value.as_deref() { - self.visit_expr(v); - } - self.rewrite_ann_assign_coercion(ann); - } - Stmt::Assign(assign) => { - walk_stmt(self, stmt); - if let [target] = assign.targets.as_slice() { - self.record_alias(target, &assign.value); - } - } - Stmt::TypeAlias(alias) => { - walk_stmt(self, stmt); - self.record_alias(&alias.name, &alias.value); - } - _ => walk_stmt(self, stmt), + let type_params = match stmt { + Stmt::FunctionDef(func) => func.type_params.as_deref(), + Stmt::ClassDef(cls) => cls.type_params.as_deref(), + _ => None, + }; + let pushed = self.push_typevar_scope_from(type_params); + walk_stmt(self, stmt); + if pushed { + self.typevar_rename_stack.pop(); } } @@ -868,6 +680,16 @@ impl<'ast> Visitor<'ast> for AnonNamedTuple<'_> { } return; } + if let Some(target) = self + .types + .anon_named_tuple_construction(tuple, self.config.min_version) + { + self.constructions.push(PendingConstruction { + range: tuple.range(), + parenthesized: tuple.parenthesized, + target, + }); + } } walk_expr(self, expr); } @@ -876,6 +698,7 @@ impl<'ast> Visitor<'ast> for AnonNamedTuple<'_> { pub(crate) struct AnonNamedTuplePass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: crate::Config, } @@ -883,11 +706,13 @@ impl<'src> AnonNamedTuplePass<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: crate::Config, ) -> Self { Self { source, written, + quotes, config, } } @@ -918,12 +743,19 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { types: &dyn TypeInfo, ctx: &mut super::ast_driver::PassContext, ) { - let mut inner = - AnonNamedTuple::new(self.source, self.written, types, self.config.clone(), stmts); + let mut inner = AnonNamedTuple::new( + self.source, + self.written, + self.quotes, + types, + self.config.clone(), + stmts, + ); inner.annotations_evaluated = ctx.annotations_evaluated; for stmt in stmts { inner.visit_stmt(stmt); } + inner.finish(); for err in std::mem::take(&mut inner.errors) { ctx.errors.push(err); } @@ -1033,6 +865,29 @@ mod tests { ); } + /// the class is hoisted ahead of everything the module binds, so a field naming + /// one of those is a forward reference there, and is quoted as one + #[test] + fn a_field_naming_a_module_binding_is_quoted() { + check( + indoc! {" + x: list[(a: F, b: int)] = [] + Alias = list[(a: F, b: int)] + class F: ... + "}, + indoc! {" + from typing import NamedTuple + class _AnonNamedTuple_58e90127(NamedTuple): + a: \"F\" + b: int + + x: \"list[_AnonNamedTuple_58e90127]\" = [] + Alias = list[\"_AnonNamedTuple_58e90127\"] + class F: ... + "}, + ); + } + /// the synthesized class body leaves the statement the field type was /// written in, so nothing else lowers it — `dynamic` reaching the output /// verbatim is a `NameError` at import, and a wrapped type is a syntax error @@ -1062,7 +917,8 @@ mod tests { 1, "expected a single class def, got: {out}" ); - assert!(out.contains(" m: TypeOf[b]"), "got: {out}"); + // `b` is bound below the hoisted class, so the field reads it as a forward reference + assert!(out.contains(" m: \"TypeOf[b]\""), "got: {out}"); } #[test] @@ -1133,10 +989,11 @@ mod tests { } #[test] - fn an_alias_below_its_use_does_not_coerce() { - // the table is built in source order, so a name is only an anon-NT alias - // from its declaration down — the same rule the binding itself follows. - // the annotation runs before `P` is bound, so it is quoted + fn an_alias_below_its_use_still_builds_the_class() { + // a `.by` annotation may name what is bound below it, and the checker + // reads `P` there as the alias. the class is declared above every + // statement, so building it there is fine; the annotation runs before `P` + // is bound, so it is quoted check( indoc! {" v: P = (\"a\", 1) @@ -1148,7 +1005,7 @@ mod tests { name: str age: int - v: \"P\" = (\"a\", 1) + v: \"P\" = _AnonNamedTuple_7bfb4772(\"a\", 1) P = _AnonNamedTuple_7bfb4772 "}, ); @@ -1682,9 +1539,7 @@ mod tests { } #[test] - fn plain_tuple_arg_unchanged() { - // Plain tuple literal at a call argument is not coerced — only return - // values inside anon-NT-returning functions are. + fn a_plain_tuple_argument_is_built_as_its_parameter() { let out = transpile( indoc! {" def foo(x: (name: str, age: int)) -> None: ... @@ -1693,7 +1548,29 @@ mod tests { &Config::test_default(), ) .unwrap(); - assert!(out.contains("foo((\"asdf\", 1))"), "got: {out}"); + assert!( + out.contains("foo(_AnonNamedTuple_7bfb4772(\"asdf\", 1))"), + "got: {out}" + ); + } + + #[test] + fn two_spellings_of_one_type_are_one_class() { + // the checker reads `(a: List[int])` and `(a: list[int])` as one type, so + // the literal it reads as that type is built as the one class both + // annotations name + let out = transpile( + indoc! {" + from typing import List + def take(p: (a: List[int])) -> None: ... + def give(q: (a: list[int])) -> None: ... + take(([1],)) + "}, + &Config::test_default(), + ) + .unwrap(); + assert_eq!(out.matches("(NamedTuple):").count(), 1, "got: {out}"); + assert!(out.contains("take(_AnonNamedTuple_"), "got: {out}"); } #[test] diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 535a9a9fad..8a86698bf7 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -34,6 +34,7 @@ use ruff_python_parser::parse_unchecked_source; use ruff_source_file::LineEnding; use ruff_text_size::{Ranged, TextRange, TextSize}; +use super::quotes::WrittenQuotes; use super::source_util::preamble_offset; use super::{ annotation, anon_named_tuple, auto_quote, build_stamps, callable, character_type, checked_cast, @@ -82,6 +83,37 @@ pub(crate) enum SemanticSource<'p> { pub(crate) enum Fragment { Lit(String), Src(TextRange), + /// a passthrough written as a string, the forward reference python reads lazily: a + /// type expression, with whatever lowers inside it lowered too. what it lowered to is + /// recorded, so the names it reads are bound where they are imported. only + /// [`WrittenQuotes::quote_span`](super::quotes::WrittenQuotes::quote_span) makes one + Quote(super::quotes::QuotedSpan), +} + +impl Fragment { + /// the source span this fragment passes through, quoted or not + fn passthrough(&self) -> Option { + match self { + Fragment::Lit(_) => None, + Fragment::Src(span) => Some(*span), + Fragment::Quote(quoted) => Some(quoted.range()), + } + } +} + +/// a string delimiter `text` does not already use, to write it as a string literal of +/// python's that holds exactly `text` +fn quote_delimiter(text: &str) -> &'static str { + ["\"", "'", "\"\"\"", "'''"] + .into_iter() + .find(|delimiter| { + !text.contains(delimiter) + && delimiter + .chars() + .next() + .is_none_or(|quote| !text.ends_with(quote)) + }) + .unwrap_or("\"") } /// Mutable state shared across every pass during a single transpile. @@ -183,6 +215,9 @@ pub(crate) struct PassContext { /// rendered text has to be spliced into the replacement or the operation is /// re-emitted from source and reaches the runtime pub(crate) symbolic_substitutions: Vec<(TextRange, String)>, + /// where python refuses an assignment expression in the module the type-aware + /// passes walk, which every pass binding a temporary asks + pub(crate) walrus_regions: super::walrus_scope::WalrusRegions, } /// a lowering whose edits another pass may leave out of its own, because that pass writes @@ -214,7 +249,9 @@ pub(crate) enum Lowering { RuntimeUnion, StatementExpression, SymbolicTypeOp, + TemplateType, TupleLiteralType, + TypedDictLiteral, Typeof, Unpack, VarianceStrip, @@ -380,10 +417,7 @@ enum SubPatch { /// whether a template spanning `start..end` only wraps it: its one passthrough /// is the whole span, and everything else it emits is text around it fn is_wrapper(frags: &[Fragment], start: usize, end: usize) -> bool { - let mut passthroughs = frags.iter().filter_map(|frag| match frag { - Fragment::Src(range) => Some(range), - Fragment::Lit(_) => None, - }); + let mut passthroughs = frags.iter().filter_map(Fragment::passthrough); passthroughs .next() .is_some_and(|range| usize::from(range.start()) == start && usize::from(range.end()) == end) @@ -401,10 +435,10 @@ fn is_wrapper(frags: &[Fragment], start: usize, end: usize) -> bool { /// /// A [`SubPatch::Relocating`] edit is the one thing that does *not* travel with /// that source. It is the stand-in a pass leaves at the span it moved a -/// construct away from — `_MISSING` where the default was written — and a -/// passthrough of exactly that span, from outside the template's own range, is -/// the place the construct moved *to*. Materializing the stand-in there too -/// would put `_MISSING` at both ends and the construct at neither, which is how +/// construct away from — the `_MISSING` marker where the default was written — +/// and a passthrough of exactly that span, from outside the template's own range, +/// is the place the construct moved *to*. Materializing the stand-in there too +/// would put the marker at both ends and the construct at neither, which is how /// a `decorator def` whose header template re-emits the whole signature came to /// emit `if tags is _MISSING: tags = _MISSING`. fn template_claimees( @@ -422,16 +456,12 @@ fn template_claimees( // a boundary insertion at a passthrough's end is left to // `apply_within`'s `include_end`, which knows whether an adjacent // span will re-emit it - let in_src = frags.iter().any(|frag| match frag { - Fragment::Lit(_) => false, - Fragment::Src(span) => { - let (span_start, span_end) = - (usize::from(span.start()), usize::from(span.end())); - s >= span_start - && e <= span_end - && !(matches!(all[m].2, SubPatch::Relocating(_)) - && (s, e) == (span_start, span_end)) - } + let in_src = frags.iter().filter_map(Fragment::passthrough).any(|span| { + let (span_start, span_end) = (usize::from(span.start()), usize::from(span.end())); + s >= span_start + && e <= span_end + && !(matches!(all[m].2, SubPatch::Relocating(_)) + && (s, e) == (span_start, span_end)) }); in_own || in_src }) @@ -458,6 +488,8 @@ fn rewrites_covered_source(range: TextRange, frags: &[Fragment], source: &str) - for frag in frags { let piece = match frag { Fragment::Lit(text) => text.as_str(), + // a quote writes delimiters no source has there + Fragment::Quote(_) => return true, Fragment::Src(span) => { match source.get(usize::from(span.start())..usize::from(span.end())) { Some(text) => text, @@ -481,6 +513,7 @@ fn rewrites_covered_source(range: TextRange, frags: &[Fragment], source: &str) - /// is charged to that offset in the line table: a hoisted `def` header, an /// injected keyword argument, a `nonlocal` line — none of them has a line of /// its own, and the construct the edit rewrites is the one they stand for. +#[expect(clippy::too_many_arguments)] fn materialize_fragments( out: &mut Replacement, frags: &[Fragment], @@ -489,10 +522,31 @@ fn materialize_fragments( contained: &[usize], anchor: usize, emitted: &mut [bool], + record_quote: &dyn Fn(&str), ) { for (i, frag) in frags.iter().enumerate() { match frag { Fragment::Lit(s) => out.push_generated(s, anchor), + Fragment::Quote(quoted_span) => { + let span = quoted_span.range(); + let mut quoted = Replacement::default(); + apply_within( + &mut quoted, + source, + usize::from(span.start()), + usize::from(span.end()), + all, + contained, + true, + emitted, + record_quote, + ); + record_quote(quoted.text()); + let delimiter = quote_delimiter(quoted.text()); + out.push_generated(delimiter, anchor); + out.append("ed); + out.push_generated(delimiter, anchor); + } Fragment::Src(span) => { // a zero-width insertion at this span's end is normally deferred // to the *next* `Src` span (which re-emits it at its start), so @@ -515,6 +569,7 @@ fn materialize_fragments( contained, include_end, emitted, + record_quote, ); } } @@ -540,6 +595,7 @@ fn apply_within( contained: &[usize], include_end: bool, emitted: &mut [bool], + record_quote: &dyn Fn(&str), ) { let mut cursor = s0; let mut k = 0; @@ -565,7 +621,7 @@ fn apply_within( .copied() .filter(|&m| all[m].0 >= s && all[m].1 <= e && all[m].0 != e) .collect(); - materialize_fragments(out, frags, source, all, &inner, s, emitted); + materialize_fragments(out, frags, source, all, &inner, s, emitted, record_quote); } } cursor = cursor.max(e); @@ -589,6 +645,7 @@ fn splice( sub_edits: Vec<(usize, usize, SubPatch)>, origins: Vec>, authors: &[Author], + record_quote: &dyn Fn(&str), ) -> (Vec<(usize, usize, Replacement)>, Vec) { let mut edits: Vec<(usize, usize, Replacement)> = Vec::new(); let mut emitted = vec![false; sub_edits.len()]; @@ -726,6 +783,7 @@ fn splice( &contained, start, &mut emitted, + record_quote, ); } } @@ -757,6 +815,7 @@ fn splice( &contained, start, &mut emitted, + record_quote, ); out } @@ -1123,7 +1182,7 @@ pub(crate) fn merge_into_own_imports(body: &mut String, imports: Vec) -> /// Coalesce repeated `from import X` lines into a single /// `from import X, Y, ...` line. Preserves any non-matching -/// lines (e.g. `import foo`, `_MISSING = object()`) in their original +/// lines (e.g. `import foo`, a synthesized class) in their original /// order. Names within a merged line are sorted and deduped fn merge_from_imports(lines: Vec) -> (Vec, Vec) { // preserve first-seen module order so tests that depend on specific @@ -1176,6 +1235,7 @@ fn merge_from_imports(lines: Vec) -> (Vec, Vec) { pub(crate) fn run_against_source<'a>( source: &'a str, written: repeated_underscore::WrittenNames, + quotes: &WrittenQuotes, config: &Config, project: SemanticSource<'_>, ) -> (Cow<'a, str>, Vec, Vec>) { @@ -1206,7 +1266,7 @@ pub(crate) fn run_against_source<'a>( ruff_db::parsed::parsed_module(sem_db, sem_db.program_file(sem_file).python_file(sem_db)) .load(sem_db); let semantic_model = - ty_python_semantic::SemanticModel::new(sem_db, sem_db.program_file(sem_file)); + ty_python_semantic::LoweringModel::new(sem_db, sem_db.program_file(sem_file)); // identity line table for the no-change early returns: stripping variance // is within-line, so every line still maps to itself @@ -1221,11 +1281,19 @@ pub(crate) fn run_against_source<'a>( let table = crate::source_map::line_table(cow.as_ref(), &[]); return (cow, Vec::new(), table); } + semantic_model.infer_in_check_order(); // the parentheses grouping an expression are visible in the tokens and // nowhere in the AST, so anything that needs them has to measure now — the // tokens go when the syntax tree is taken out of the parse let accessor_value_ranges = properties::collect_value_ranges(&parsed.syntax().body, parsed.tokens()); + // a default's marker shows the default on one line and without its comments, which + // only the tokens tell apart from the text of a string + let default_markers = mutable_defaults::DefaultMarkers::collect( + &parsed.syntax().body, + parsed.tokens(), + source_ref, + ); // a statement an AST pass rewrote is re-emitted in the source's own indentation, // since the lines it passes through keep theirs let indentation = ruff_python_codegen::Stylist::from_tokens(parsed.tokens(), source_ref) @@ -1352,12 +1420,22 @@ pub(crate) fn run_against_source<'a>( let dedent_string_pass = dedent_string::DedentString::new(source_ref); let super_keyword_pass = super_keyword::SuperKeyword::new(written); let postfix_await_pass = postfix_await::PostfixAwait::new(source_ref); - let mutable_defaults_pass = - mutable_defaults::MutableDefaultsPass::new(source_ref, written, config.is_stub); + let mutable_defaults_pass = mutable_defaults::MutableDefaultsPass::new( + source_ref, + written, + &default_markers, + config.is_stub, + ); let unique_loop_bindings_pass = unique_loop_bindings::UniqueLoopBindingsPass::new(source_ref, config.unique_loop_bindings); - let auto_quote_pass = auto_quote::AutoQuote::new(source_ref); - let init_method_pass = init_method::InitMethod::new(source_ref, written, config.clone()); + let auto_quote_pass = auto_quote::AutoQuote { quotes }; + let init_method_pass = init_method::InitMethod::new( + source_ref, + written, + quotes, + &default_markers, + config.clone(), + ); let properties_pass = properties::PropertiesPass::new( source_ref, written, @@ -1371,7 +1449,7 @@ pub(crate) fn run_against_source<'a>( return_value_use::collect(parsed_handle.suite(), &semantic_model), ); let raises_guard_pass = - raises_clause::RaisesGuardPass::new(source_ref, config.runtime_raises_checks); + raises_clause::RaisesGuardPass::new(source_ref, written, config.runtime_raises_checks); let type_fn_pass = type_fn::TypeFnPass::new(source_ref); let match_type_pass = match_type::MatchTypePass::new(source_ref, written); let modifiers_pass = modifiers::ModifiersPass::new(source_ref, written, config); @@ -1397,6 +1475,7 @@ pub(crate) fn run_against_source<'a>( let typed_dict_literal_pass = typed_dict_literal::TypedDictLiteralPass::new( source_ref, written, + quotes, config.clone(), parsed_handle.suite(), &semantic_model, @@ -1407,9 +1486,10 @@ pub(crate) fn run_against_source<'a>( kw_subscript::KwSubscriptPass::new(source_ref, written, config.min_version); let generic_call_pass = generic_call::GenericCallStripPass::new(source_ref); let reified_generic_pass = - reified_generic::ReifiedGenericPass::new(source_ref, config.min_version); + reified_generic::ReifiedGenericPass::new(source_ref, written, config.min_version); let reified_class_pass = reified_class::ReifiedClassPass::new(source_ref, config.min_version); - let type_reification_pass = type_reification::TypeReificationPass::new(config.min_version); + let type_reification_pass = + type_reification::TypeReificationPass::new(written, config.min_version); let visibility_rename_pass = visibility_rename::VisibilityRenamePass { written }; let parametric_is_pass = parametric_is::ParametricIsPass::new(source_ref, written); let implicit_typing_pass = implicit_typing::ImplicitTypingPass::new(); @@ -1422,13 +1502,14 @@ pub(crate) fn run_against_source<'a>( ); let template_type_pass = template_type::TemplateTypePass { written }; let tuple_types_pass = - annotation::TupleLiteralTypePass::new(source_ref, written, config.clone()); + annotation::TupleLiteralTypePass::new(source_ref, written, quotes, config.clone()); let literal_types_pass = literal_types::LiteralTypePass::new(source_ref, written, config.float_literals); - let callable_pass = callable::CallableSyntaxPass::new(source_ref, written, config.clone()); + let callable_pass = + callable::CallableSyntaxPass::new(source_ref, written, quotes, config.clone()); let protocol_type_pass = - protocol_type::ProtocolTypePass::new(source_ref, written, config.clone()); - let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref); + protocol_type::ProtocolTypePass::new(source_ref, written, quotes, config.clone()); + let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref, &comments); let force_unwrap_pass = force_unwrap::ForceUnwrapPass::new(source_ref); let flexible_keyword_pass = flexible_keyword::FlexibleKeywordPass; let some_ctor_pass = some_ctor::SomeCtorPass::new(); @@ -1437,9 +1518,10 @@ pub(crate) fn run_against_source<'a>( let optional_type_pass = optional_type::OptionalTypePass::new(source_ref, written, config.min_version); let runtime_union_pass = runtime_union::RuntimeUnionPass::new(written, config.min_version); - let generics_pass = generics::GenericPolyfillPass::new(source_ref, written, config.clone()); + let generics_pass = + generics::GenericPolyfillPass::new(source_ref, written, quotes, config.clone()); let soundness_pass = soundness::SoundnessPass::new(source_ref, written, config); - let checked_cast_pass = checked_cast::CheckedCastPass { written }; + let checked_cast_pass = checked_cast::CheckedCastPass { written, quotes }; let module_api_pass = module_api::ModuleApiPass::new(source_ref); let trailing_lambda_pass = trailing_lambda::TrailingLambdaPass::new(source_ref); let if_let_pass = if_let::IfLetPass::new(source_ref); @@ -1457,7 +1539,7 @@ pub(crate) fn run_against_source<'a>( let frameworks_pass = frameworks::FrameworksPass::new(source_ref); let variance_pass = decl_site_variance::VarianceStripPass::new(source_ref); let anon_named_tuple_pass = - anon_named_tuple::AnonNamedTuplePass::new(source_ref, written, config.clone()); + anon_named_tuple::AnonNamedTuplePass::new(source_ref, written, quotes, config.clone()); // Order matters: passes that read source ranges via `text_edits` mode // must run BEFORE passes that mutate the AST (which zero source ranges @@ -1721,7 +1803,7 @@ pub(crate) fn run_against_source<'a>( // …); receiver spans pass through as `Src` fragments, so sibling // lowerings inside compose &grapheme_string_pass, - // mutable defaults → `_MISSING` sentinel swap + body-prologue guard; + // mutable defaults → a `_MISSING` marker per default + body-prologue guard; // narrow edits, so the function body's own lowerings still apply &mutable_defaults_pass, // per-iteration loop bindings: a closure made in a loop body is applied @@ -1748,6 +1830,7 @@ pub(crate) fn run_against_source<'a>( // anon-NT spans generics polyfill leaked verbatim into class headers &anon_named_tuple_pass, ]; + ctx.walrus_regions = super::walrus_scope::WalrusRegions::collect(parsed_handle.suite()); for pass in type_aware { if config.is_stub && pass.runtime_only() { continue; @@ -2116,7 +2199,13 @@ pub(crate) fn run_against_source<'a>( ) })) .collect(); - let (spliced, lost) = splice(source_ref, sub_edits, origins, &authorship.authors); + let (spliced, lost) = splice( + source_ref, + sub_edits, + origins, + &authorship.authors, + &|quoted| quotes.record(quoted), + ); edits.extend(spliced); ctx.errors.extend(lost); // line table for the spliced body, built from the ascending edit list @@ -2253,7 +2342,7 @@ mod driver_tests { SubPatch::Text(edit.1.to_owned()), )); let origins = vec![Vec::new(); sub_edits.len()]; - splice(source, sub_edits, origins, &[]).1 + splice(source, sub_edits, origins, &[], &|_| {}).1 } /// the call of `stmt`, an assignment of one @@ -2356,6 +2445,7 @@ mod driver_tests { sub_edits, vec![vec![0], vec![1]], &[writer, inserter], + &|_| {}, ) .1 } @@ -2405,7 +2495,7 @@ mod driver_tests { (6, 7, SubPatch::Text(" | None".to_owned())), ]; let authors = [author(None, &[]), author(Some(Lowering::OptionalType), &[])]; - let errors = splice(source, sub_edits, vec![vec![0], vec![1]], &authors).1; + let errors = splice(source, sub_edits, vec![vec![0], vec![1]], &authors, &|_| {}).1; assert_eq!(errors, Vec::::new()); } @@ -2415,6 +2505,7 @@ mod driver_tests { let (out, _, _) = run_against_source( src, repeated_underscore::WrittenNames::new(src), + &WrittenQuotes::default(), &Config::test_default(), SemanticSource::Fresh(ruff_python_ast::PythonVersion::latest_ty()), ); @@ -2457,7 +2548,16 @@ mod driver_tests { Fragment::Lit("Y".to_owned()), ]; let mut out = Replacement::default(); - materialize_fragments(&mut out, &frags, source, &all, &[0], 0, &mut [false]); + materialize_fragments( + &mut out, + &frags, + source, + &all, + &[0], + 0, + &mut [false], + &|_| {}, + ); assert_eq!(out.text(), "[1])Y"); } @@ -2473,7 +2573,16 @@ mod driver_tests { Fragment::Src(TextRange::new(TextSize::from(3u32), TextSize::from(4u32))), ]; let mut out = Replacement::default(); - materialize_fragments(&mut out, &frags, source, &all, &[0], 0, &mut [false]); + materialize_fragments( + &mut out, + &frags, + source, + &all, + &[0], + 0, + &mut [false], + &|_| {}, + ); assert_eq!(out.text(), "[1])W"); } } @@ -2739,12 +2848,23 @@ mod subsumed_lowerings { Lowering::FloatConst, Lowering::Typeof, ]; + // the class is hoisted ahead of the module's own bindings, so a field reading one — + // the `y` a `typeof` names — is written as a forward reference + let (reads_binding, reads_none): (Vec, Vec) = fields + .iter() + .partition(|lowering| **lowering == Lowering::Typeof); cases.extend(leaves( "AnonNamedTuplePass", - &fields, + &reads_none, |leaf| format!("def f(x: (a: {leaf}, b: str)) -> None: ..."), |lowered| format!("a: {lowered}"), )); + cases.extend(leaves( + "AnonNamedTuplePass", + &reads_binding, + |leaf| format!("def f(x: (a: {leaf}, b: str)) -> None: ..."), + |lowered| format!("a: \"{lowered}\""), + )); cases.extend(leaves( "ProtocolTypePass", &fields, @@ -2826,15 +2946,172 @@ mod subsumed_lowerings { &["??"], ), case( - "NoneCoalescePass", - &[Lowering::NoneChain], - "class A:\n b: int? = None\n\ndef f(a: A?) -> int:\n return a?.b ?? 1\n", - &[], - &["?.", "??"], + "ParametricIsPass", + &[Lowering::VisibilityRename], + "private class Kept:\n pass\n\ndef f(x: object) -> bool:\n return x is Kept\n", + &["isinstance(x, _Kept)"], + &["isinstance(x, Kept)"], + ), + case( + "CheckedCastPass", + &[Lowering::VisibilityRename], + "private class Kept:\n pass\n\ndef f(x: object) -> object:\n return x cast! Kept\n", + &["_checked_cast(x, _Kept)"], + &["_checked_cast(x, Kept)"], ), ] } + /// a checked cast and a type test build their check from the checker's reading of the + /// target, so a lowering of the target as written is not what they print: the check + /// spells what the target means, or, where the operand's type cannot hold the target, + /// is settled without it + fn type_test_target_cases() -> Vec { + let cast = |operand: &str, target: &str| { + format!("y = 1\n\ndef f(x: {operand}) -> object:\n return x cast! {target}\n") + }; + let test = |operand: &str, target: &str| { + format!("y = 1\n\ndef f(x: {operand}) -> bool:\n return x is {target}\n") + }; + let never = "_checked_cast_pred(x, lambda _by_cast_value: False)"; + let mut cases = vec![ + case( + "CheckedCastPass", + &[Lowering::AnonNamedTuple], + &cast("str", "(a: int, b: str)"), + &[never], + &["(a: int"], + ), + case( + "CheckedCastPass", + &[Lowering::Callable], + &cast("str", "list[(int) -> str]"), + &[never], + &["(int) ->"], + ), + case( + "CheckedCastPass", + &[Lowering::DynamicKeyword], + &cast("object", "list[dynamic]"), + &["_checked_cast(x, list)"], + &["dynamic"], + ), + case( + "CheckedCastPass", + &[Lowering::FloatConst], + &cast("str", "float.inf"), + &[never], + &["float.inf"], + ), + case( + "CheckedCastPass", + &[Lowering::JustFloat], + &cast("int", "float"), + &[never], + &["cast! float"], + ), + case( + "CheckedCastPass", + &[Lowering::JustFloat], + &cast("object", "list[float]"), + &["_parametric_is_lenient(_by_cast_value, list[float], (0,))"], + &["list[JustFloat]"], + ), + case( + "CheckedCastPass", + &[Lowering::LiteralType], + &cast("str", "1"), + &[never], + &["Literal[1]"], + ), + case( + "CheckedCastPass", + &[Lowering::OptionalType], + &cast("object", "int?"), + &["(isinstance(_by_cast_value, int) or _by_cast_value is None)"], + &["int?"], + ), + case( + "CheckedCastPass", + &[Lowering::SymbolicTypeOp], + &cast("object", "typeof y + 1"), + &["_by_cast_value == 2)"], + &["typeof"], + ), + case( + "CheckedCastPass", + &[Lowering::TemplateType], + &cast("object", "f\"a{int}b\""), + &["_by_pattern_is(_by_cast_value, \"a(?:0|-?[1-9][0-9]*)b\")"], + &["f\"a{int}b\""], + ), + case( + "CheckedCastPass", + &[Lowering::TupleLiteralType], + &cast("object", "(int, str)"), + &["_parametric_is_lenient(_by_cast_value, tuple[int, str], (1,))"], + &["cast! (int, str)"], + ), + case( + "CheckedCastPass", + &[Lowering::TypedDictLiteral], + &cast("object", "{\"a\": int}"), + &["_checked_cast(x, dict)"], + &["{\"a\": int}"], + ), + case( + "CheckedCastPass", + &[Lowering::Typeof], + &cast("object", "typeof y"), + &["_by_cast_value == 1)"], + &["typeof"], + ), + ]; + cases.extend([ + case( + "ParametricIsPass", + &[Lowering::AnonNamedTuple], + &test("str", "(a: int, b: str)"), + &["return False"], + &["(a: int"], + ), + case( + "ParametricIsPass", + &[Lowering::Callable], + &test("str", "list[(int) -> str]"), + &["return False"], + &["(int) ->"], + ), + case( + "ParametricIsPass", + &[Lowering::DynamicKeyword], + &test("object", "list[dynamic]"), + &["isinstance(x, list)"], + &["dynamic"], + ), + case( + "ParametricIsPass", + &[Lowering::JustFloat], + &test("object", "list[float]"), + &["_parametric_is(x, list[float], (0,))"], + &["list[JustFloat]"], + ), + case( + "ParametricIsPass", + &[Lowering::Typeof], + &test("object", "typeof y"), + &["x == 1)"], + &["typeof"], + ), + ]); + cases + } + + #[test] + fn a_type_test_target_is_written_as_what_it_means() { + check(&type_test_target_cases(), PythonVersion::PY312); + } + #[test] fn a_type_expression_a_pass_moves_is_written_lowered() { check(&type_expression_cases(), PythonVersion::PY310); @@ -2856,6 +3133,7 @@ mod subsumed_lowerings { let cases: Vec = type_expression_cases() .into_iter() .chain(whole_construct_cases()) + .chain(type_test_target_cases()) .collect(); let mut failures = Vec::new(); diff --git a/crates/by_transforms/src/transforms/auto_quote.rs b/crates/by_transforms/src/transforms/auto_quote.rs index f909b96dfa..820fb5161d 100644 --- a/crates/by_transforms/src/transforms/auto_quote.rs +++ b/crates/by_transforms/src/transforms/auto_quote.rs @@ -21,35 +21,39 @@ //! ([`crate::annotations_evaluated`]): on python >= 3.14 they are deferred //! natively (PEP 649), and a `from __future__ import annotations` — written by //! the user, opted into, or written for a target below 3.10 — defers every one, -//! where a quote would reach `__annotations__` as a string inside a string. a class -//! *base*, and a value-position subscript in a class body (`list[A]()`), -//! evaluates while the class is being built on every version, so a -//! self-reference there is always quoted. a direct base (`class A(A):`) is -//! left alone — that is a runtime error regardless of quoting +//! where a quote would reach `__annotations__` as a string inside a string +//! +//! the checker reads two more kinds of type expression as deferred, though python +//! evaluates them where they stand on every version: the type arguments of a +//! specialization written as a value (`list[F]()`, `Alias = dict[str, F]`, a base +//! `Foo[F]`), and the type arguments of a typing construct assigned to a name +//! (`T = TypeVar("T", bound=F)`). which ones it read that way is its answer +//! ([`TypeInfo::is_read_deferred`]), and a forward reference in one is quoted too: +//! +//! `a = list[F](); class F: ...` → `a = list["F"]()` +//! +//! a position the checker reads where it is written — a base itself +//! (`class A(A)`), a reified class's type arguments — reports a name not yet bound +//! there, and is left alone //! //! an annotation holding a forward reference is quoted whole, as one wrapper //! template passing its source through. no lowering reaches past the annotation //! it lowers, so whatever the annotation becomes — a callable arrow, a `T?`, a //! renamed type parameter, even when a lowering rewrites the whole annotation -//! as text — ends up between the quotes. a base or a value-position subscript -//! cannot be a string, so there only the self-reference itself is quoted +//! as text — ends up between the quotes. a specialization cannot be a string, so +//! there only the type arguments holding a reference are quoted -use ruff_python_ast::visitor::{Visitor, walk_expr}; -use ruff_python_ast::{AnyParameterRef, Expr, ExprName, Stmt, StmtClassDef}; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{AnyParameterRef, Expr, ExprName, Stmt, TypeParams}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; -use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr}; +use super::quotes::WrittenQuotes; +use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr_with}; use crate::type_info::TypeInfo; pub(crate) struct AutoQuote<'src> { - source: &'src str, -} - -impl<'src> AutoQuote<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } - } + pub(crate) quotes: &'src WrittenQuotes, } impl TypeAwarePass for AutoQuote<'_> { @@ -61,13 +65,21 @@ impl TypeAwarePass for AutoQuote<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut walk = Walk { - source: self.source, + quotes: self.quotes, types, quote_annotations: ctx.annotations_evaluated, edits: Vec::new(), }; walk.block(stmts, Scope::Module); - ctx.template_edits.extend(walk.edits); + let mut deferred = DeferredReads { + quotes: self.quotes, + types, + edits: walk.edits, + }; + for stmt in stmts { + deferred.visit_stmt(stmt); + } + ctx.template_edits.extend(deferred.edits); } } @@ -80,7 +92,7 @@ enum Scope { } struct Walk<'a> { - source: &'a str, + quotes: &'a WrittenQuotes, types: &'a dyn TypeInfo, quote_annotations: bool, edits: Vec<(TextRange, Vec)>, @@ -95,7 +107,7 @@ impl Walk<'_> { fn stmt(&mut self, stmt: &Stmt, scope: Scope) { match stmt { - Stmt::ClassDef(class) => self.class(class), + Stmt::ClassDef(class) => self.block(&class.body, Scope::Class), Stmt::FunctionDef(function) => { if self.quote_annotations { for parameter in function @@ -153,7 +165,7 @@ impl Walk<'_> { let types = self.types; let is_forward = |name: &ExprName| types.is_forward_reference(name) == Some(true); let mut quoter = Quoter { - source: self.source, + quotes: self.quotes, is_forward: &is_forward, edits: &mut self.edits, skip_root: false, @@ -162,84 +174,117 @@ impl Walk<'_> { quoter.quote(annotation.range()); } } +} + +/// quotes the forward references in what the checker reads as deferred though python +/// evaluates it where it stands: the type arguments of a specialization written as a value +/// (`list[F]()`, `Alias = dict[str, F]`, a base `Foo[F]`), and the type-expression arguments of +/// a typing construct assigned to a name (`T = TypeVar("T", bound=F)`). which those are is the +/// checker's answer ([`TypeInfo::is_read_deferred`]), so a position it reads where it is +/// written — a base itself, a reified class's type arguments, the type an `is` tests for — is +/// left alone, and a name in it that is not yet bound stays the error the checker reports +struct DeferredReads<'a> { + quotes: &'a WrittenQuotes, + types: &'a dyn TypeInfo, + /// every quote written so far, the annotations' included: nothing inside one is evaluated + /// where it stands any more + edits: Vec<(TextRange, Vec)>, +} + +impl DeferredReads<'_> { + fn quoted_already(&self, range: TextRange) -> bool { + self.edits + .iter() + .any(|(quoted, _)| quoted.contains_range(range)) + } - /// a class's bases and the value-position subscripts in its body run while - /// the class is being built, whatever the version, and the class's own name - /// is not bound until it is - fn class(&mut self, class: &StmtClassDef) { - let class_name = class.name.id.as_str(); - let is_self = |name: &ExprName| name.id.as_str() == class_name; - if let Some(arguments) = &class.arguments { - for base in &arguments.args { - // a direct `class A(A)` base must not be quoted — that would - // mask a runtime error rather than fix it - let mut quoter = Quoter { - source: self.source, - is_forward: &is_self, - edits: &mut self.edits, - skip_root: true, - }; - walk_one_type_expr(base, &mut quoter); + /// quotes the type expression `expr` whole if it holds a forward reference. with + /// `skip_root`, `expr` is the specialization, which is evaluated where it stands, and each + /// of its type arguments is quoted that way instead + fn quote_forward_references(&mut self, expr: &Expr, skip_root: bool) { + let types = self.types; + let is_forward = |name: &ExprName| types.is_forward_reference(name) == Some(true); + let mut quoter = Quoter { + quotes: self.quotes, + is_forward: &is_forward, + edits: &mut self.edits, + skip_root, + }; + walk_one_type_expr_with(expr, Some(self.types), &mut quoter); + } +} + +impl<'a> Visitor<'a> for DeferredReads<'_> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + // a `type` statement's value is evaluated lazily, and the type parameters are the + // generics polyfill's to write + Stmt::TypeAlias(_) => {} + // a `TypedDict`'s `extra_items=` is a type expression python evaluates as the class + // is built + Stmt::ClassDef(class) => { + for keyword in class.keywords() { + if self.types.is_read_deferred(&keyword.value) { + self.quote_forward_references(&keyword.value, false); + } + } + walk_stmt(self, stmt); } + _ => walk_stmt(self, stmt), } - for stmt in &class.body { - let value = match stmt { - Stmt::Expr(node) => Some(node.value.as_ref()), - Stmt::Assign(node) => Some(node.value.as_ref()), - Stmt::AnnAssign(node) => node.value.as_deref(), - _ => None, - }; - if let Some(value) = value { - let mut quoter = Quoter { - source: self.source, - is_forward: &is_self, - edits: &mut self.edits, - skip_root: false, - }; - quoter.value_subscripts(value); + } + + // an annotation is quoted whole where python evaluates it, and is never evaluated + // elsewhere + fn visit_annotation(&mut self, _expr: &'a Expr) {} + + fn visit_type_params(&mut self, _type_params: &'a TypeParams) {} + + fn visit_expr(&mut self, expr: &'a Expr) { + if self.quoted_already(expr.range()) { + return; + } + match expr { + Expr::Subscript(subscript) if self.types.is_read_deferred(&subscript.slice) => { + self.quote_forward_references(expr, true); + } + Expr::Call(call) => { + for argument in self.types.call_type_expression_arguments(call) { + if self.types.is_read_deferred(argument) { + self.quote_forward_references(argument, false); + } + } } + _ => {} } - self.block(&class.body, Scope::Class); + // what a quote left out may hold more: the metadata of an `Annotated`, a call's + // other arguments + walk_expr(self, expr); } } /// quotes the forward references in one type expression struct Quoter<'a> { - source: &'a str, + quotes: &'a WrittenQuotes, is_forward: &'a dyn Fn(&ExprName) -> bool, edits: &'a mut Vec<(TextRange, Vec)>, - /// when walking a class base, the root expression must not be quoted even - /// if it's a bare self-reference — that would mask the runtime error + /// the root is the specialization whose type arguments are deferred, which is + /// evaluated where it stands itself skip_root: bool, } impl TypeExprVisitor for Quoter<'_> { + /// a type expression holding a forward reference is quoted whole, so whatever it lowers + /// to — a callable arrow, a `T?`, a tuple type, a literal type — ends up between the + /// quotes. quoting one name inside it would leave the construct around it evaluated fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { if std::mem::take(&mut self.skip_root) { - return match expr { - Expr::Subscript(_) | Expr::BinOp(_) => Recurse::Descend, - _ => Recurse::Stop, - }; - } - if !self.contains_forward_reference(expr) { - return Recurse::Stop; + return Recurse::Descend; } - match expr { - Expr::Tuple(_) => Recurse::Descend, - // a generic whose base is not itself a forward reference: quote the - // references inside its arguments at their own level - Expr::Subscript(subscript) if !self.contains_forward_reference(&subscript.value) => { - Recurse::Descend - } - // a union is quoted whole: quoting one arm alone would evaluate - // `str | NoneType` at runtime. anything else that holds a reference - // — a name, a generic rooted at one, an arrow type — is quoted whole - // as the reference it is - _ => { - self.quote(expr.range()); - Recurse::Stop - } + if self.contains_forward_reference(expr) { + self.quote(expr.range()); } + Recurse::Stop } } @@ -271,43 +316,9 @@ impl Quoter<'_> { finder.found } - /// `list[A]()` and similar — quote a self-reference inside a value-position - /// subscript on the LHS of a call. doesn't descend into call arguments - fn value_subscripts(&mut self, expr: &Expr) { - match expr { - Expr::Subscript(subscript) => { - walk_one_type_expr(subscript.slice.as_ref(), self); - self.value_subscripts(&subscript.value); - } - Expr::Call(call) => self.value_subscripts(&call.func), - Expr::Attribute(attribute) => self.value_subscripts(&attribute.value), - _ => {} - } - } - - /// wrap `range` in quotes, as one template passing the source through so - /// the lowerings inside it land between the quotes. the delimiter is one the - /// source it wraps does not already use fn quote(&mut self, range: TextRange) { - let text = &self.source[range]; - let delimiter = ["\"", "'", "\"\"\"", "'''"] - .into_iter() - .find(|delimiter| { - !text.contains(delimiter) - && delimiter - .chars() - .next() - .is_none_or(|quote| !text.ends_with(quote)) - }) - .unwrap_or("\""); - self.edits.push(( - range, - vec![ - Fragment::Lit(delimiter.to_owned()), - Fragment::Src(range), - Fragment::Lit(delimiter.to_owned()), - ], - )); + self.edits + .push((range, vec![self.quotes.quote_span(range)])); } } @@ -387,8 +398,280 @@ mod tests { #[test] fn nested_self_reference() { check( + indoc! {" + class Node[T]: ... + class Tree(Node[Tree]): ... + "}, + indoc! {" + from typing import TypeVar, Generic + _T = TypeVar(\"_T\") + class Node(Generic[_T]): ... + class Tree(Node[\"Tree\"]): ... + "}, + ); + } + + /// what the checker cannot resolve it cannot read as a specialization, so a + /// name in its subscript is left as it is + #[test] + fn an_unresolved_generic_quotes_nothing() { + check( + "class Tree(Node[Tree]): ...\n", "class Tree(Node[Tree]): ...\n", - "class Tree(Node[\"Tree\"]): ...\n", + ); + } + + /// a specialization written as a value reads its type arguments when it runs, + /// so a class defined further down is quoted wherever one stands + #[test] + fn a_later_class_in_a_value_specialization_is_quoted() { + check( + indoc! {" + from typing import Callable, Union + items = list[F]() + Alias = dict[str, F] + Maybe = Union[F, None] + Handler = Callable[[F], F] + nested = dict[str, list[F | None]]() + class F: ... + "}, + indoc! {" + from typing import Callable, Union + items = list[\"F\"]() + Alias = dict[str, \"F\"] + Maybe = Union[\"F\", None] + Handler = Callable[[\"F\"], \"F\"] + nested = dict[str, \"list[F | None]\"]() + class F: ... + "}, + ); + } + + /// a base's type arguments are a specialization like any other + #[test] + fn a_later_class_in_a_base_type_argument_is_quoted() { + check( + indoc! {" + class Base[T]: ... + class Bar(Base[F]): ... + class F: ... + "}, + indoc! {" + from typing import TypeVar, Generic + _T = TypeVar(\"_T\") + class Base(Generic[_T]): ... + class Bar(Base[\"F\"]): ... + class F: ... + "}, + ); + } + + /// the types a typing construct assigned to a name takes as call arguments + #[test] + fn a_later_class_in_a_typing_construct_is_quoted() { + check( + indoc! {" + from typing import NamedTuple, NewType, TypedDict, TypeVar + T = TypeVar(\"T\", bound=list[F]) + Id = NewType(\"Id\", F) + Pair = NamedTuple(\"Pair\", [(\"first\", F)]) + Movie = TypedDict(\"Movie\", {\"lead\": F}) + class F: ... + "}, + indoc! {" + from typing import NamedTuple, NewType, TypedDict, TypeVar + T = TypeVar(\"T\", bound=\"list[F]\") + Id = NewType(\"Id\", \"F\") + Pair = NamedTuple(\"Pair\", [(\"first\", \"F\")]) + Movie = TypedDict(\"Movie\", {\"lead\": \"F\"}) + class F: ... + "}, + ); + } + + /// a type argument holding a reference is quoted whole, so the tuple type it + /// lowers to is inside the quote + #[test] + fn a_tuple_type_argument_is_quoted_whole() { + check( + indoc! {" + from typing import TypeVar + Pairs = list[(F, int)] + T = TypeVar(\"T\", bound=(F, int)) + class F: ... + "}, + indoc! {" + from typing import TypeVar + Pairs = list[\"tuple[F, int]\"] + T = TypeVar(\"T\", bound=\"tuple[F, int]\") + class F: ... + "}, + ); + } + + /// a `Callable` parameter list written as a tuple becomes python's list, and + /// each parameter holding a reference is quoted on its own, as a parameter of + /// a list is + #[test] + fn a_callable_tuple_parameter_list_quotes_each_parameter() { + check( + indoc! {" + from typing import Callable + Both = Callable[(F, int), F] + Returns = Callable[(int, str), F] + Nested = list[Callable[(F, int), bool]] + class F: ... + "}, + indoc! {" + from typing import Callable + Both = Callable[[\"F\", int], \"F\"] + Returns = Callable[[int, str], \"F\"] + Nested = list[\"Callable[[F, int], bool]\"] + class F: ... + "}, + ); + } + + /// basedpython type syntax around a reference lowers inside the quote + #[test] + fn a_basedpython_type_argument_is_quoted_with_its_lowering() { + check( + indoc! {" + Maybe = list[F?] + Handler = list[(F) -> None] + Method = list[int.() -> F] + class F: ... + "}, + indoc! {" + from typing import Callable + Maybe = list[\"F | None\"] + Handler = list[\"Callable[[F], None]\"] + Method = list[\"Callable[[int], F]\"] + class F: ... + "}, + ); + } + + /// a `TypedDict`'s `extra_items=` is a type python evaluates as the class is + /// built, and a keyword like `metaclass=` is an object it is built with + #[test] + fn extra_items_is_quoted_and_a_metaclass_is_not() { + check( + indoc! {" + from typing_extensions import TypedDict + class Movie(TypedDict, extra_items=F): ... + class Meta(type): ... + class Plain(metaclass=Meta): ... + class F: ... + "}, + indoc! {" + from typing_extensions import TypedDict + class Movie(TypedDict, extra_items=\"F\"): ... + class Meta(type): ... + class Plain(metaclass=Meta): ... + class F: ... + "}, + ); + } + + /// a string in a type argument is a literal type, and lands in the quote + /// with the reference beside it + #[test] + fn a_literal_type_beside_a_quoted_reference_is_lowered_inside_the_quote() { + check( + indoc! {" + items = list[\"a\" | F]() + class F: ... + "}, + indoc! {" + from typing import Literal + items = list['Literal[\"a\"] | F']() + class F: ... + "}, + ); + } + + /// the specialization itself is evaluated where it stands, so a generic bound + /// only on some paths, which counts as a forward reference, is left as it is + #[test] + fn a_possibly_unbound_generic_is_not_quoted() { + check( + indoc! {" + import sys + if len(sys.argv) > 0: + Box = list + items = Box[F]() + class F: ... + "}, + indoc! {" + import sys + if len(sys.argv) > 0: + Box = list + items = Box[\"F\"]() + class F: ... + "}, + ); + } + + /// a specialization inside a reference already quoted is quoted with it, + /// and not a second time inside the quote + #[test] + fn a_specialization_inside_a_quote_is_not_quoted_again() { + check( + indoc! {" + from typing import Annotated + items = list[F | Annotated[int, dict[str, G]()]]() + class F: ... + class G: ... + "}, + indoc! {" + from typing import Annotated + items = list[\"F | Annotated[int, dict[str, G]()]\"]() + class F: ... + class G: ... + "}, + ); + } + + /// a type argument is quoted whole, as an annotation is, so the metadata of + /// an `Annotated` in it is read with it + #[test] + fn an_annotated_type_argument_is_quoted_whole() { + check( + indoc! {" + from typing import Annotated + items = list[Annotated[F, len]]() + class F: ... + "}, + indoc! {" + from typing import Annotated + items = list[\"Annotated[F, len]\"]() + class F: ... + "}, + ); + } + + /// a name a function binds further down is a forward reference within it, + /// and one the module binds is bound by the time the function runs + #[test] + fn a_function_body_quotes_only_its_own_later_names() { + check( + indoc! {" + def f(): + x = list[G]() + y = list[F]() + G = int + return x, y + class F: ... + "}, + indoc! {" + def f(): + x = list[\"G\"]() + y = list[F]() + G = int + return x, y + class F: ... + "}, ); } @@ -404,7 +687,7 @@ mod tests { fn self_ref_in_nested_subscript() { check( "class A(dict[str, list[A]]): ...\n", - "class A(dict[str, list[\"A\"]]): ...\n", + "class A(dict[str, \"list[A]\"]): ...\n", ); } diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index d6a1cc7a4a..80cd6ea3a6 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -33,6 +33,7 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Lowering, PassContext, TypeAwarePass}; use super::intersection::{collect_intersect, collect_union, is_intersection_node}; use super::just_float::rewrite_type_expr_with_imports; +use super::quotes::WrittenQuotes; use super::repeated_underscore::{WrittenNames, lowered_callable_parameters}; use super::type_expr_walker::RootKind; use crate::config::{Config, FloatLiteralLowering}; @@ -59,6 +60,8 @@ pub(crate) struct CallableSyntax<'src> { source: &'src str, /// the module as written, whose names a repeated `_` is numbered around written: WrittenNames<'src>, + /// where the synthesized protocols' member types are written as strings + quotes: &'src WrittenQuotes, types: Option<&'src (dyn TypeInfo + 'src)>, /// ranges already folded by `symbolic_type_op` (e.g. `1 + typeof d` → /// `Literal[3]`). a claimed sub-expression is opaque here: descending into @@ -115,10 +118,16 @@ struct ProtocolShape { } impl<'src> CallableSyntax<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: &Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + config: &Config, + ) -> Self { Self { source, written, + quotes, types: None, claimed_ranges: &[], owned_typeof: None, @@ -516,7 +525,7 @@ impl<'src> CallableSyntax<'src> { }, _ => name.to_string(), }; - let ty = quote_forward_ref(&self.rewrite_or_leaf(&named.value)); + let ty = self.quotes.quote(&self.rewrite_or_leaf(&named.value)); parts.push(format!("{name}: {ty}")); } Expr::Starred(s) => match s.value.as_ref() { @@ -526,7 +535,7 @@ impl<'src> CallableSyntax<'src> { // members are emitted one by one match self.unpacked_kwargs(&inner.value) { Some(UnpackedKwargsLowering::TypedDict) => { - let ty = quote_forward_ref(&format!( + let ty = self.quotes.quote(&format!( "{}[{}]", self.written.imported("typing", "Unpack"), self.rewrite_or_leaf(&inner.value) @@ -544,11 +553,11 @@ impl<'src> CallableSyntax<'src> { } for (name, member_ty) in members { parts - .push(format!("{name}: {}", quote_forward_ref(&member_ty))); + .push(format!("{name}: {}", self.quotes.quote(&member_ty))); } } Some(UnpackedKwargsLowering::ParameterPack) | None => { - let ty = quote_forward_ref(&self.rewrite_or_leaf(&inner.value)); + let ty = self.quotes.quote(&self.rewrite_or_leaf(&inner.value)); parts.push(format!("**kwargs: {ty}")); } } @@ -561,12 +570,13 @@ impl<'src> CallableSyntax<'src> { let unpacks = self.is_unpacked_variadic(&s.value); let inner = self.rewrite_or_leaf(&s.value); let ty = - quote_forward_ref(&if unpacks { format!("*{inner}") } else { inner }); + self.quotes + .quote(&if unpacks { format!("*{inner}") } else { inner }); parts.push(format!("*args: {ty}")); } }, _ => { - let ty = quote_forward_ref(&self.rewrite_or_leaf(arg)); + let ty = self.quotes.quote(&self.rewrite_or_leaf(arg)); parts.push(format!("{}: {ty}", names[i])); } } @@ -648,11 +658,11 @@ impl<'src> CallableSyntax<'src> { format!( "{}: {}", receiver_parameter_name(ct), - quote_forward_ref(&rendered) + self.quotes.quote(&rendered) ) }); let params = self.render_protocol_params(ct, false, "self", implicit_receiver); - let returns = quote_forward_ref(&self.render_return(ct)); + let returns = self.quotes.quote(&self.render_return(ct)); let shape = ProtocolShape { params, returns }; Some(self.class_name_for(shape)) } @@ -899,13 +909,6 @@ impl<'src> CallableSyntax<'src> { } } -/// render a synthesized annotation as a forward-reference string literal so it -/// is never evaluated at class-body time (the hoisted protocol class precedes -/// the definitions its annotations mention) -fn quote_forward_ref(ty: &str) -> String { - format!("\"{}\"", ty.replace('\\', "\\\\").replace('"', "\\\"")) -} - /// The synthesized name of a receiver parameter in a protocol `__call__`. The /// receiver is unnamed in the surface syntax, so any name works — but a callable /// may itself declare a parameter called `_receiver`, and two parameters of the @@ -965,16 +968,18 @@ fn is_named(expr: &Expr, ident: &str) -> bool { /// rendering path, which is what lets one wide edit carry rewrites it subsumes. /// /// `root` says where the text is written, which decides how a union in it is spelled +#[expect(clippy::too_many_arguments)] pub(crate) fn lower_type_expr_full( source: &str, written: WrittenNames, + quotes: &WrittenQuotes, types: &dyn TypeInfo, expr: &Expr, substitutions: &[(TextRange, String)], config: &Config, root: RootKind, ) -> Option { - let mut inner = CallableSyntax::new(source, written, config).with_types(types); + let mut inner = CallableSyntax::new(source, written, quotes, config).with_types(types); inner.set_evaluated(root == RootKind::Evaluated); for (range, name) in substitutions { inner.add_substitution(*range, name.clone()); @@ -1060,6 +1065,16 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> { return crate::transforms::type_expr_walker::Recurse::Stop; } + // a generic's type arguments are lowered one by one, each where it stands, so a + // pass that writes one of them whole, a forward reference quoted where python reads + // it lazily, wraps that argument's lowering rather than landing inside a rewrite of + // the whole subscript. a `typeof` and a decorated type are syntax of their own + if let Expr::Subscript(s) = expr + && !s.is_typeof + && !s.is_type_decoration + { + return crate::transforms::type_expr_walker::Recurse::Descend; + } // `rewrite` is the single type-expression lowerer — it owns every // structural type-form (callable arrows, `&` / `and`, `or`, `not`, // `typeof`, subscripts) and composes leaves through `lower_leaf`. it @@ -1081,14 +1096,21 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> pub(crate) struct CallableSyntaxPass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, } impl<'src> CallableSyntaxPass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + config: Config, + ) -> Self { Self { source, written, + quotes, config, } } @@ -1157,7 +1179,7 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { // `required_imports` / `edits` uses below let claimed = ctx.claimed_type_op_ranges.clone(); let structural_typeof = ctx.structural_typeof_ranges.clone(); - let mut inner = CallableSyntax::new(self.source, self.written, &self.config) + let mut inner = CallableSyntax::new(self.source, self.written, self.quotes, &self.config) .with_types(types) .with_claimed_ranges(&claimed) .with_owned_typeof(&structural_typeof); diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 3d10027ac8..513d345aee 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -51,6 +51,7 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::parametric_is::{PredicateRuntime, ProbeStrictness, build_predicate}; +use super::quotes::WrittenQuotes; use super::repeated_underscore::WrittenNames; use crate::type_info::{CastCheck, SoundnessCheck, TypeInfo}; @@ -117,6 +118,9 @@ impl Helper { struct CastLower<'a> { types: &'a dyn TypeInfo, written: WrittenNames<'a>, + /// where a `typing.cast` target that names something not bound there is written as a + /// string + quotes: &'a WrittenQuotes, /// the name `typing.cast` is written under cast: String, edits: Vec<(TextRange, Vec)>, @@ -126,10 +130,11 @@ struct CastLower<'a> { } impl<'a> CastLower<'a> { - fn new(types: &'a dyn TypeInfo, written: WrittenNames<'a>) -> Self { + fn new(types: &'a dyn TypeInfo, written: WrittenNames<'a>, quotes: &'a WrittenQuotes) -> Self { Self { types, written, + quotes, cast: written.imported("typing", "cast"), edits: Vec::new(), used: BTreeSet::new(), @@ -170,9 +175,10 @@ impl<'a> CastLower<'a> { // no parametric claim (or none that can be checked): the shallow // `isinstance` target, or the written type when ty offers no faithful one match self.types.cast_check_plan(type_arg) { - Some(CastCheck::Kind(SoundnessCheck::Isinstance(target))) => { - (helper, vec![Fragment::Lit(target.to_string())]) - } + Some(CastCheck::Kind(SoundnessCheck::Isinstance(target))) => ( + helper, + vec![Fragment::Lit(self.written.spelled(&target.spelling()))], + ), // a `Literal` is a set of values, not a class: `isinstance` against // one raises, and testing the values' class would accept every other // value of it. the check is membership @@ -192,23 +198,27 @@ impl<'a> CastLower<'a> { let value_range = value_arg.range(); // unchecked `typing.cast(type, value)` keeps the exact written type (it // never reaches `isinstance`) and takes the type first + // + // `typing.cast` never reads its type, but python evaluates it all the same, so a + // type naming something that does not hold there the binding the checker resolved — + // a class imported only under `if TYPE_CHECKING:`, or declared further on — is + // written as a string, as python's own spelling of such a cast is let (helper, first, rest) = if helper == Helper::TypingCast { - ( - helper, - Fragment::Src(type_arg.range()), - vec![Fragment::Src(value_range)], - ) + let first = if self.types.reads_names_bound_where_written(type_arg) { + vec![Fragment::Src(type_arg.range())] + } else { + vec![self.quotes.quote_span(type_arg.range())] + }; + (helper, first, vec![Fragment::Src(value_range)]) } else { let (helper, rest) = self.target(value_arg, type_arg, helper); - (helper, Fragment::Src(value_range), rest) + (helper, vec![Fragment::Src(value_range)], rest) }; self.used.insert(helper); - let mut fragments = vec![ - Fragment::Lit(helper.open_call(&self.cast)), - first, - Fragment::Lit(", ".to_owned()), - ]; + let mut fragments = vec![Fragment::Lit(helper.open_call(&self.cast))]; + fragments.extend(first); + fragments.push(Fragment::Lit(", ".to_owned())); fragments.extend(rest); fragments.push(Fragment::Lit(")".to_owned())); self.edits.push((whole, fragments)); @@ -248,11 +258,35 @@ impl<'ast> Visitor<'ast> for CastLower<'_> { pub(crate) struct CheckedCastPass<'src> { pub(crate) written: WrittenNames<'src>, + pub(crate) quotes: &'src WrittenQuotes, } impl TypeAwarePass for CheckedCastPass<'_> { + /// a checked cast builds its runtime check from the checker's reading of the type it + /// checks against, and leaves the type out where the checker settles the check without + /// it (an operand whose type cannot hold the target), so it writes these lowerings of the + /// type as written itself. where it falls back to the type as written, the lowerings + /// inside pass through with it. the emitted name of a `private` class is included + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[ + super::ast_driver::Lowering::AnonNamedTuple, + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::FloatConst, + super::ast_driver::Lowering::JustFloat, + super::ast_driver::Lowering::LiteralType, + super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::SymbolicTypeOp, + super::ast_driver::Lowering::TemplateType, + super::ast_driver::Lowering::TupleLiteralType, + super::ast_driver::Lowering::TypedDictLiteral, + super::ast_driver::Lowering::Typeof, + super::ast_driver::Lowering::VisibilityRename, + ] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = CastLower::new(types, self.written); + let mut inner = CastLower::new(types, self.written, self.quotes); for stmt in stmts { inner.visit_stmt(stmt); } diff --git a/crates/by_transforms/src/transforms/coalesce.rs b/crates/by_transforms/src/transforms/coalesce.rs index acbcd0ffdd..ce0ef0f1bb 100644 --- a/crates/by_transforms/src/transforms/coalesce.rs +++ b/crates/by_transforms/src/transforms/coalesce.rs @@ -1,151 +1,168 @@ -use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Operator, Stmt}; +use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::visitor::{Visitor, walk_expr}; +use ruff_python_ast::{AnyNodeRef, Comprehension, Expr, ExprBinOp, Operator, Stmt}; +use ruff_python_trivia::CommentRanges; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::repeated_underscore::WrittenNames; +use super::walrus_scope::WalrusRegions; use crate::type_info::TypeInfo; /// rewrites `a ?? b` to `a if a is not None else b`. /// /// emits *template* edits: operands are `Src` passthrough spans, so any -/// sibling lowering inside an operand (`?.`, `!`, `expr.N`, …) is materialized -/// inside the rewrite instead of being clobbered by first-wins overlap dedup -pub(crate) struct NoneCoalesce<'src> { +/// sibling lowering inside an operand (`?.`, `!`, `expr.N`, a `??` of its own) +/// is materialized inside the rewrite instead of being clobbered by first-wins +/// overlap dedup +pub(crate) struct NoneCoalesce<'src, 'ast> { source: &'src str, types: &'src dyn TypeInfo, + comments: &'src CommentRanges, + regions: &'src WalrusRegions, + /// the node the next `??` visited is written directly under, when python reads + /// a conditional expression there as something else, or refuses one: an operand + /// of `and` / `or`, the test or value of a conditional expression, a + /// comprehension's iterable or condition, what a `*` or `**` unpacks + tight_parent: Option>, edits: Vec<(TextRange, Vec)>, } -impl<'src> NoneCoalesce<'src> { - fn new(source: &'src str, types: &'src dyn TypeInfo) -> Self { - Self { - source, - types, - edits: Vec::new(), +impl<'ast> Visitor<'ast> for NoneCoalesce<'_, 'ast> { + fn visit_expr(&mut self, expr: &'ast Expr) { + let tight_parent = self.tight_parent.take(); + match expr { + Expr::BinOp(coalesce) if matches!(coalesce.op, Operator::Coalesce) => { + // a literal that cannot be `None` is the whole value, and the + // fallback never runs + if is_non_none_literal(&coalesce.left) { + self.edits + .push((expr.range(), vec![Fragment::Src(coalesce.left.range())])); + self.visit_expr(&coalesce.left); + return; + } + let (mut template, conditional) = self.lower(coalesce); + if let Some(parent) = tight_parent + && conditional + && parenthesized_range(expr.into(), parent, self.comments, self.source) + .is_none() + { + template.insert(0, Fragment::Lit("(".to_owned())); + template.push(Fragment::Lit(")".to_owned())); + } + self.edits.push((expr.range(), template)); + self.visit_expr(&coalesce.left); + self.visit_expr(&coalesce.right); + } + Expr::BoolOp(bool_op) => { + for value in &bool_op.values { + self.tight_parent = Some(expr.into()); + self.visit_expr(value); + } + } + Expr::If(conditional) => { + self.tight_parent = Some(expr.into()); + self.visit_expr(&conditional.test); + self.tight_parent = Some(expr.into()); + self.visit_expr(&conditional.body); + self.visit_expr(&conditional.orelse); + } + Expr::Starred(starred) => { + self.tight_parent = Some(expr.into()); + self.visit_expr(&starred.value); + } + Expr::Dict(dict) => { + for item in &dict.items { + match &item.key { + Some(key) => self.visit_expr(key), + None => self.tight_parent = Some(expr.into()), + } + self.visit_expr(&item.value); + } + } + _ => walk_expr(self, expr), } } -} - -fn expand_none_chain(expr: &Expr, source: &str, types: &dyn TypeInfo) -> Option> { - let (form, guards, base) = super::none_chain::expand_chain(expr, source, types)?; - Some(super::none_chain::build_expansion( - &guards, - &form, - &super::none_chain::temp_var(WrittenNames::new(source), 0), - base, - )) -} - -impl<'ast> Visitor<'ast> for NoneCoalesce<'_> { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - walk_stmt(self, stmt); - } - fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::BinOp(b) = expr - && matches!(b.op, Operator::Coalesce) - { - // build the whole (possibly chained) `??` expansion in one edit over - // the outer expression's range, then stop — recursing into the - // operands here would emit overlapping edits for any nested `??` - let template = self.expand_coalesce(expr); - self.edits.push((expr.range(), template)); - return; + fn visit_comprehension(&mut self, comprehension: &'ast Comprehension) { + self.visit_expr(&comprehension.target); + self.tight_parent = Some(comprehension.into()); + self.visit_expr(&comprehension.iter); + for condition in &comprehension.ifs { + self.tight_parent = Some(comprehension.into()); + self.visit_expr(condition); } - walk_expr(self, expr); } } -impl NoneCoalesce<'_> { - /// Lower a `??` expression to a conditional template. Chained `??` - /// recurses, so `a ?? b ?? c` becomes a nested `… if … is not None else …` - /// rather than stranding the inner `??` as verbatim source. The walrus - /// temp is reused safely: a nested one only lives inside a branch that is - /// no longer referenced once the enclosing walrus is taken. - fn expand_coalesce(&self, expr: &Expr) -> Vec { - let Expr::BinOp(b) = expr else { - return self.operand_value(expr); - }; - if !matches!(b.op, Operator::Coalesce) { - return self.operand_value(expr); - } - // a literal LHS statically known to be non-None short-circuits to the LHS - // (avoids a `1 is not None` constant-fold warning) - if is_non_none_literal(&b.left) { - return self.operand_value(&b.left); - } - let rhs = self.operand_value(&b.right); +impl NoneCoalesce<'_, '_> { + /// `a ?? b`, with `a` evaluated once and `b` only when `a` is `None`. each operand + /// passes through as source, so a `??` written inside one lowers in place. the + /// flag says whether the template is a conditional expression, which binds looser + /// than any operator + fn lower(&self, coalesce: &ExprBinOp) -> (Vec, bool) { + let left = Fragment::Src(coalesce.left.range()); + let right = Fragment::Src(coalesce.right.range()); // a wrapped optional (`int??`, a generic `T?`) keeps its present value // inside the runtime wrapper — the present branch unwraps with `.value` - let unwrap = if self.types.wrapped_optional(&b.left) { + let unwrap = if self.types.wrapped_optional(&coalesce.left) { ".value" } else { "" }; - let temp = super::none_chain::temp_var(WrittenNames::new(self.source), 0); - match expand_none_chain(&b.left, self.source, self.types) { - Some(expanded) => { - let mut t = vec![Fragment::Lit(format!("{temp}{unwrap} if ({temp} := "))]; - t.extend(expanded); - t.push(Fragment::Lit(") is not None else ".to_owned())); - t.extend(rhs); - t - } - None if is_trivially_pure(&b.left) => { - let mut t = vec![ - Fragment::Src(b.left.range()), - Fragment::Lit(format!("{unwrap} if ")), - Fragment::Src(b.left.range()), - Fragment::Lit(" is not None else ".to_owned()), - ]; - t.extend(rhs); - t - } - None => { - // a chained (left-associative) `??` puts another coalesce on the - // left; recurse through `operand_value` so it is lowered rather - // than copied verbatim. hoist to walrus so the LHS runs once - let mut t = vec![Fragment::Lit(format!("{temp}{unwrap} if ({temp} := "))]; - t.extend(self.operand_value(&b.left)); - t.push(Fragment::Lit(") is not None else ".to_owned())); - t.extend(rhs); - t - } - } - } - - /// Render an operand: a nested `??` is expanded, a `?.` chain is lowered, - /// otherwise it is a passthrough span (any sibling edits inside it apply - /// when the template is materialized). - fn operand_value(&self, expr: &Expr) -> Vec { - if let Expr::BinOp(b) = expr - && matches!(b.op, Operator::Coalesce) - { - return self.expand_coalesce(expr); + if is_trivially_pure(&coalesce.left) { + let template = vec![ + left.clone(), + Fragment::Lit(format!("{unwrap} if ")), + left, + Fragment::Lit(" is not None else ".to_owned()), + right, + ]; + return (template, true); } - if let Some(expanded) = expand_none_chain(expr, self.source, self.types) { - return expanded; + // every `??` binds the same temporary. each reads it back straight after + // binding it, before anything else in the expression runs, so a `??` nested + // in an operand never overwrites a value still to be read + let temp = super::none_chain::temp_var(WrittenNames::new(self.source), 0); + if self.regions.allows(coalesce.range) { + let template = vec![ + Fragment::Lit(format!("{temp}{unwrap} if ({temp} := ")), + left, + Fragment::Lit(") is not None else ".to_owned()), + right, + ]; + (template, true) + } else { + // where python refuses `:=` the temporary is a comprehension variable, + // and the fallback is chosen by `or` on whether the comprehension kept + // the value: `a` is its first iterable and `b` is outside it, so both + // are still evaluated in the scope they were written in + let template = vec![ + Fragment::Lit(format!("([{temp}{unwrap} for {temp} in (")), + left, + Fragment::Lit(format!(",) if {temp} is not None] or [")), + right, + Fragment::Lit("])[0]".to_owned()), + ]; + (template, false) } - vec![Fragment::Src(expr.range())] } } -/// Expressions whose evaluation has no side effects and whose value is -/// stable across two reads — safe to re-emit in both branches of the -/// rewrite without changing program semantics +/// expressions whose evaluation has no side effects and whose value is stable +/// across two reads, so they are safe to re-emit in both branches of the rewrite. +/// a unary operator is not one: `-x` calls `x.__neg__()`, which may do anything pub(crate) fn is_trivially_pure(expr: &Expr) -> bool { - match expr { + matches!( + expr, Expr::Name(_) - | Expr::NumberLiteral(_) - | Expr::StringLiteral(_) - | Expr::BytesLiteral(_) - | Expr::BooleanLiteral(_) - | Expr::NoneLiteral(_) - | Expr::EllipsisLiteral(_) => true, - Expr::UnaryOp(u) => is_trivially_pure(&u.operand), - _ => false, - } + | Expr::NumberLiteral(_) + | Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + ) } /// Literals whose value cannot be `None`. `??` on these is a no-op and the @@ -166,11 +183,12 @@ fn is_non_none_literal(expr: &Expr) -> bool { pub(crate) struct NoneCoalescePass<'src> { source: &'src str, + comments: &'src CommentRanges, } impl<'src> NoneCoalescePass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, comments: &'src CommentRanges) -> Self { + Self { source, comments } } } @@ -179,13 +197,15 @@ impl TypeAwarePass for NoneCoalescePass<'_> { Some(super::ast_driver::Lowering::NoneCoalesce) } - /// an optional chain on the left of a `??` is written into the coalesce's own test - fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { - &[super::ast_driver::Lowering::NoneChain] - } - fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = NoneCoalesce::new(self.source, types); + let mut inner = NoneCoalesce { + source: self.source, + types, + comments: self.comments, + regions: &ctx.walrus_regions, + tight_parent: None, + edits: Vec::new(), + }; for stmt in stmts { inner.visit_stmt(stmt); } @@ -205,6 +225,16 @@ mod tests { ); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + #[test] fn basic_coalesce() { check("x = a ?? b\n", "x = a if a is not None else b\n"); @@ -245,7 +275,7 @@ mod tests { "}, indoc::indoc! {" def f(a): - __by_t_0__ if (__by_t_0__ := None if a is None else a.a.b) is not None else 1 + __by_t_0__ if (__by_t_0__ := (None if a is None else a.a.b)) is not None else 1 "}, ); } @@ -261,7 +291,7 @@ mod tests { "}, indoc::indoc! {" def f(a, b): - a if a is not None else None if b is None else b.c + a if a is not None else (None if b is None else b.c) "}, ); } @@ -299,7 +329,7 @@ mod tests { /// which printed the signature again over this edit and left the program refused #[test] fn coalesce_in_a_parameter_default() { - check( + check_beside_runtime( indoc::indoc! {" g: int? = None @@ -308,7 +338,8 @@ mod tests { "}, indoc::indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"g ?? 1\", __name__, \"_MISSING\") g: int | None = None def f(y: int = _MISSING) -> int: @@ -322,28 +353,20 @@ mod tests { /// the same default with the body written on the clause's line #[test] fn coalesce_in_a_parameter_default_with_an_inline_body() { - check( + check_beside_runtime( indoc::indoc! {" g: int? = None def f(y: int = g ?? 1) -> int: return y "}, - indoc::indoc! {" - from typing import Any - _MISSING: Any = object() - g: int | None = None - - def f(y: int = _MISSING) -> int: \n if y is _MISSING: - y = g if g is not None else 1 - return y - "}, + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"g ?? 1\", __name__, \"_MISSING\")\ng: int | None = None\n\ndef f(y: int = _MISSING) -> int: \n if y is _MISSING:\n y = g if g is not None else 1\n return y\n", ); } /// and a chain longer than one link, which recurses in the same edit #[test] fn a_chained_coalesce_in_a_parameter_default() { - check( + check_beside_runtime( indoc::indoc! {" g: int? = None h: int? = None @@ -353,7 +376,8 @@ mod tests { "}, indoc::indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"g ?? h ?? 1\", __name__, \"_MISSING\") g: int | None = None h: int | None = None @@ -365,6 +389,82 @@ mod tests { ); } + /// a `??` inside an operand of another one, short of being the operand itself, is + /// lowered by its own edit, inside the source the outer one passes through + #[test] + fn coalesce_inside_an_operand() { + check( + "x = (a ?? {}).get(k) ?? []\n", + "x = __by_t_0__ if (__by_t_0__ := (a if a is not None else {}).get(k)) is not None else []\n", + ); + check( + "x = a ?? f(b ?? 1)\n", + "x = a if a is not None else f(b if b is not None else 1)\n", + ); + check( + "x = f\"{a ?? 1}\"\n", + "x = f\"{a if a is not None else 1}\"\n", + ); + } + + /// a conditional expression binds looser than every operator, so where the `??` + /// is an operand of `and` / `or`, the test or value of a conditional expression, a + /// comprehension's iterable or condition, or what a `*` / `**` unpacks, it is + /// parenthesized — unless the source already did + #[test] + fn coalesce_keeps_its_grouping() { + check( + "x = a ?? b or c\n", + "x = (a if a is not None else b) or c\n", + ); + check( + "x = a ?? b if c else d\n", + "x = (a if a is not None else b) if c else d\n", + ); + check( + "x = c if a ?? b else d\n", + "x = c if (a if a is not None else b) else d\n", + ); + check( + "x = [i for i in xs if a ?? b]\n", + "x = [i for i in xs if (a if a is not None else b)]\n", + ); + check( + "x = (a ?? b) or c\n", + "x = (a if a is not None else b) or c\n", + ); + check("x = [*a ?? b]\n", "x = [*(a if a is not None else b)]\n"); + check("x = {**a ?? b}\n", "x = {**(a if a is not None else b)}\n"); + } + + /// python refuses `:=` in a comprehension's iterable, so the value is bound as a + /// comprehension variable there instead + #[test] + fn coalesce_in_a_comprehension_iterable() { + check( + "x = [i for i in f() ?? []]\n", + "x = [i for i in ([__by_t_0__ for __by_t_0__ in (f(),) if __by_t_0__ is not None] or [[]])[0]]\n", + ); + } + + /// and in every part of a comprehension in a class body + #[test] + fn coalesce_in_a_class_body_comprehension() { + check( + "class C:\n x = [f(i) ?? 0 for i in xs]\n y = f(0) ?? 0\n", + "class C:\n x = [([__by_t_0__ for __by_t_0__ in (f(i),) if __by_t_0__ is not None] or [0])[0] for i in xs]\n y = __by_t_0__ if (__by_t_0__ := f(0)) is not None else 0\n", + ); + } + + /// a unary operator calls a method of its operand, so its result is bound once + #[test] + fn unary_lhs_hoisted_to_walrus_single_eval() { + check( + "x = -a ?? 1\n", + "x = __by_t_0__ if (__by_t_0__ := -a) is not None else 1\n", + ); + } + #[test] fn python_unchanged() { let src = "x = a if a is not None else b\n"; diff --git a/crates/by_transforms/src/transforms/decorator_keyword.rs b/crates/by_transforms/src/transforms/decorator_keyword.rs index 8b550a498f..e0948c38ad 100644 --- a/crates/by_transforms/src/transforms/decorator_keyword.rs +++ b/crates/by_transforms/src/transforms/decorator_keyword.rs @@ -535,6 +535,16 @@ mod tests { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + /// the options are passed as `@d(option=...)`, so a repeated `_` among them would /// be keyword-only, and the name only a keyword reaches it by is not the author's #[test] @@ -837,14 +847,15 @@ mod tests { /// and `@d` raised `TypeError: object of type 'object' has no len()` #[test] fn a_mutable_option_default_reaches_the_guard_and_not_the_signature() { - check( + check_beside_runtime( indoc! {" decorator def d(fn: (...) -> object, tags: list[str] = []) -> int: return len(tags) "}, indoc! {" from typing import Any, Callable, overload - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") @overload def d(fn: Callable[..., object], *, tags: list[str] = ...) -> int: ... @overload diff --git a/crates/by_transforms/src/transforms/destructure.rs b/crates/by_transforms/src/transforms/destructure.rs index 96e98d9ac2..61c6fd408d 100644 --- a/crates/by_transforms/src/transforms/destructure.rs +++ b/crates/by_transforms/src/transforms/destructure.rs @@ -67,7 +67,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, Fragment, PassContext}; use super::repeated_underscore::WrittenNames; -use super::source_util::{line_indent, line_start, temporary_name}; +use super::source_util::{line_indent, line_start, standalone_expression, temporary_name}; pub(crate) struct DestructurePass<'src> { source: &'src str, @@ -197,7 +197,7 @@ impl DestructureLower<'_> { return; } let indent = line_indent(self.source, let_stmt.range().start()).to_owned(); - let subject = vec![Fragment::Src(let_stmt.value.range())]; + let subject = standalone_expression(self.source, let_stmt.value.range()); if let_stmt.orelse.is_empty() { let mut fragments = Vec::new(); @@ -462,13 +462,17 @@ impl DestructureLower<'_> { // the subject is evaluated once, and every case guard reads it from // there. `if True:` hosts the cases at the indentation their bodies // already have, so no body moves + let mut header = vec![Fragment::Lit(format!("{subject_name} = "))]; + header.extend(standalone_expression( + self.source, + match_stmt.subject.range(), + )); + header.push(Fragment::Lit(format!( + "\n{indent}{selector} = 0\n{indent}if True:" + ))); let mut edits = vec![( TextRange::new(match_stmt.range().start(), header_colon), - vec![ - Fragment::Lit(format!("{subject_name} = ")), - Fragment::Src(match_stmt.subject.range()), - Fragment::Lit(format!("\n{indent}{selector} = 0\n{indent}if True:")), - ], + header, )]; for (index, case) in match_stmt.cases.iter().enumerate() { @@ -488,12 +492,16 @@ impl DestructureLower<'_> { let mut fragments = vec![Fragment::Lit(format!( "if {selector} == 0:\n{guard_indent}" ))]; + let guard = case + .guard + .as_deref() + .map(|guard| standalone_expression(self.source, guard.range())); if !self.push_destructure_with_guard( &mut fragments, &guard_indent, &[Fragment::Lit(subject_name.clone())], &case.pattern, - case.guard.as_deref(), + guard.as_deref(), &format!("{selector} = {}", index + 1), ) { return; @@ -541,7 +549,7 @@ impl DestructureLower<'_> { indent: &str, subject: &[Fragment], pattern: &Pattern, - guard: Option<&Expr>, + guard: Option<&[Fragment]>, on_match: &str, ) -> bool { match push_destructure( @@ -576,7 +584,7 @@ pub(crate) fn push_destructure( indent: &str, subject: &[Fragment], pattern: &Pattern, - guard: Option<&Expr>, + guard: Option<&[Fragment]>, on_match: &str, names: &mut NameGen<'_>, ) -> Result<(), String> { @@ -624,7 +632,7 @@ pub(crate) fn push_destructure( && let Some(guard) = guard { fragments.push(Fragment::Lit(" if ".to_owned())); - fragments.push(Fragment::Src(guard.range())); + fragments.extend(guard.iter().cloned()); } step_indent = format!("{step_indent} "); fragments.push(Fragment::Lit(format!(":\n{step_indent}"))); diff --git a/crates/by_transforms/src/transforms/erased_union.rs b/crates/by_transforms/src/transforms/erased_union.rs index 8d2ce380aa..32692069f7 100644 --- a/crates/by_transforms/src/transforms/erased_union.rs +++ b/crates/by_transforms/src/transforms/erased_union.rs @@ -56,6 +56,8 @@ use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Expr, PythonVersion, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange}; +use ty_python_semantic::RuntimeSpelling; + use crate::type_info::TypeInfo; /// the stem of a synthesized type parameter's name. dunder-ish and namespaced @@ -87,6 +89,9 @@ fn plan( if taken.contains(&name) { return None; } + // the rewrite is source the phases after this one read, so a module-level + // `private` class is spelled as the source declares it and renamed with every + // other reference to it let width = union.fixed.len() + 1; let arguments: Vec = (0..width) .map(|position| { @@ -97,18 +102,26 @@ fn plan( .fixed .iter() .find(|(at, _)| *at == position) - .map(|(_, text)| text.clone()) + .map(|(_, spelling)| spelling.source()) .unwrap_or_default() } }) .collect(); Some(Rewrite { annotation: annotation.range(), - replacement: format!("{}[{}]", union.origin, arguments.join(", ")), + replacement: format!("{}[{}]", union.origin.source(), arguments.join(", ")), // spelled as a *type mapping*, which is what a union of arms is: the // parameter ranges over exactly these types and no others. a bare // `: (int, str)` would be a tuple *bound* instead, which no arm satisfies - declaration: format!("reified {name} in ({})", union.arms.join(", ")), + declaration: format!( + "reified {name} in ({})", + union + .arms + .iter() + .map(RuntimeSpelling::source) + .collect::>() + .join(", ") + ), }) } diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 594375bc84..6be0dc104f 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -8,6 +8,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::config::Config; use crate::transforms::callable::lower_type_expr_full; +use crate::transforms::quotes::WrittenQuotes; use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::RootKind; use crate::type_info::TypeInfo; @@ -21,6 +22,8 @@ use ruff_python_ast::PythonVersion; pub(crate) struct GenericPolyfill<'src> { source: &'src str, written: WrittenNames<'src>, + /// where a bound read through the classes it stands in is written as a string + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: Config, edits: Vec, @@ -208,9 +211,11 @@ struct ProcessedTypeParams { } impl<'src> GenericPolyfill<'src> { + #[expect(clippy::too_many_arguments)] fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: Config, symbolic_substitutions: Vec<(TextRange, String)>, @@ -221,6 +226,7 @@ impl<'src> GenericPolyfill<'src> { Self { source, written, + quotes, types, config, edits: Vec::new(), @@ -427,6 +433,7 @@ impl<'src> GenericPolyfill<'src> { let rendered = lower_type_expr_full( self.source, self.written, + self.quotes, self.types, default, &self.subsume_within(default.range()), @@ -483,12 +490,13 @@ impl<'src> GenericPolyfill<'src> { ) -> ProcessedTypeParams { // a bound that reads a class's name through the class is written ahead of the class, // so it is a string, which `TypeVar` keeps as a forward reference + let quotes = self.quotes; let through_classes = |text: String| { let read_through = apply_renames_to_rendered(&text, qualified); if read_through == text { text } else { - super::source_util::python_string_literal(&read_through) + quotes.quote(&read_through) } }; let mut generic_args: Vec = Vec::new(); @@ -579,6 +587,7 @@ impl<'src> GenericPolyfill<'src> { lower_type_expr_full( self.source, self.written, + self.quotes, self.types, bound, &self.subsume_within(bound.range()), @@ -1086,6 +1095,7 @@ impl<'src> GenericPolyfill<'src> { let value_src = lower_type_expr_full( self.source, self.written, + self.quotes, self.types, &alias.value, &substitutions, @@ -2054,14 +2064,21 @@ fn names_read(expr: &Expr) -> Vec<&str> { pub(crate) struct GenericPolyfillPass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, } impl<'src> GenericPolyfillPass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + config: Config, + ) -> Self { Self { source, written, + quotes, config, } } @@ -2097,6 +2114,7 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { let mut inner = GenericPolyfill::new( self.source, self.written, + self.quotes, types, self.config.clone(), ctx.symbolic_substitutions.clone(), diff --git a/crates/by_transforms/src/transforms/grapheme_string.rs b/crates/by_transforms/src/transforms/grapheme_string.rs index 768a2f7b53..480f31f2a6 100644 --- a/crates/by_transforms/src/transforms/grapheme_string.rs +++ b/crates/by_transforms/src/transforms/grapheme_string.rs @@ -52,6 +52,8 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::coalesce::is_trivially_pure; use super::repeated_underscore::WrittenNames; +use super::source_util::temporary_name; +use super::walrus_scope::WalrusRegions; use crate::type_info::TypeInfo; #[expect( @@ -61,6 +63,7 @@ use crate::type_info::TypeInfo; struct GraphemeString<'src> { types: &'src dyn TypeInfo, written: WrittenNames<'src>, + regions: &'src WalrusRegions, /// attribute nodes that are the direct callee of a call — the grapheme /// properties are not callable there, so they pass through untouched callee_attrs: HashSet, @@ -204,12 +207,26 @@ impl GraphemeString<'_> { Fragment::Lit(" else None)".to_owned()), ] } else { - vec![ - Fragment::Lit("(Character(_by_graphemes(_s)".to_owned()), - Fragment::Lit(format!("[{index}]) if (_s := ")), - Fragment::Src(receiver), - Fragment::Lit(") else None)".to_owned()), - ] + let temp = temporary_name(self.written, "s", 0); + if self.regions.allows(expr.range()) { + vec![ + Fragment::Lit(format!( + "(Character(_by_graphemes({temp})[{index}]) if ({temp} := " + )), + Fragment::Src(receiver), + Fragment::Lit(") else None)".to_owned()), + ] + } else { + // where python refuses `:=` (see `WalrusRegions`) the receiver is + // bound as a comprehension variable instead + vec![ + Fragment::Lit(format!( + "([Character(_by_graphemes({temp})[{index}]) for {temp} in (" + )), + Fragment::Src(receiver), + Fragment::Lit(format!(",) if {temp}] or [None])[0]")), + ] + } }; self.template(expr, fragments); } @@ -296,6 +313,7 @@ impl TypeAwarePass for GraphemeStringPass<'_> { let mut inner = GraphemeString { types, written: self.written, + regions: &ctx.walrus_regions, callee_attrs: HashSet::new(), needs_grapheme_helper: false, needs_character_class: false, @@ -428,7 +446,7 @@ def _by_suffix(_text, _n): "{CHAR_IMPORT}{HELPER}{}", indoc! {" def g() -> str: ... - a = (Character(_by_graphemes(_s)[0]) if (_s := g()) else None) + a = (Character(_by_graphemes(__by_s_0__)[0]) if (__by_s_0__ := g()) else None) "} ), ); diff --git a/crates/by_transforms/src/transforms/if_let.rs b/crates/by_transforms/src/transforms/if_let.rs index 0545e4e7d8..da173834f2 100644 --- a/crates/by_transforms/src/transforms/if_let.rs +++ b/crates/by_transforms/src/transforms/if_let.rs @@ -42,7 +42,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::destructure::{NameGen, push_destructure}; use super::repeated_underscore::WrittenNames; -use super::source_util::{line_indent, temporary_name}; +use super::source_util::{line_indent, standalone_expression, temporary_name}; use crate::type_info::TypeInfo; pub(crate) struct IfLetPass<'src> { @@ -191,7 +191,7 @@ impl IfLetLower<'_, '_> { &selector, 1, clause.pattern, - test, + &standalone_expression(self.source, test.range()), &mut self.names, ) { self.errors @@ -208,7 +208,7 @@ impl IfLetLower<'_, '_> { &selector, index + 1, clause.pattern, - test, + &standalone_expression(self.source, test.range()), &mut self.names, ) { self.errors @@ -253,21 +253,22 @@ impl IfLetLower<'_, '_> { } /// Appends the guard that records clause `value` as taken: the destructuring a -/// pattern clause stands for, a plain `if` for a condition +/// pattern clause stands for, a plain `if` for a condition. `test` is the clause's +/// subject or condition, as [`standalone_expression`] writes it fn push_guard( fragments: &mut Vec, indent: &str, selector: &str, value: usize, pattern: Option<&Pattern>, - test: &Expr, + test: &[Fragment], names: &mut NameGen<'_>, ) -> Result<(), String> { match pattern { Some(pattern) => push_destructure( fragments, indent, - &[Fragment::Src(test.range())], + test, pattern, None, &format!("{selector} = {value}"), @@ -275,7 +276,7 @@ fn push_guard( ), None => { fragments.push(Fragment::Lit("if ".to_owned())); - fragments.push(Fragment::Src(test.range())); + fragments.extend(test.iter().cloned()); fragments.push(Fragment::Lit(format!( ":\n{indent} {selector} = {value}" ))); diff --git a/crates/by_transforms/src/transforms/inferred_annotation.rs b/crates/by_transforms/src/transforms/inferred_annotation.rs index 4d9e2e226e..4619aeb4bf 100644 --- a/crates/by_transforms/src/transforms/inferred_annotation.rs +++ b/crates/by_transforms/src/transforms/inferred_annotation.rs @@ -376,7 +376,10 @@ mod tests { ) .unwrap(); assert!(out.contains("decimal = _lazy_module(\"decimal\")"), "{out}"); - assert!(out.contains("d: \"decimal.Decimal\" = Dec(1)"), "{out}"); + assert!( + out.contains("d: \"decimal.Decimal\" = _by_lazy.Dec(1)"), + "{out}" + ); assert!(!out.contains("TYPE_CHECKING"), "{out}"); } diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index f5c6f6a24f..fca51952b4 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -21,33 +21,48 @@ //! `T?`, a bare `float`). use std::cell::RefCell; +use std::collections::HashSet; use ruff_python_ast::helpers::MemberVisibility; -use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Parameter, Stmt, StmtFunctionDef}; use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::callable::lower_type_expr_full; -use super::mutable_defaults::{parameter_guards, undeclarable_error}; +use super::mutable_defaults::{ + DefaultMarkers, define_markers, parameter_guards, undeclarable_error, +}; +use super::quotes::WrittenQuotes; use super::repeated_underscore::WrittenNames; use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; use super::type_expr_walker::RootKind; +use super::visibility_rename::renamed_reference; use crate::config::Config; use crate::type_info::TypeInfo; pub(crate) struct InitMethod<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + markers: &'src DefaultMarkers, config: Config, } impl<'src> InitMethod<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + markers: &'src DefaultMarkers, + config: Config, + ) -> Self { Self { source, written, + quotes, + markers, config, } } @@ -58,22 +73,22 @@ impl TypeAwarePass for InitMethod<'_> { let mut state = State { source: self.source, written: self.written, + quotes: self.quotes, types, symbolic_substitutions: ctx.symbolic_substitutions.clone(), + module_private: types.private_module_symbols().into_iter().collect(), config: &self.config, edits: RefCell::new(Vec::new()), templates: RefCell::new(Vec::new()), relocating: RefCell::new(Vec::new()), errors: RefCell::new(Vec::new()), - needs_missing: RefCell::new(false), + markers: self.markers, + definitions: RefCell::new(Vec::new()), }; for stmt in stmts { state.visit_stmt(stmt); } - if state.needs_missing.into_inner() { - ctx.required_imports - .extend(super::mutable_defaults::sentinel_definition(self.written)); - } + define_markers(self.written, state.definitions.into_inner(), ctx); ctx.text_edits.extend(state.edits.into_inner()); ctx.template_edits.extend(state.templates.into_inner()); ctx.relocating_edits.extend(state.relocating.into_inner()); @@ -121,7 +136,7 @@ fn literal_text(frags: &[Fragment]) -> String { .iter() .map(|frag| match frag { Fragment::Lit(lit) => lit.as_str(), - Fragment::Src(_) => "", + Fragment::Src(_) | Fragment::Quote(_) => "", }) .collect() } @@ -129,19 +144,24 @@ fn literal_text(frags: &[Fragment]) -> String { struct State<'src> { source: &'src str, written: WrittenNames<'src>, + /// where a synthesized protocol's member types are written as strings + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, /// `(range, rendered)` for every symbolic fold in the module. the `__init__` /// line is fresh output, so a fold inside a parameter annotation is dropped /// unless it is spliced in here symbolic_substitutions: Vec<(TextRange, String)>, + /// the module-level names the module declares `private` + module_private: HashSet, config: &'src Config, edits: RefCell>, templates: RefCell)>>, /// the `_MISSING` substitutions, whose defaults the guards re-evaluate relocating: RefCell)>>, errors: RefCell>, - /// whether the `_MISSING` sentinel reached the output - needs_missing: RefCell, + markers: &'src DefaultMarkers, + /// the lines binding the `_MISSING` markers the generated signatures name + definitions: RefCell>, } impl State<'_> { @@ -184,18 +204,44 @@ impl State<'_> { /// verbatim, or the invalid basedpython surface leaks into the `.py` output. /// the required imports and any hoisted `Protocol` class come from the /// sibling passes' visit of the same parameter annotation, so this only - /// needs to reproduce the lowered text. falls back to the source verbatim - /// when nothing lowers + /// needs to reproduce the lowered text, a name a visibility keyword renamed + /// included. falls back to the source verbatim when nothing lowers fn lower_annotation(&self, ann: &Expr) -> String { + struct Renames<'a, 'src> { + state: &'a State<'src>, + renames: Vec<(TextRange, String)>, + } + impl<'ast> Visitor<'ast> for Renames<'_, '_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && let Some(renamed) = renamed_reference( + self.state.types, + self.state.written, + &self.state.module_private, + name, + ) + { + self.renames.push((name.range, renamed)); + } + walk_expr(self, expr); + } + } + let mut renames = Renames { + state: self, + renames: Vec::new(), + }; + renames.visit_expr(ann); let substitutions: Vec<(TextRange, String)> = self .symbolic_substitutions .iter() .filter(|(range, _)| ann.range().contains_range(*range)) .cloned() + .chain(renames.renames) .collect(); lower_type_expr_full( self.source, self.written, + self.quotes, self.types, ann, &substitutions, @@ -353,11 +399,13 @@ impl State<'_> { sentinels, written, guards, + definitions, undeclarable, } = parameter_guards( + self.source, func, self.written, - &super::mutable_defaults::sentinel_name(self.written), + self.markers, self.types, self.config.is_stub, ); @@ -386,6 +434,7 @@ impl State<'_> { // are written whether or not the body gains anything self.relocating.borrow_mut().extend(sentinels); self.templates.borrow_mut().extend(written); + self.definitions.borrow_mut().extend(definitions); let pos = func.range.end(); if guards.is_empty() { // no passthrough to carry, so emit plain text: a template @@ -394,12 +443,6 @@ impl State<'_> { self.push(TextRange::new(pos, pos), literal_text(&frags)); return; } - if guards - .iter() - .any(super::mutable_defaults::Guard::uses_sentinel) - { - *self.needs_missing.borrow_mut() = true; - } self.templates .borrow_mut() .push((TextRange::new(pos, pos), frags)); @@ -425,6 +468,16 @@ mod tests { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + #[test] fn a_declaring_parameter_lowers_its_annotation_once() { // the parser synthesizes `self.a: T = a` from the parameter, with the @@ -462,6 +515,30 @@ mod tests { ); } + /// the attribute's annotation names a `private` class the way the parameter's does: by + /// the name the class is emitted under, inside a type form the line lowers too + #[test] + fn a_declared_attribute_names_a_private_class_by_its_emitted_name() { + check( + indoc! {" + private class Kept: + pass + + class Holder: + init(let k: Kept, let ks: list[Kept]?) + "}, + indoc! {" + class _Kept: + pass + + class Holder: + def __init__(self, k: _Kept, ks: list[_Kept] | None): + self.k: _Kept = k + self.ks: list[_Kept] | None = ks + "}, + ); + } + /// a real modifier still applies on the parameters either side of a variadic #[test] fn modifier_survives_alongside_a_variadic() { @@ -497,6 +574,26 @@ mod tests { ); } + /// a soft keyword is the name of the parameter its modifiers precede, and of the + /// attribute it declares + #[test] + fn a_soft_keyword_names_a_declaring_parameter() { + check( + indoc! {" + class A: + init(let cast: int, var match: int, private let type: int, local lazy: int, let case: int) + "}, + indoc! {" + class A: + def __init__(self, cast: int, match: int, type: int, lazy: int, case: int): + self.cast: int = cast + self.match: int = match + self.__type: int = type + self.case: int = case + "}, + ); + } + // `var` is accepted as the mutable counterpart of `let`; for an init // parameter it self-assigns identically #[test] @@ -736,7 +833,7 @@ mod tests { /// stored as the sentinel #[test] fn a_declaration_follows_the_guard_for_its_own_default() { - check( + check_beside_runtime( indoc! {" class A: init(let a: list[int] = []): @@ -745,7 +842,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class A: def __init__(self, a: list[int] = _MISSING): \"\"\"doc\"\"\" @@ -782,20 +880,12 @@ mod tests { /// write at the offset after the docstring, in the order they run #[test] fn two_passes_write_under_a_docstring_on_the_clause_line() { - check( + check_beside_runtime( indoc! {" class A: init(let a: list[int] = []): \"\"\"doc\"\"\" "}, - indoc! {" - from typing import Any - _MISSING: Any = object() - class A: - def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\" - if a is _MISSING: - a = [] - self.a: list[int] = a - "}, + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\nclass A:\n def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\"\n if a is _MISSING:\n a = []\n self.a: list[int] = a\n", ); } @@ -804,20 +894,12 @@ mod tests { /// that follows it #[test] fn a_docstring_followed_by_statements_on_the_clause_line_is_given_a_line_of_its_own() { - check( + check_beside_runtime( indoc! {" class A: init(let a: list[int] = []): \"\"\"doc\"\"\"; print(a) "}, - indoc! {" - from typing import Any - _MISSING: Any = object() - class A: - def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\"; \n if a is _MISSING: - a = [] - \n self.a: list[int] = a - print(a) - "}, + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\nclass A:\n def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\"; \n if a is _MISSING:\n a = []\n \n self.a: list[int] = a\n print(a)\n", ); } diff --git a/crates/by_transforms/src/transforms/lazy_import.rs b/crates/by_transforms/src/transforms/lazy_import.rs index 7b81e1c7f9..1d1bf452d4 100644 --- a/crates/by_transforms/src/transforms/lazy_import.rs +++ b/crates/by_transforms/src/transforms/lazy_import.rs @@ -3,10 +3,12 @@ //! Two emission strategies, chosen by target Python version: //! - **`min_version >= 3.15`** — prepend the `lazy` keyword (PEP 810) //! - **`min_version < 3.15`** — rewrite the statement to call a runtime -//! polyfill (`_lazy_module` for module imports, `_lazy_attr` for `from` -//! imports). The helpers wrap `importlib.util.LazyLoader` and a small proxy -//! class, and live in [`crate::runtime`] with the rest of what the emitted -//! python calls +//! polyfill. a module import binds `_lazy_module(...)`, a module whose body +//! runs on first attribute access. a `from` import binds nothing: it +//! registers its names with the module's `_LazyFrom` (`_lazy_from(...)`), +//! and every read of one is written as a read through it, which resolves +//! the name on first use — see [`deferrable_reads`]. The helpers live in +//! [`crate::runtime`] with the rest of what the emitted python calls //! //! Both modes skip: //! - `from __future__ import ...` — compiler directive @@ -15,7 +17,8 @@ //! bound to `False` //! //! The polyfill additionally skips forms it can't safely rewrite: -//! - relative imports (`from .pkg import x`) +//! - a `from` import name whose reads cannot all be rewritten, or which is +//! read where only the object bound at the import will do //! - `import a.b` without an alias (binds the top package, which //! `LazyLoader` does not register) //! - bootstrap modules (`sys`, `importlib*`) — the helpers depend on them — and @@ -27,9 +30,13 @@ //! A stub defers nothing — see [`Deferral::Never`]. use ruff_diagnostics::{Edit, Fix}; +use ruff_python_ast::visitor::source_order::{ + SourceOrderVisitor, walk_interpolated_string_element, walk_t_string, +}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom}; use ruff_text_size::{Ranged, TextRange, TextSize}; +use std::collections::{HashMap, HashSet}; /// How a module-level import is deferred. #[derive(Clone, Copy, PartialEq, Eq)] @@ -58,22 +65,26 @@ pub(crate) struct LazyImport<'src> { /// registers it at import, so deferring the import defers the conformance /// out of existence eager: Vec, - /// bindings that must be bound to the real object rather than to a proxy. - /// cpython's `except` checks that what it catches is a class inheriting - /// `BaseException` and never consults `__instancecheck__`, so an exception - /// class reached through the proxy raises `TypeError` from the handler; a - /// special form, and a name an annotation reads, are told apart by identity - /// by whatever reads them — see [`annotation_names`] + /// bindings that must be made where the import is written: an exception + /// class, a special form, and a name an annotation or a decorator reads — see + /// [`ty_python_semantic::SemanticModel::eagerly_imported_names`], + /// [`annotation_names`] and [`decorator_names`] eager_names: Vec, deferral: Deferral, pub(crate) edits: Vec, /// True when at least one statement was rewritten to call /// `_lazy_module`; the preamble must define the module helper pub(crate) needs_module_helper: bool, - /// True when at least one statement was rewritten to call `_lazy_attr`; - /// the preamble must define the `_LazyAttr` proxy (and `_lazy_module`, - /// which `_lazy_attr` calls) - pub(crate) needs_attr_helper: bool, + /// the `from` import names [`deferrable_from_imports`] found every read of, and + /// whose names may be deferred, by the range of their `ast::Alias` + /// + /// [`deferrable_from_imports`]: ty_python_semantic::SemanticModel::deferrable_from_imports + deferrable: HashSet, + /// the name the module's `_LazyFrom` is bound under, which every read of a + /// deferred name is written through + holder: &'src str, + /// each name the walk deferred, with the range of the alias that imports it + pub(crate) deferred: Vec<(TextRange, String)>, /// True when at least one `from ty_extensions import X` was rewritten to /// a `_TyExtMarker` assignment; the preamble must define the marker pub(crate) needs_ty_ext_marker: bool, @@ -90,6 +101,8 @@ impl<'src> LazyImport<'src> { deferral: Deferral, eager: &[String], eager_names: &[String], + deferrable: HashSet, + holder: &'src str, ) -> Self { Self { source, @@ -99,7 +112,9 @@ impl<'src> LazyImport<'src> { deferral, edits: Vec::new(), needs_module_helper: false, - needs_attr_helper: false, + deferrable, + holder, + deferred: Vec::new(), needs_ty_ext_marker: false, needs_character_class: false, } @@ -157,8 +172,8 @@ impl<'src> LazyImport<'src> { /// a module the polyfill leaves eager. the helpers run on `sys` and `importlib`, and /// `builtins` is loaded before any module runs, so deferring it defers nothing — while - /// its proxy would stand where a lowering reads a builtin under a name of its own, and - /// fail an identity test such as `type(x) is str2` + /// a lowering's import of a builtin under a name of its own is one of the imports the + /// runtime preamble is placed after fn is_bootstrap(name: &str) -> bool { matches!(name, "sys" | "importlib" | "builtins") || name.starts_with("importlib.") } @@ -243,8 +258,9 @@ impl<'src> LazyImport<'src> { .is_some_and(|m| m.id.as_str() == "__future__"); let is_star = node.names.iter().any(|a| a.name.id.as_str() == "*"); // `TYPE_CHECKING` is a flag a checker reads statically, not an ordinary - // binding: deferring it rebinds the name to a proxy, and the name may - // only ever be bound to `False` + // binding: deferring it would make each read of it a read through the + // module's `_LazyFrom`, and a checker only ever accepts the name bound to + // `False` let binds_type_checking = node .names .iter() @@ -318,6 +334,10 @@ impl<'src> LazyImport<'src> { // `lines` the one import line they share is written into let mut eager_spellings: Vec = Vec::new(); let mut eager_line: Option = None; + // the names this statement registers with the module's `_LazyFrom`, as + // `(bound, imported)`, and the slot in `lines` the registration goes in + let mut registered: Vec<(&str, &str)> = Vec::new(); + let mut registration_line: Option = None; // whether the statement imported a bare `Character` that contributed no // binding line — the import must still be *removed* (the preamble class // defines `Character`), so force an empty replacement below @@ -352,11 +372,26 @@ impl<'src> LazyImport<'src> { } continue; } - if self.eager_names.iter().any(|eager| eager == bind) { - // the proxy cannot stand here, so re-emit the import as written. - // ahead of the relative branches: a relative import is the - // common shape inside a package, and its exceptions are caught - // just the same + let eager = self.eager_names.iter().any(|eager| eager == bind); + // `from . import x` is no different: `x` is read off the package, and + // imported as its submodule only when the package has no `x` + if !eager + // a name beginning with two underscores could be one the holder's own type + // answers, `__doc__` among them, and a read of it inside a class body is + // mangled the moment it is written as an attribute + && !bind.starts_with("__") + && self.deferrable.contains(&alias.range()) + { + if registration_line.is_none() { + registration_line = Some(lines.len()); + lines.push(String::new()); + } + registered.push((bind, name)); + self.deferred.push((alias.range(), bind.to_owned())); + } else { + // re-emit the import as written: the name is read where only the + // object itself will do, or somewhere a read of it cannot be + // rewritten let spelling = if bind == name { name.to_owned() } else { @@ -364,31 +399,12 @@ impl<'src> LazyImport<'src> { }; // names that shared one import statement go on writing one line // between them. the line is held at the position of the first of - // them, so it keeps its place among the lazy bindings around it + // them, so it keeps its place among the deferred names around it if eager_line.is_none() { eager_line = Some(lines.len()); lines.push(String::new()); } eager_spellings.push(spelling); - } else if is_relative && module_part.is_empty() { - // `from . import x` — `x` is a submodule of the current - // package. Resolve the relative target at runtime - self.needs_module_helper = true; - let rel = format!("{dots}{name}"); - lines.push(format!("{bind} = _lazy_module(\"{rel}\", __package__)")); - } else if is_relative { - // `from .pkg import x` — lazy attribute on the resolved - // parent, matching the `from pkg import x` shape - self.needs_attr_helper = true; - let rel = format!("{dots}{module_part}"); - lines.push(format!( - "{bind} = _lazy_attr(\"{rel}\", \"{name}\", __package__)" - )); - } else { - self.needs_attr_helper = true; - lines.push(format!( - "{bind} = _lazy_attr(\"{module_part}\", \"{name}\")" - )); } } if let Some(index) = eager_line { @@ -397,6 +413,18 @@ impl<'src> LazyImport<'src> { eager_spellings.join(", ") ); } + if let Some(index) = registration_line { + let names: Vec = registered + .iter() + .map(|(bind, name)| format!("\"{bind}\": \"{name}\"")) + .collect(); + let package = if is_relative { ", __package__" } else { "" }; + lines[index] = format!( + "_lazy_from({}, \"{dots}{module_part}\", {{{}}}{package})", + self.holder, + names.join(", ") + ); + } // `character_only` means a bare `Character` import produced no lines but // must still be dropped (the preamble class defines `Character`) so it // doesn't survive as a runtime `from ty_extensions import Character` @@ -409,10 +437,12 @@ impl<'src> LazyImport<'src> { } return; } - let indent = self.line_indent(node.range()); - let separator = format!("\n{indent}"); + // one line, padded to the statement's height: everything after the + // statement keeps the line it had, and the statement can share its line + // with others, as `if c: from m import x` does + let height = self.source[node.range()].matches('\n').count(); self.edits.push(Fix::safe_edit(Edit::range_replacement( - lines.join(&separator), + format!("{}{}", lines.join("; "), "\n".repeat(height)), node.range(), ))); } @@ -420,11 +450,12 @@ impl<'src> LazyImport<'src> { /// every name the annotations python keeps for introspection read, strings included /// -/// under the polyfill a `from` import is a proxy, and whatever reads an annotation back is -/// handed that proxy rather than the object: `dataclasses` tells a `KW_ONLY` pseudo-field -/// apart by asking whether the annotation *is* `dataclasses.KW_ONLY`, and -/// `typing.get_type_hints` answers with the proxy, which is not the class to anything -/// comparing by identity. a name read here is therefore bound eagerly +/// whatever reads an annotation back reads its names from the module's namespace — +/// `typing.get_type_hints` evaluates a string annotation against the module's globals, and +/// `dataclasses` tells a `KW_ONLY` pseudo-field apart by asking whether the annotation *is* +/// `dataclasses.KW_ONLY` as the class is made. under the polyfill a deferred `from` import +/// is in that namespace only once something has read it, so a name read here is bound +/// where the import is written /// /// the annotations are the ones python evaluates, or keeps as a string to evaluate on /// request: a parameter's and a return's, of a `def` at any depth, and a variable's in a @@ -517,18 +548,45 @@ pub(crate) fn annotation_names(suite: &[Stmt], source: &str) -> Vec { names.read } +/// every name the type expressions in `quotes` read: the forward references the lowering +/// wrote as strings, each by the text inside its quotes +/// +/// python evaluates such a string against the module's namespace whenever something asks for +/// the type, so a name in it is bound where it is imported. the lowering records each quote as +/// it writes it, with whatever lowered inside it lowered, so a name a lowering wrote into one — +/// a hoisted class, a `typing` form — is read as surely as one the source spelled. a string +/// inside the type, a literal's, is a value and names nothing +pub(crate) fn forward_reference_names(quotes: &[String]) -> Vec { + struct Names(Vec); + impl<'a> Visitor<'a> for Names { + fn visit_expr(&mut self, expr: &'a ruff_python_ast::Expr) { + if let ruff_python_ast::Expr::Name(name) = expr { + self.0.push(name.id.to_string()); + } + walk_expr(self, expr); + } + } + let mut names = Names(Vec::new()); + for quote in quotes { + if let Ok(parsed) = ruff_python_parser::parse_expression(quote) { + names.visit_expr(parsed.expr()); + } + } + names.0 +} + /// The names every decorator in `suite` reads, which are bound eagerly. /// /// A decorator is the other thing read by identity rather than by value. `typing.overload` /// says the `def` under it is one arm of an overload set — but only while it *is* -/// `typing.overload`; reached through the polyfill's proxy it is an unknown callable, and a -/// checker reading the emitted python sees two ordinary functions with `...` bodies instead -/// of the overloads the source declared. The same goes for `final`, `abstractmethod`, +/// `typing.overload`; written as a read through the polyfill's `_LazyFrom` it is an unknown +/// callable, and a checker reading the emitted python sees two ordinary functions with `...` +/// bodies instead of the overloads the source declared. The same goes for `final`, `abstractmethod`, /// `dataclass` and everything else whose whole meaning is which object it is. /// /// Binding these eagerly costs no laziness at all: a decorator is applied while the module -/// is executing, so a proxy standing in for one resolves at import time anyway — the defer -/// buys nothing and loses the declaration. +/// is executing, so a deferred one resolves at import time anyway — the defer buys nothing +/// and loses the declaration pub(crate) fn decorator_names(suite: &[Stmt]) -> Vec { struct Reads<'n> { names: &'n mut Vec, @@ -572,6 +630,75 @@ pub(crate) fn decorator_names(suite: &[Stmt]) -> Vec { names.read } +/// the aliases whose names the polyfill may defer, each with every read of the name: the +/// imports [`deferrable_from_imports`] found every read of, less those with a read whose source +/// text python keeps — `f"{x=}"` prints the expression as written, and a t-string's +/// `Interpolation` carries it — since a read written through the module's `_LazyFrom` would +/// change that text +/// +/// [`deferrable_from_imports`]: ty_python_semantic::SemanticModel::deferrable_from_imports +pub(crate) fn deferrable_reads( + suite: &[Stmt], + imports: Vec, +) -> HashMap> { + #[derive(Default)] + struct KeptText { + in_t_string: bool, + ranges: Vec, + } + + impl<'a> SourceOrderVisitor<'a> for KeptText { + fn visit_t_string(&mut self, t_string: &'a ruff_python_ast::TString) { + let outer = std::mem::replace(&mut self.in_t_string, true); + walk_t_string(self, t_string); + self.in_t_string = outer; + } + + fn visit_interpolated_string_element( + &mut self, + element: &'a ruff_python_ast::InterpolatedStringElement, + ) { + if let ruff_python_ast::InterpolatedStringElement::Interpolation(interpolation) = + element + && (self.in_t_string || interpolation.debug_text.is_some()) + { + self.ranges.push(interpolation.range()); + } + // an f-string nested in a t-string's interpolation is part of its text all the same + walk_interpolated_string_element(self, element); + } + } + + let mut kept = KeptText::default(); + kept.visit_body(suite); + imports + .into_iter() + .filter(|import| { + !import + .reads + .iter() + .any(|read| kept.ranges.iter().any(|range| range.contains_range(*read))) + }) + .map(|import| (import.alias, import.reads)) + .collect() +} + +/// the statements that bind `holder`, the module's `_LazyFrom`, to every name in `names`, and +/// serve those names to other modules through the module's `__getattr__` and `__dir__` +/// +/// `globals` is the name the module reads the builtin under +pub(crate) fn holder_lines(holder: &str, names: &[String], globals: &str) -> String { + let names: Vec = names.iter().map(|name| format!("\"{name}\"")).collect(); + // a one-element tuple needs its comma + let trailing = if names.len() == 1 { "," } else { "" }; + format!( + "{holder} = _LazyFrom({globals}(), ({}{trailing}))\n\ + __getattr__ = {holder}.__by_getattr__\n\ + __dir__ = {holder}.__by_dir__\n", + names.join(", ") + ) +} + /// whether `test` is the `TYPE_CHECKING` flag, spelled bare or through the module it /// comes from fn is_type_checking_test(test: &ruff_python_ast::Expr) -> bool { @@ -609,8 +736,8 @@ impl<'ast> Visitor<'ast> for LazyImport<'_> { self.at_module_level = outer; } // an `if TYPE_CHECKING:` body never runs, so an import inside it has - // no execution to defer — and deferring it would rebind the name to - // a proxy, which is the one thing a type expression cannot read + // no execution to defer — and the checker reading the emitted python + // needs the name it binds for the type expressions that read it Stmt::If(node) if is_type_checking_test(&node.test) => { let outer = std::mem::replace(&mut self.at_module_level, false); walk_stmt(self, stmt); @@ -622,27 +749,28 @@ impl<'ast> Visitor<'ast> for LazyImport<'_> { } /// The runtime helpers a polyfill-mode lazified module calls, by the name it -/// calls them under. What each of those needs in turn — the proxy class, the -/// `sys` binding, the operator forwarding — is settled in [`crate::runtime`], -/// off the calls in their own bodies. +/// calls them under. What each of those needs in turn — `_lazy_module`, the +/// `sys` binding — is settled in [`crate::runtime`], off the calls in their own +/// bodies #[expect( clippy::fn_params_excessive_bools, reason = "independent which-helpers-are-needed flags, not a state machine" )] pub(crate) fn polyfill_helpers( needs_module: bool, - needs_attr: bool, + needs_from: bool, needs_ty_ext: bool, needs_character_class: bool, ) -> Vec { let mut helpers = Vec::new(); - // each only where the emitted code names it: a `from` import calls - // `_lazy_attr` alone, and what that calls in turn is the runtime's business + // each only where the emitted code names it, and what those call in turn + // is the runtime's business if needs_module { helpers.push(crate::runtime::LAZY_MODULE); } - if needs_attr { - helpers.push(crate::runtime::LAZY_ATTR); + if needs_from { + helpers.push(crate::runtime::LAZY_FROM_HOLDER); + helpers.push(crate::runtime::LAZY_FROM); } if needs_ty_ext { helpers.push(crate::runtime::TY_EXT_MARKER); @@ -703,8 +831,7 @@ mod tests { #[test] fn keyword_type_checking_unchanged() { - // deferring the flag would rebind the name to a proxy; a checker only - // ever accepts it bound to `False` + // a checker only ever accepts the flag bound to `False` check( "from typing import TYPE_CHECKING\n", "from typing import TYPE_CHECKING\n", @@ -760,8 +887,8 @@ mod tests { ) .unwrap(); assert!( - out.contains("def _lazy_module("), - "missing _lazy_module helper in:\n{out}" + out.contains("def _lazy_module(") || out.contains("class _LazyFrom:"), + "missing the polyfill's runtime in:\n{out}" ); assert!( out.ends_with(expected_body), @@ -787,32 +914,6 @@ mod tests { ); } - #[test] - fn attr_proxy_is_isinstance_transparent() { - // other lowerings (soundness checks, checked casts) pass imported - // names as runtime types; the proxy must delegate isinstance/issubclass - let out = transpile( - "from os.path import basename\n", - &Config { - lazy_imports: true, - ..Config::test_default() - }, - ) - .unwrap(); - assert!( - out.contains( - "def __instancecheck__(self, o): return isinstance(o, self._by_resolve())" - ), - "proxy must delegate __instancecheck__ in:\n{out}" - ); - assert!( - out.contains( - "def __subclasscheck__(self, o): return issubclass(o, self._by_resolve())" - ), - "proxy must delegate __subclasscheck__ in:\n{out}" - ); - } - #[test] fn polyfill_dotted_with_alias() { check_polyfill_body("import os.path as p\n", "p = _lazy_module(\"os.path\")\n"); @@ -836,7 +937,7 @@ mod tests { fn polyfill_from_import() { check_polyfill_body( "from os import path\n", - "path = _lazy_attr(\"os\", \"path\")\n", + "_lazy_from(_by_lazy, \"os\", {\"path\": \"path\"})\n", ); } @@ -844,10 +945,7 @@ mod tests { fn polyfill_from_import_multiple() { check_polyfill_body( "from os import path, getcwd\n", - indoc! {" - path = _lazy_attr(\"os\", \"path\") - getcwd = _lazy_attr(\"os\", \"getcwd\") - "}, + "_lazy_from(_by_lazy, \"os\", {\"path\": \"path\", \"getcwd\": \"getcwd\"})\n", ); } @@ -900,10 +998,10 @@ mod tests { } #[test] - fn polyfill_relative_submodule() { + fn polyfill_relative_package() { check_polyfill_body( "from . import x\n", - "x = _lazy_module(\".x\", __package__)\n", + "_lazy_from(_by_lazy, \".\", {\"x\": \"x\"}, __package__)\n", ); } @@ -911,7 +1009,7 @@ mod tests { fn polyfill_relative_attr() { check_polyfill_body( "from .pkg import x\n", - "x = _lazy_attr(\".pkg\", \"x\", __package__)\n", + "_lazy_from(_by_lazy, \".pkg\", {\"x\": \"x\"}, __package__)\n", ); } @@ -919,7 +1017,7 @@ mod tests { fn polyfill_relative_double_dot() { check_polyfill_body( "from .. import x\n", - "x = _lazy_module(\"..x\", __package__)\n", + "_lazy_from(_by_lazy, \"..\", {\"x\": \"x\"}, __package__)\n", ); } @@ -1019,8 +1117,7 @@ mod tests { #[test] fn polyfill_exception_class_stays_eager() { - // cpython's `except` refuses anything that is not a real class - // inheriting `BaseException`, so the proxy cannot stand for one + // an exception class is bound where it is imported let source = indoc! {" from json import JSONDecodeError @@ -1030,7 +1127,7 @@ mod tests { except JSONDecodeError: pass "}; - // nothing was deferred, so the proxy runtime is not emitted at all + // nothing was deferred, so the polyfill's runtime is not emitted at all assert_eq!(transpile_polyfill(source), source); } @@ -1039,10 +1136,9 @@ mod tests { // the statement is split: only the exception has to be bound for real let out = transpile_polyfill("from json import JSONDecodeError, dumps\n"); assert!( - out.ends_with(indoc! {" - from json import JSONDecodeError - dumps = _lazy_attr(\"json\", \"dumps\") - "}), + out.ends_with( + "from json import JSONDecodeError; _lazy_from(_by_lazy, \"json\", {\"dumps\": \"dumps\"})\n" + ), "got:\n{out}" ); } @@ -1059,17 +1155,17 @@ mod tests { #[test] fn polyfill_special_forms_stay_eager() { // a special form is recognised by what it *is*: a checker reading the output - // rejects `ClassVar[int]` once `ClassVar` names a variable, and `dataclasses` - // tells a bare `ClassVar` or `Final` annotation from a field by identity, which - // a proxy does not have. an ordinary function beside them is still deferred + // rejects `ClassVar[int]` once `ClassVar` is anything else, and `dataclasses` + // tells a bare `ClassVar` or `Final` annotation from a field by identity as the + // class is made. an ordinary function beside them is still deferred let out = transpile_polyfill(indoc! {" from typing import ClassVar, Final, Literal, Annotated as Note, cast "}); assert!( - out.ends_with(indoc! {" - from typing import ClassVar, Final, Literal, Annotated as Note - cast = _lazy_attr(\"typing\", \"cast\") - "}), + out.ends_with( + "from typing import ClassVar, Final, Literal, Annotated as Note; \ + _lazy_from(_by_lazy, \"typing\", {\"cast\": \"cast\"})\n" + ), "got:\n{out}" ); } @@ -1089,7 +1185,7 @@ mod tests { "the enum no longer lowers through ClassVar, got:\n{out}" ); assert!( - !out.contains("ClassVar = _lazy_attr"), + !out.contains("\"ClassVar\": "), "a special form was deferred, got:\n{out}" ); assert!( @@ -1141,17 +1237,17 @@ mod tests { "got:\n{out}" ); assert!( - !out.contains("overload = _lazy_attr"), + !out.contains("\"overload\": "), "the overload declaration was deferred, got:\n{out}" ); } /// a decorator says what the definition under it *is*, and it says so by identity. - /// deferred, `@dataclass` is a proxy a checker reading the emitted python cannot + /// deferred, `@dataclass` is a read a checker reading the emitted python cannot /// recognize: it read `Row` as an ordinary class and reported `Row("x")` as too many /// positional arguments, where the source declares a one-field dataclass. deferring one - /// buys nothing either — a decorator is applied while the module executes, so the proxy - /// standing in for it resolves at import time regardless + /// buys nothing either — a decorator is applied while the module executes, so a + /// deferred one resolves at import time regardless #[test] fn polyfill_a_decorator_stays_eager() { let out = transpile_polyfill(indoc! {" @@ -1161,9 +1257,10 @@ mod tests { def f() -> int: return reduce(lambda a, b: a + b, [1, 2], 0) "}); - assert!(out.contains("from functools import cache\n"), "got:\n{out}"); + assert!(out.contains("from functools import cache; "), "got:\n{out}"); assert!( - out.contains("reduce = _lazy_attr(\"functools\", \"reduce\")"), + out.contains("_lazy_from(_by_lazy, \"functools\", {\"reduce\": \"reduce\"})") + && out.contains("return _by_lazy.reduce("), "a name only called stopped being deferred, got:\n{out}" ); } @@ -1212,10 +1309,10 @@ mod tests { return dumps(value) "}, indoc! {" - dumps = _lazy_attr(\"json\", \"dumps\") + _lazy_from(_by_lazy, \"json\", {\"dumps\": \"dumps\"}) def show(value: int) -> str: - return dumps(value) + return _by_lazy.dumps(value) "}, ); } @@ -1230,7 +1327,7 @@ mod tests { value: Fraction = 1 "}); assert!( - out.contains("Fraction = _lazy_attr(\"fractions\", \"Fraction\")"), + out.contains("_lazy_from(_by_lazy, \"fractions\", {\"Fraction\": \"Fraction\"})"), "got:\n{out}" ); } @@ -1238,7 +1335,7 @@ mod tests { #[test] fn polyfill_an_annotation_under_future_annotations_stays_eager() { // every annotation is a string at runtime, and `typing.get_type_hints` evaluates - // it against the module's globals, where a proxy would be what it finds + // it against the module's globals, where a deferred name is not until it is read let out = transpile_polyfill(indoc! {" from __future__ import annotations from fractions import Fraction @@ -1252,6 +1349,260 @@ mod tests { ); } + fn assert_eager(source: &str, out: &str) { + assert!( + out.contains("from json import dumps") && !out.contains("_by_lazy"), + "the import was deferred:\n{source}\n---got---\n{out}" + ); + } + + /// a read reaches the module's binding from any scope that does not bind the name + /// itself, and each one is written through the module's `_LazyFrom` + #[test] + fn polyfill_every_read_of_a_deferred_name_is_written_through_the_holder() { + let out = transpile_polyfill(indoc! {r#" + from json import dumps + + def nested() -> object: + def inner() -> object: + return dumps + return inner() + + class Body: + value = dumps + + def default(f: object = dumps) -> object: + return f + + def reads_global() -> object: + global dumps + return dumps + + later = lambda: dumps + listed = [dumps for _ in range(1)] + text = f"{dumps!r:>4}" + "#}); + for read in [ + "return _by_lazy.dumps\n", + "value = _by_lazy.dumps\n", + // a default is evaluated per call in `.by`, so it is read in the function + "f = _by_lazy.dumps\n", + "lambda: _by_lazy.dumps\n", + "[_by_lazy.dumps for _ in range(1)]", + "f\"{_by_lazy.dumps!r:>4}\"", + ] { + assert!(out.contains(read), "missing `{read}` in:\n{out}"); + } + assert_eq!(out.matches("_by_lazy.dumps").count(), 7, "{out}"); + } + + /// anything else that binds the name, anywhere in the module, leaves a read that may find + /// either binding, so the import is left as it was written + #[test] + fn polyfill_a_name_bound_again_stays_eager() { + for rebinding in [ + "dumps = 5", + "dumps += 1", + "dumps: int = 1", + "def dumps() -> None:\n pass", + "class dumps:\n pass", + "for dumps in range(1):\n pass", + "with open(__file__) as dumps:\n pass", + "try:\n pass\nexcept Exception as dumps:\n pass", + "if (dumps := 1):\n pass", + "print([(dumps := i) for i in range(1)])", + "del dumps", + "from json import dumps", + "def f() -> None:\n global dumps\n dumps = 1", + "def f() -> None:\n global dumps\n del dumps", + // a binding that is not a name node: only the scope's own table shows it + "def f() -> None:\n global dumps\n def dumps() -> None:\n pass", + "def f() -> None:\n global dumps\n import json as dumps", + "class C:\n global dumps\n dumps = 1", + "match [1]:\n case [dumps]:\n pass", + ] { + let source = format!("from json import dumps\n{rebinding}\nprint(dumps)\n"); + assert_eager(&source, &transpile_polyfill(&source)); + } + } + + /// a class body reads a name it binds itself from its own namespace and then the + /// module's — never from a function around it — so a read the body has not bound yet + /// reads the import, even where an enclosing function binds the name + #[test] + fn polyfill_a_class_body_read_before_its_own_binding_reads_the_module() { + let out = transpile_polyfill(indoc! {" + from json import dumps + + def outer() -> object: + dumps = 1 + class Body: + seen = dumps + dumps = 2 + return Body.seen + "}); + assert!(out.contains("seen = _by_lazy.dumps"), "{out}"); + } + + /// a read the body has bound on some paths and not on others finds either + #[test] + fn polyfill_a_class_body_that_may_have_bound_the_name_keeps_it_eager() { + let source = indoc! {" + from json import dumps + + def flip() -> bool: + return True + + class Body: + if flip(): + dumps = 1 + seen = dumps + "}; + assert_eager(source, &transpile_polyfill(source)); + } + + /// `f"{x=}"` prints the expression as written, and a t-string's `Interpolation` keeps + /// it, so a read there cannot be written as anything else + #[test] + fn polyfill_a_name_whose_text_python_keeps_stays_eager() { + let source = "from json import dumps\nprint(f\"{dumps=}\")\n"; + assert_eager(source, &transpile_polyfill(source)); + let source = "from json import dumps\nprint(t\"{dumps}\")\n"; + let out = transpile( + source, + &Config { + lazy_imports: true, + min_version: PythonVersion::from((3, 14)), + ..Config::test_default() + }, + ) + .unwrap(); + assert_eager(source, &out); + } + + /// a name beginning with two underscores could be one the holder's own type answers + #[test] + fn polyfill_a_name_beginning_with_two_underscores_stays_eager() { + let out = transpile_polyfill("from json import __doc__ as __text\nprint(__text)\n"); + assert!( + out.contains("from json import __doc__ as __text\n") && !out.contains("_by_lazy"), + "{out}" + ); + } + + /// a star import can bind any name unseen, and a module's own `__getattr__` or + /// `__dir__` is where a deferred import would be served from + #[test] + fn polyfill_a_module_that_could_rebind_a_name_unseen_defers_nothing() { + for source in [ + "from os import *\nfrom json import dumps\nprint(dumps)\n", + "from json import dumps\ndef __getattr__(name: str) -> object:\n return dumps\n", + "from json import dumps\ndef __dir__() -> list[str]:\n return [str(dumps)]\n", + ] { + assert_eager(source, &transpile_polyfill(source)); + } + } + + /// a deferred name is served to other modules, and declared to the holder so that a read + /// before its import has run is the `NameError` python raises + #[test] + fn polyfill_the_holder_is_made_ahead_of_the_module_body() { + let out = transpile_polyfill("\"\"\"doc\"\"\"\nfrom json import dumps, loads as parse\n"); + let holder = out.find("_by_lazy = _LazyFrom(globals(), (\"dumps\", \"parse\"))\n"); + let statement = out + .find("_lazy_from(_by_lazy, \"json\", {\"dumps\": \"dumps\", \"parse\": \"loads\"})\n"); + assert!( + holder.zip(statement).is_some_and(|(h, s)| h < s), + "got:\n{out}" + ); + assert!( + out.contains("__getattr__ = _by_lazy.__by_getattr__\n"), + "{out}" + ); + assert!(out.contains("__dir__ = _by_lazy.__by_dir__\n"), "{out}"); + assert!(out.starts_with("\"\"\"doc\"\"\"\n"), "{out}"); + } + + /// the replacement keeps the statement's height, so every line after it keeps its number + #[test] + fn polyfill_a_multi_line_import_keeps_its_height() { + let out = + transpile_polyfill("from json import (\n dumps,\n loads,\n)\nprint(dumps)\n"); + assert!( + out.ends_with( + "_lazy_from(_by_lazy, \"json\", {\"dumps\": \"dumps\", \"loads\": \"loads\"})\n\n\n\n\ + print(_by_lazy.dumps)\n" + ), + "got:\n{out}" + ); + } + + /// the module binds `globals` to something of its own, so the holder reads the builtin + /// under a name of its own + #[test] + fn polyfill_the_holder_reads_the_builtin_globals() { + let out = transpile_polyfill( + "from json import dumps\n\ndef globals() -> int:\n return 1\n\nprint(dumps)\n", + ); + assert!( + out.contains("from builtins import globals as globals2\n") + && out.contains("_by_lazy = _LazyFrom(globals2(), (\"dumps\",))"), + "got:\n{out}" + ); + } + + /// a type parameter's bound that reads a name of the class it stands in is written ahead + /// of the class, as a string `TypeVar` keeps as a forward reference. python evaluates that + /// string against the module's namespace when anything evaluates it, so every name it + /// reads is bound where it is imported + #[test] + fn polyfill_a_name_a_written_forward_reference_reads_stays_eager() { + let out = transpile_polyfill(indoc! {" + from json import JSONDecoder + + class C: + class Inner: ... + def m[T: dict[JSONDecoder, Inner]](self, x: T) -> T: ... + "}); + assert!( + out.contains("bound=\"dict[JSONDecoder, C.Inner]\"") + && out.contains("from json import JSONDecoder\n") + && !out.contains("\"JSONDecoder\": "), + "{out}" + ); + } + + /// a string `.by` reads as a literal type is written as one, and names nothing + #[test] + fn polyfill_a_string_type_argument_names_nothing() { + let out = transpile_polyfill( + "from json import JSONDecoder\nAlias = list[\"JSONDecoder\"]\nprint(JSONDecoder)\n", + ); + assert!( + out.contains("_lazy_from(_by_lazy, \"json\", {\"JSONDecoder\": \"JSONDecoder\"})"), + "{out}" + ); + } + + /// every name a recorded quote reads is collected, and a string inside one is a value + #[test] + fn forward_reference_names_reads_every_name_a_quote_reads() { + let quotes = [ + "A".to_owned(), + "B | None".to_owned(), + "Callable[[C], D]".to_owned(), + "tuple[E, Literal[\"NotAName\"]]".to_owned(), + "not an expression(".to_owned(), + ]; + let mut names = super::forward_reference_names("es); + names.sort(); + names.dedup(); + assert_eq!( + names, + ["A", "B", "C", "Callable", "D", "E", "Literal", "tuple"] + ); + } + #[test] fn annotation_names_reads_the_annotations_python_keeps() { use ruff_python_parser::parse_module; diff --git a/crates/by_transforms/src/transforms/literal_types.rs b/crates/by_transforms/src/transforms/literal_types.rs index 912015d39b..0f18b3d254 100644 --- a/crates/by_transforms/src/transforms/literal_types.rs +++ b/crates/by_transforms/src/transforms/literal_types.rs @@ -35,6 +35,8 @@ pub(crate) struct LiteralType<'src> { float_literals: FloatLiteralLowering, pub(crate) edits: Vec, pub(crate) needs_literal_import: bool, + /// every string the edits write as a literal type + literal_strings: Vec, } impl<'src> LiteralType<'src> { @@ -52,6 +54,7 @@ impl<'src> LiteralType<'src> { float_literals, edits: Vec::new(), needs_literal_import: false, + literal_strings: Vec::new(), } } @@ -86,6 +89,9 @@ impl<'src> LiteralType<'src> { return; } if is_literal_expr(expr, self.float_literals) { + if expr.is_string_literal_expr() { + self.literal_strings.push(expr.range()); + } self.needs_literal_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( format!("{}[{}]", self.literal, self.src(expr.range())), @@ -122,6 +128,13 @@ impl<'src> LiteralType<'src> { } return; } + // the parameter list of a `Callable[[A, B], R]` + if let Expr::List(list) = expr { + for element in &list.elts { + self.emit_type_edits(element, false); + } + return; + } if let Expr::Subscript(s) = expr { if self.is_literal_name(&s.value) { return; @@ -137,18 +150,20 @@ impl<'src> LiteralType<'src> { if !self.is_type_subscript(&s.value) { return; } + // a string here is a literal type, as it is anywhere else in a `.by` type + // expression — `list["a"]` is `list[Literal["a"]]` — while python would read + // it as a forward reference to a name `a` + // a slice the value is not specialized by is a value, an enum member's name + let specializes = self.types.subscript_specializes(&s.value); match s.slice.as_ref() { Expr::Tuple(t) if !t.parenthesized => { for e in &t.elts { - // bare strings inside a generic subscript are PEP 484 - // forward references; don't promote them to `Literal` - if matches!(e, Expr::StringLiteral(_)) { - continue; + if specializes || !matches!(e, Expr::StringLiteral(_)) { + self.emit_type_edits(e, false); } - self.emit_type_edits(e, false); } } - Expr::StringLiteral(_) => {} + Expr::StringLiteral(_) if !specializes => {} slice => self.emit_type_edits(slice, false), } } @@ -163,6 +178,10 @@ impl<'src> LiteralType<'src> { if !parts.iter().any(|p| { is_literal_expr(p, float_literals) || nominal_float_type(p, float_literals).is_some() }) { + // no arm is a literal, but an arm may hold one: `list["a"] | int` + for part in parts { + self.emit_type_edits(part, false); + } return; } @@ -202,6 +221,9 @@ impl<'src> LiteralType<'src> { group_start = Some(p.range().start()); } } + if p.is_string_literal_expr() { + self.literal_strings.push(p.range()); + } group_list.push(self.src(p.range()).to_owned()); group_end = p.range().end(); } else { @@ -217,6 +239,26 @@ impl<'src> LiteralType<'src> { } } +/// every string in `stmts` that the lowering writes as a literal type, because the checker reads +/// a string there as one — which python, reading the same text, takes for a forward reference +/// +/// the reverse direction unquotes exactly these, so that a forward reference python wrote comes +/// back from a round trip as the reference it was +pub(crate) fn literal_type_strings( + source: &str, + stmts: &[Stmt], + types: &dyn TypeInfo, +) -> Vec { + let mut inner = LiteralType::new( + source, + WrittenNames::new(source), + types, + FloatLiteralLowering::default(), + ); + walk_type_positions_skipping(stmts, Some(types), &[], &mut inner); + inner.literal_strings +} + pub(crate) struct LiteralTypePass<'src> { source: &'src str, written: WrittenNames<'src>, @@ -855,33 +897,97 @@ mod tests { unchanged("a: 1 | 2\n"); } + /// a string in a `.by` type expression is a literal type wherever it is, a subscript's + /// slice included: the checker reads `Foo["Later"]` as `Foo[Literal["Later"]]`, and python + /// would read the bare string as a forward reference to a name `Later` #[test] - fn forward_ref_in_generic_class_subscript_not_promoted() { - // `Foo["Later"]` is a PEP 484 forward reference, not a Literal; - // promoting it to `Foo[Literal["Later"]]` would change the program. + fn a_string_in_a_generic_subscript_is_a_literal_type() { check( indoc! {" - class Foo: pass + class Foo[T]: pass class Bar(Foo[\"Later\"]): pass "}, indoc! {" - class Foo: pass - class Bar(Foo[\"Later\"]): pass + from typing import Literal, TypeVar, Generic + _T = TypeVar(\"_T\") + class Foo(Generic[_T]): pass + class Bar(Foo[Literal[\"Later\"]]): pass "}, ); } #[test] - fn forward_ref_inside_dict_arg_not_promoted() { + fn a_string_argument_of_a_generic_annotation_is_a_literal_type() { check( - indoc! {" - class Foo: pass - a: dict[str, \"Later\"] - "}, - indoc! {" - class Foo: pass - a: dict[str, \"Later\"] - "}, + "a: dict[str, \"Later\"]\n", + "from typing import Literal\na: dict[str, Literal[\"Later\"]]\n", + ); + check( + "a: list[\"x\" | \"y\"]\n", + "from typing import Literal\na: list[Literal[\"x\", \"y\"]]\n", + ); + } + + #[test] + fn a_string_in_a_callable_s_parameter_list_is_a_literal_type() { + check( + "from typing import Callable\na: Callable[[\"x\", int], \"y\"]\n", + "from typing import Callable, Literal\na: Callable[[Literal[\"x\"], int], Literal[\"y\"]]\n", + ); + } + + /// a union with no literal arm can still hold one inside an arm + #[test] + fn a_literal_inside_an_arm_of_a_union_is_lowered() { + check( + "a: list[\"x\"] | int\n", + "from typing import Literal\na: list[Literal[\"x\"]] | int\n", + ); + check( + "a: list[1] | int\n", + "from typing import Literal\na: list[Literal[1]] | int\n", + ); + } + + /// subscripting an enum class looks a member up by name at run time: the string is the + /// name, not a type + #[test] + fn a_runtime_lookup_on_a_class_keeps_its_string() { + let source = indoc! {" + from enum import Enum + class Colour(Enum): + Red = 1 + red = Colour[\"Red\"] + "}; + check(source, source); + } + + /// nor is a subscript the checker resolves to no type at all + #[test] + fn an_unresolved_subscript_on_a_class_keeps_its_string() { + let out = transpile( + "enum class Shape:\n case Circle(radius: float)\n case Point\n\nkind = Shape[\"Circle\"]\n", + &Config::test_default(), + ) + .unwrap(); + assert!(out.contains("kind = Shape[\"Circle\"]"), "{out}"); + } + + /// the value of a `TypeAlias` is a type expression + #[test] + fn a_string_type_alias_value_is_a_literal_type() { + check( + "from typing import TypeAlias\nX: TypeAlias = \"Later\"\n", + "from typing import TypeAlias, Literal\nX: TypeAlias = Literal[\"Later\"]\n", + ); + } + + /// `Annotated`'s metadata and `Literal`'s own arguments are not type expressions + #[test] + fn a_string_that_is_no_type_expression_stays_a_string() { + check( + "from typing import Annotated\na: Annotated[\"x\", \"meta\"]\n", + "from typing import Annotated, Literal\na: Annotated[Literal[\"x\"], \"meta\"]\n", ); } } diff --git a/crates/by_transforms/src/transforms/mod.rs b/crates/by_transforms/src/transforms/mod.rs index beaaabc8f6..ceb26d7734 100644 --- a/crates/by_transforms/src/transforms/mod.rs +++ b/crates/by_transforms/src/transforms/mod.rs @@ -63,6 +63,8 @@ pub(crate) mod postfix_await; pub(crate) mod propagate; pub(crate) mod properties; pub(crate) mod protocol_type; +pub(crate) mod python_scopes; +pub(crate) mod quotes; pub(crate) mod raises_clause; pub(crate) mod reified_class; pub(crate) mod reified_generic; @@ -96,3 +98,4 @@ pub(crate) mod unique_loop_bindings; pub(crate) mod unpack; pub(crate) mod use_site_variance; pub(crate) mod visibility_rename; +pub(crate) mod walrus_scope; diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index db5b12080f..62d649ee76 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -1,14 +1,26 @@ //! Text-edit pass: replaces non-scalar default arguments with a `_MISSING` -//! sentinel and injects a guard at the top of each function body. +//! marker and injects a guard at the top of each function body. //! -//! def f(x=[]): → def f(x=_MISSING): -//! ... if x is _MISSING: -//! x = [] -//! ... +//! ```text +//! def f(x=[]): → _MISSING: Any = _by_default("[]", __name__, "_MISSING") +//! ... def f(x=_MISSING): +//! if x is _MISSING: +//! x = [] +//! ... +//! ``` //! //! Only number, bool, None, string, and ellipsis literals (and unary +/- //! on a number) are kept as-is; everything else is re-evaluated per call. //! +//! Such a default has no value until a call evaluates it, so nothing truthful can +//! stand in the signature as its value. The marker standing there instead carries +//! the default's text, which its `repr` shows, so `inspect.signature` reports +//! `x=` rather than an anonymous object. Each default has a marker of +//! its own, bound at module level ([`DefaultMarkers`]), and its guard tests for that +//! one object, so a marker handed back as an argument — `BoundArguments.apply_defaults` +//! does — is the argument not given, as leaving it out is, while another default's +//! marker is a value like any other. +//! //! The same sentinel machinery lowers basedpython's relaxed parameter order — //! a `def` may declare a required parameter after a defaulted one (so a //! trailing lambda can bind the last parameter while earlier parameters keep @@ -44,17 +56,24 @@ //! not have read yet. A dropped default is silent when the reading is wrong, so the //! python → basedpython direction leaves every written default where it is. +use std::cell::RefCell; +use std::collections::HashMap; + use ruff_python_ast::helpers::is_immutable_scalar_default; +use ruff_python_ast::token::{TokenKind, Tokens}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, ParameterWithDefault, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::repeated_underscore::{WrittenNames, reads_underscore, rebound_underscore}; -use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; +use super::source_util::{ + PrologueStatement, body_prologue, first_body_statement, python_string_literal, + standalone_expression, +}; use crate::type_info::TypeInfo; -/// what a `_MISSING`-sentinel guard does when the argument was not supplied +/// what a `_MISSING`-marker guard does when the argument was not supplied pub(crate) enum Guard { /// re-evaluate the written default per call (mutable defaults). the default /// is carried as its source *range*, not its text: the guard re-emits it @@ -63,14 +82,14 @@ pub(crate) enum Guard { /// rather than being dropped with the signature they came from Reevaluate { name: String, - sentinel: String, - default: TextRange, + marker: String, + default: Vec, }, /// raise — the parameter is required, its sentinel default only exists /// because python rejects a required parameter after a defaulted one Required { name: String, - sentinel: String, + marker: String, function: String, /// the name the builtin `TypeError` is written under type_error: String, @@ -82,37 +101,27 @@ pub(crate) enum Guard { Rebind { parameter: String }, } -impl Guard { - /// whether the guard tests for the `_MISSING` sentinel, which the output then defines - pub(crate) fn uses_sentinel(&self) -> bool { - match self { - Guard::Reevaluate { .. } | Guard::Required { .. } => true, - Guard::Rebind { .. } => false, - } - } -} - impl PrologueStatement for Guard { fn push(&self, frags: &mut Vec, base: &str) { match self { Guard::Reevaluate { name, - sentinel, + marker, default, } => { frags.push(Fragment::Lit(format!( - "if {name} is {sentinel}:\n{base} {name} = " + "if {name} is {marker}:\n{base} {name} = " ))); - frags.push(Fragment::Src(*default)); + frags.extend(default.iter().cloned()); } Guard::Required { name, - sentinel, + marker, function, type_error, } => { frags.push(Fragment::Lit(format!( - "if {name} is {sentinel}:\n{base} raise {type_error}(\"{function}() missing required argument: '{name}'\")" + "if {name} is {marker}:\n{base} raise {type_error}(\"{function}() missing required argument: '{name}'\")" ))); } Guard::Rebind { parameter } => { @@ -125,13 +134,15 @@ impl PrologueStatement for Guard { struct MutableDefaults<'src> { source: &'src str, written: WrittenNames<'src>, + markers: &'src DefaultMarkers, types: &'src dyn TypeInfo, edits: Vec<(TextRange, Vec)>, /// the `_MISSING` substitutions, whose defaults the guards re-evaluate relocating: Vec<(TextRange, Vec)>, /// the guard suites, anchored at the body statement they precede guards: Vec<(TextSize, Vec)>, - used: bool, + /// the lines binding the markers the module's signatures name + definitions: Vec, /// functions whose body starts with parser-synthesized statements, so a /// guard has no source position to anchor to unanchored: Vec, @@ -141,41 +152,150 @@ struct MutableDefaults<'src> { undeclarable: Vec<(String, String)>, } -/// replace a default with the sentinel. a template rather than plain text so -/// it *absorbs* the zero-width insertions a lowering anchored to the default's -/// first token left behind (`_force_unwrap(`), which a plain-text replacement -/// leaves stranded in the signature. the guard re-emits them -fn sentinel_edit(default: TextRange, sentinel: &str) -> (TextRange, Vec) { - (default, vec![Fragment::Lit(sentinel.to_owned())]) +/// replace a default with its marker. a template rather than plain text so it +/// *absorbs* the zero-width insertions a lowering anchored to the default's first +/// token left behind (`_force_unwrap(`), which a plain-text replacement leaves +/// stranded in the signature. the guard re-emits them +fn sentinel_edit(default: TextRange, marker: &str) -> (TextRange, Vec) { + (default, vec![Fragment::Lit(marker.to_owned())]) } -/// The name the sentinel goes into the output under. -/// -/// It stands for "no argument was given", so it has to be a value nothing else can be — -/// which a name the module binds itself is not. Taken past whatever the module spells, so a -/// module that writes `_MISSING` of its own keeps it and the sentinel goes somewhere else. -pub(crate) fn sentinel_name(written: WrittenNames) -> String { - written.fresh("_MISSING") +/// what a marker stands in for +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum MarkerSite { + /// the default at this range, re-evaluated per call + Default(TextRange), + /// the parameter at this range, required after a defaulted one + Required(TextRange), } -/// The lines the output needs before it can use the sentinel: the sentinel itself, and the -/// import its annotation names. +/// The marker standing in the signature for each default re-evaluated per call, and for +/// each required parameter after a defaulted one: one object per site, bound at module +/// level, which the site's own guard tests for by identity. /// -/// The sentinel stands where the source wrote an annotated default, so an unannotated -/// `object()` there is a default the parameter cannot hold and a checker reading the output -/// says so. Declaring it `Any` is how typeshed declares its own sentinels, and it costs the -/// output nothing: the value never reaches the body, whose guard replaces it before anything -/// reads it. -pub(crate) fn sentinel_definition(written: WrittenNames) -> [String; 2] { - [ - written.import_from("typing", &["Any"]), - format!( - "{}: {} = {}()", - sentinel_name(written), +/// A marker stands for "no argument was given" at exactly one parameter. Shared between +/// sites, one function's marker read off its signature would be an argument not given to +/// any other, so each site gets its own, under a name nothing else in the module has — +/// `_MISSING`, `_MISSING2`, … in the order the lowerings ask for them, which is the same +/// for every transpile of the module. The guard's test is then the one identity test the +/// sentinel always was, at the same cost per call. +/// +/// The text a marker shows is the default as basedpython reads it — after the +/// lowerings that run ahead of this pass, such as the qualification of an enum member — +/// on one line: the comments in it are dropped, and a line break between two of its +/// tokens becomes a space, or nothing just inside a bracket. A string literal keeps its +/// own text, a triple-quoted one its own line breaks. +#[derive(Debug, Default)] +pub(crate) struct DefaultMarkers { + /// the text each default's marker shows, by the default's range + texts: HashMap, + /// each site named so far, with its name + named: RefCell>, +} + +impl DefaultMarkers { + /// the text of every parameter default `module` writes, read off `tokens`, the tokens of + /// `source` + pub(crate) fn collect(module: &[Stmt], tokens: &Tokens, source: &str) -> Self { + struct Defaults<'a> { + tokens: &'a Tokens, + source: &'a str, + texts: HashMap, + } + + impl<'a> Visitor<'a> for Defaults<'a> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::FunctionDef(f) = stmt { + for default in f + .parameters + .iter_non_variadic_params() + .filter_map(|parameter| parameter.default.as_deref()) + { + self.texts.insert( + default.range(), + one_line(self.tokens, self.source, default.range()), + ); + } + } + walk_stmt(self, stmt); + } + } + + let mut defaults = Defaults { + tokens, + source, + texts: HashMap::new(), + }; + defaults.visit_body(module); + Self { + texts: defaults.texts, + named: RefCell::default(), + } + } + + /// the name `site`'s marker is bound under, and the line binding it + fn marker(&self, written: WrittenNames, site: MarkerSite, source: &str) -> (String, String) { + let mut named = self.named.borrow_mut(); + let name = if let Some((_, name)) = named.iter().find(|(named, _)| *named == site) { + name.clone() + } else { + let name = written.fresh_outside("_MISSING", |candidate| { + named.iter().any(|(_, name)| name == candidate) + }); + named.push((site, name.clone())); + name + }; + let text = match site { + MarkerSite::Default(range) => python_string_literal( + self.texts + .get(&range) + .map_or(&source[range], String::as_str), + ), + MarkerSite::Required(_) => "None".to_owned(), + }; + let definition = format!( + "{name}: {} = {}({text}, __name__, \"{name}\")", written.imported("typing", "Any"), - written.builtin("object") - ), - ] + crate::runtime::DEFAULT.name() + ); + (name, definition) + } +} + +/// the text of `range`, an expression in `source`, on one line and without its comments +fn one_line(tokens: &Tokens, source: &str, range: TextRange) -> String { + let mut text = String::new(); + let mut previous: Option<(TokenKind, TextSize)> = None; + let mut broken = false; + for token in tokens.in_range(range) { + match token.kind() { + TokenKind::Comment => {} + TokenKind::NonLogicalNewline | TokenKind::Newline => broken = true, + kind => { + if let Some((before, end)) = previous { + let gap = &source[TextRange::new(end, token.start())]; + // a backslash continuation is a line break too + if broken || gap.contains(['\n', '\r']) { + let opens = matches!( + before, + TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace + ); + let closes = + matches!(kind, TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace); + if !opens && !closes { + text.push(' '); + } + } else { + text.push_str(gap); + } + } + text.push_str(&source[token.range()]); + previous = Some((kind, token.end())); + broken = false; + } + } + } + text } /// A default written into the signature at the end of the parameter it belongs to. @@ -206,6 +326,9 @@ pub(crate) struct ParameterGuards { /// relocate nothing; they add text at the end of a parameter pub(crate) written: Vec<(TextRange, Vec)>, pub(crate) guards: Vec, + /// the lines binding the markers the signature names, which the module defines ahead + /// of everything that reads them + pub(crate) definitions: Vec, /// in a stub, the parameters python cannot declare where they stand: a /// required one after a defaulted one. python rejects the shape, and the /// sentinel default a module gets for it would declare the parameter @@ -218,16 +341,23 @@ pub(crate) struct ParameterGuards { /// sentinels they re-evaluate — the defaults it writes are never shared between /// calls. pub(crate) fn parameter_guards( + source: &str, f: &StmtFunctionDef, written_names: WrittenNames, - sentinel: &str, + markers: &DefaultMarkers, types: &dyn TypeInfo, is_stub: bool, ) -> ParameterGuards { let mut sentinels = Vec::new(); let mut written = Vec::new(); let mut guards = Vec::new(); + let mut definitions = Vec::new(); let mut undeclarable = Vec::new(); + let mut marker = |site| { + let (name, definition) = markers.marker(written_names, site, source); + definitions.push(definition); + name + }; let params = f.parameters.as_ref(); // positional parameters: swap non-scalar defaults for the sentinel, and give // basedpython's required-after-defaulted parameters a sentinel default plus @@ -239,12 +369,13 @@ pub(crate) fn parameter_guards( Some(d) => { seen_default = true; if !is_stub && !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) { - sentinels.push(sentinel_edit(d.range(), sentinel)); + let marker = marker(MarkerSite::Default(d.range())); + sentinels.push(sentinel_edit(d.range(), &marker)); guards.push(Guard::Reevaluate { name: crate::python_parameter_name(params, &pw.parameter, written_names) .to_string(), - sentinel: sentinel.to_owned(), - default: d.range(), + marker, + default: standalone_expression(source, d.range()), }); } } @@ -260,11 +391,12 @@ pub(crate) fn parameter_guards( ); } None if seen_default => { - written.push(written_default(pw, sentinel)); + let marker = marker(MarkerSite::Required(pw.range())); + written.push(written_default(pw, &marker)); guards.push(Guard::Required { name: crate::python_parameter_name(params, &pw.parameter, written_names) .to_string(), - sentinel: sentinel.to_owned(), + marker, function: f.name.id.to_string(), type_error: written_names.builtin("TypeError"), }); @@ -275,12 +407,13 @@ pub(crate) fn parameter_guards( for pw in ¶ms.kwonlyargs { match pw.default.as_deref() { Some(d) if !is_stub && !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) => { - sentinels.push(sentinel_edit(d.range(), sentinel)); + let marker = marker(MarkerSite::Default(d.range())); + sentinels.push(sentinel_edit(d.range(), &marker)); guards.push(Guard::Reevaluate { name: crate::python_parameter_name(params, &pw.parameter, written_names) .to_string(), - sentinel: sentinel.to_owned(), - default: d.range(), + marker, + default: standalone_expression(source, d.range()), }); } Some(_) => {} @@ -303,6 +436,7 @@ pub(crate) fn parameter_guards( sentinels, written, guards, + definitions, undeclarable, } } @@ -375,14 +509,17 @@ impl MutableDefaults<'_> { sentinels, written, guards, + definitions, undeclarable, } = parameter_guards( + self.source, f, self.written, - &sentinel_name(self.written), + self.markers, self.types, self.is_stub, ); + self.definitions.extend(definitions); self.undeclarable.extend( undeclarable .into_iter() @@ -393,7 +530,6 @@ impl MutableDefaults<'_> { if guards.is_empty() { return; } - self.used |= guards.iter().any(Guard::uses_sentinel); // the guards go at the start of the first body statement the source // actually wrote @@ -419,29 +555,60 @@ impl<'ast> Visitor<'ast> for MutableDefaults<'_> { pub(crate) struct MutableDefaultsPass<'src> { source: &'src str, written: WrittenNames<'src>, + markers: &'src DefaultMarkers, is_stub: bool, } impl<'src> MutableDefaultsPass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, is_stub: bool) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + markers: &'src DefaultMarkers, + is_stub: bool, + ) -> Self { Self { source, written, + markers, is_stub, } } } +/// Define `definitions`, the lines binding markers a module's signatures name, ahead of +/// everything that reads them, with the import their annotation names and the runtime +/// class they are made of. +/// +/// A marker stands where the source wrote an annotated default, so a marker of a class of +/// its own there is a default the parameter cannot hold and a checker reading the output +/// says so. Declaring each marker `Any` is how typeshed declares its own sentinels, and it +/// costs the output nothing: a marker never reaches the body, whose guard replaces it +/// before anything reads it. +pub(crate) fn define_markers( + written: WrittenNames, + definitions: Vec, + ctx: &mut PassContext, +) { + if definitions.is_empty() { + return; + } + ctx.runtime.insert(crate::runtime::DEFAULT); + ctx.required_imports + .push(written.import_from("typing", &["Any"])); + ctx.required_imports.extend(definitions); +} + impl TypeAwarePass for MutableDefaultsPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = MutableDefaults { source: self.source, written: self.written, + markers: self.markers, types, edits: Vec::new(), relocating: Vec::new(), guards: Vec::new(), - used: false, + definitions: Vec::new(), unanchored: Vec::new(), is_stub: self.is_stub, undeclarable: Vec::new(), @@ -464,10 +631,7 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { )); return; } - if inner.used { - ctx.required_imports - .extend(sentinel_definition(self.written)); - } + define_markers(self.written, inner.definitions, ctx); ctx.template_edits.extend(inner.edits); ctx.relocating_edits.extend(inner.relocating); ctx.statement_inserts.extend(inner.guards); @@ -484,9 +648,13 @@ mod tests { check_at(crate::Config::test_default().min_version, input, expected); } + /// the runtime is taken from a module written beside the output, as a build writes it, + /// so an expectation shows the import of the helper that makes the markers rather than + /// its definition fn check_at(min_version: crate::PythonVersion, input: &str, expected: &str) { let config = crate::Config { min_version, + runtime_module: Some("_by_runtime".to_owned()), ..crate::Config::test_default() }; assert_eq!( @@ -583,7 +751,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class A: def f(self, a = _MISSING): if a is _MISSING: @@ -639,7 +808,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -653,7 +823,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -674,7 +845,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(None, __name__, \"_MISSING\") class S: def __init__(self, first: int = 1, second: str = _MISSING): if second is _MISSING: @@ -697,7 +869,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -731,7 +904,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class S: def __init__(self, items: list[int] = _MISSING) -> None: if items is _MISSING: @@ -750,7 +924,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") def f(x=_MISSING): if x is _MISSING: x = [] @@ -768,7 +943,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"{}\", __name__, \"_MISSING\") def f(x=_MISSING): if x is _MISSING: x = {} @@ -786,7 +962,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"{1, 2}\", __name__, \"_MISSING\") def f(x=_MISSING): if x is _MISSING: x = {1, 2} @@ -840,11 +1017,13 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() - def f(x=_MISSING, y=_MISSING): + from _by_runtime import _by_default + _MISSING2: Any = _by_default(\"{}\", __name__, \"_MISSING2\") + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") + def f(x=_MISSING, y=_MISSING2): if x is _MISSING: x = [] - if y is _MISSING: + if y is _MISSING2: y = {} pass "}, @@ -861,7 +1040,8 @@ mod tests { "#}, indoc! {r#" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default("[]", __name__, "_MISSING") def f(x=_MISSING): """doc""" if x is _MISSING: @@ -879,7 +1059,7 @@ mod tests { fn a_docstring_on_the_def_line_is_given_a_line_of_its_own() { check( "def f(x=[]): \"\"\"doc\"\"\"\n", - "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"\n if x is _MISSING:\n x = []\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"\n if x is _MISSING:\n x = []\n", ); } @@ -893,7 +1073,7 @@ mod tests { fn a_docstring_followed_by_statements_on_the_def_line_is_given_a_line_of_its_own() { check( "def f(x=[]): \"\"\"doc\"\"\"; x.append(1); return x\n", - "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", ); } @@ -903,7 +1083,7 @@ mod tests { fn statements_on_the_def_line_without_a_docstring_take_one_break() { check( "def f(x=[]): x.append(1); return x\n", - "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n x.append(1); return x\n", ); } @@ -913,7 +1093,7 @@ mod tests { fn a_method_body_on_the_clause_line_is_broken_at_its_own_indentation() { check( "class C:\n def m(self, x=[]): \"\"\"doc\"\"\"; x.append(1); return x\n", - "from typing import Any\n_MISSING: Any = object()\nclass C:\n def m(self, x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\nclass C:\n def m(self, x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", ); } @@ -937,13 +1117,15 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING2: Any = _by_default(\"{}\", __name__, \"_MISSING2\") + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") def f(x=_MISSING): if x is _MISSING: x = [] pass - def g(y=_MISSING): - if y is _MISSING: + def g(y=_MISSING2): + if y is _MISSING2: y = {} pass "}, @@ -959,7 +1141,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(None, __name__, \"_MISSING\") def f(x=1, a=_MISSING): if a is _MISSING: raise TypeError(\"f() missing required argument: 'a'\") @@ -977,7 +1160,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(None, __name__, \"_MISSING\") def f(x: int = 1, a: int = _MISSING): if a is _MISSING: raise TypeError(\"f() missing required argument: 'a'\") @@ -997,11 +1181,13 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() - def f(x=_MISSING, a=_MISSING): + from _by_runtime import _by_default + _MISSING2: Any = _by_default(None, __name__, \"_MISSING2\") + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") + def f(x=_MISSING, a=_MISSING2): if x is _MISSING: x = [] - if a is _MISSING: + if a is _MISSING2: raise TypeError(\"f() missing required argument: 'a'\") print(x, a) "}, @@ -1034,7 +1220,8 @@ mod tests { "#}, indoc! {r#" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default("f\"asdf{data}\"", __name__, "_MISSING") data = "fdsa" def f(a=_MISSING): if a is _MISSING: @@ -1056,7 +1243,8 @@ mod tests { "#}, indoc! {r#" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default("t\"asdf{data}\"", __name__, "_MISSING") data = "fdsa" def f(a=_MISSING): if a is _MISSING: @@ -1081,7 +1269,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"a + 1\", __name__, \"_MISSING\") def f(a, b = _MISSING): if b is _MISSING: b = a + 1 @@ -1104,7 +1293,7 @@ mod tests { a: int = [] ) -> int: ... "}, - "from typing import Any\n_MISSING: Any = object()\ndef f(\n a: int = _MISSING\n) -> int: \n if a is _MISSING:\n a = []\n ...\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(\n a: int = _MISSING\n) -> int: \n if a is _MISSING:\n a = []\n ...\n", ); } @@ -1112,7 +1301,7 @@ mod tests { fn inline_ellipsis_body() { check( "def f(x=[]): ...", - "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n ...", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n ...", ); } @@ -1129,7 +1318,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[1 is int]\", __name__, \"_MISSING\") def f(x = _MISSING): if x is _MISSING: x = [True] @@ -1151,7 +1341,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"1 is int\", __name__, \"_MISSING\") def f(x = _MISSING): if x is _MISSING: x = True @@ -1197,7 +1388,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") def f(xs: list[int] = _MISSING) -> int: if xs is _MISSING: xs = [] @@ -1276,7 +1468,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING2: Any = object() + from _by_runtime import _by_default + _MISSING2: Any = _by_default(\"[]\", __name__, \"_MISSING2\") _MISSING = 5 def f(xs: list[int] = _MISSING2) -> int: diff --git a/crates/by_transforms/src/transforms/none_chain.rs b/crates/by_transforms/src/transforms/none_chain.rs index e16893546d..15fb988b6d 100644 --- a/crates/by_transforms/src/transforms/none_chain.rs +++ b/crates/by_transforms/src/transforms/none_chain.rs @@ -1,12 +1,13 @@ use std::fmt::Write as _; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt}; +use ruff_python_ast::{Expr, ExprName, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::source_util::temporary_name; +use crate::transforms::walrus_scope::WalrusRegions; use crate::type_info::TypeInfo; /// rewrites `a?.b` to `(None if a is None else a.b)` and chains like `a?.b?.c` @@ -14,17 +15,9 @@ use crate::type_info::TypeInfo; pub(crate) struct NoneChain<'src> { source: &'src str, types: &'src dyn TypeInfo, + regions: &'src WalrusRegions, template_edits: Vec<(TextRange, Vec)>, -} - -impl<'src> NoneChain<'src> { - fn new(source: &'src str, types: &'src dyn TypeInfo) -> Self { - Self { - source, - types, - template_edits: Vec::new(), - } - } + errors: Vec, } /// How many temporaries a single expansion may need before it starts reusing @@ -48,16 +41,16 @@ fn pick_temp_var(written: WrittenNames<'_>, types: &dyn TypeInfo, anchor: &Expr) /// when any `?.` is present, where `python_form` has all `?.` replaced by `.`, /// `guards` is the ordered list of accumulated sub-expressions that must be /// non-None before each subsequent optional access is safe, and `base` is the -/// source range of the first `?.`'s receiver — everything before the first `?` +/// first `?.`'s receiver — everything before the first `?` /// /// a `?.` whose base is a *wrapped* optional (`int??`, a generic `T?`) /// reaches its present value through the runtime wrapper: the guard still /// tests the wrapper against `None`, but the access reads `.value` first -pub(super) fn expand_chain( - expr: &Expr, +fn expand_chain<'a>( + expr: &'a Expr, source: &str, types: &dyn TypeInfo, -) -> Option<(String, Vec, TextRange)> { +) -> Option<(String, Vec, &'a Expr)> { let Expr::Attribute(attr) = expr else { return None; }; @@ -84,7 +77,7 @@ pub(super) fn expand_chain( Some(( format!("{v_form}{unwrap}.{field}"), vec![v_form], - attr.value.range(), + &attr.value, )) } } @@ -92,7 +85,8 @@ pub(super) fn expand_chain( /// builds a `None if ... is None else ...` chain from guards and final result, /// using walrus assignment to avoid evaluating compound intermediate -/// expressions twice. +/// expressions twice. `trailers` — the calls, subscripts and attributes applied to +/// the chain's last access — pass through after the result /// /// returns fragments rather than text: a compound chain base is walrus-bound /// exactly once, and passing it through as a [`Fragment::Src`] span lets @@ -100,16 +94,26 @@ pub(super) fn expand_chain( /// instead of being clobbered. only the first guard can carry the raw base — /// every later guard is an incremental `temp.suffix` — and a name-only base /// has no interior for an edit to target, so the `Src` is needed exactly there -pub(super) fn build_expansion( +/// +/// where python refuses `:=` (see [`WalrusRegions`]), the guards from the first +/// compound one on are written as the clauses of a comprehension instead, each +/// binding the temporary as its variable — `([t.c for t in (a.b,) if t is not None +/// for t in (t.d,) if t is not None] or [None])[0]` — which evaluates each guard +/// once and stops at the first `None` the same way +fn build_expansion( guards: &[String], result: &str, temp: &str, base: TextRange, + trailers: TextRange, + walrus: bool, ) -> Vec { let mut fragments: Vec = Vec::new(); let mut s = String::new(); let mut use_t = false; let mut prev_guard: Option<&str> = None; + // the comprehension's clauses, once a compound guard has opened one + let mut clauses: Vec = Vec::new(); for (position, guard) in guards.iter().enumerate() { let guard_expr = if let Some(prev) = prev_guard.filter(|_| use_t) { @@ -121,7 +125,7 @@ pub(super) fn build_expansion( if guard_expr.chars().all(|c| c.is_alphanumeric() || c == '_') { let _ = write!(s, "None if {guard_expr} is None else "); - } else { + } else if walrus { if position == 0 { let _ = write!(s, "None if ({temp} := "); fragments.push(Fragment::Lit(std::mem::take(&mut s))); @@ -131,6 +135,18 @@ pub(super) fn build_expansion( let _ = write!(s, "None if ({temp} := {guard_expr}) is None else "); } use_t = true; + } else { + if clauses.is_empty() { + s.push_str("(["); + } + clauses.push(Fragment::Lit(format!(" for {temp} in ("))); + if position == 0 { + clauses.push(Fragment::Src(base)); + } else { + clauses.push(Fragment::Lit(guard_expr)); + } + clauses.push(Fragment::Lit(format!(",) if {temp} is not None"))); + use_t = true; } prev_guard = Some(guard.as_str()); } @@ -141,10 +157,12 @@ pub(super) fn build_expansion( } else { s.push_str(result); } - if !s.is_empty() { - fragments.push(Fragment::Lit(s)); + fragments.push(Fragment::Lit(s)); + fragments.push(Fragment::Src(trailers)); + if !clauses.is_empty() { + fragments.extend(clauses); + fragments.push(Fragment::Lit("] or [None])[0]".to_owned())); } - fragments } @@ -157,7 +175,7 @@ fn chain_head<'a>( expr: &'a Expr, source: &str, types: &dyn TypeInfo, -) -> Option<(&'a Expr, String, Vec, TextRange)> { +) -> Option<(&'a Expr, String, Vec, &'a Expr)> { if let Some((form, guards, base)) = expand_chain(expr, source, types) { return Some((expr, form, guards, base)); } @@ -185,39 +203,89 @@ impl TypeAwarePass for NoneChainPass<'_> { } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = NoneChain::new(self.source, types); + let mut inner = NoneChain { + source: self.source, + types, + regions: &ctx.walrus_regions, + template_edits: Vec::new(), + errors: Vec::new(), + }; for stmt in stmts { inner.visit_stmt(stmt); } ctx.template_edits.extend(inner.template_edits); + ctx.errors.extend(inner.errors); } } impl NoneChain<'_> { + /// where python refuses `:=` the trailers move into the comprehension that binds + /// the chain's temporary. a name they read in a class body's own scope — from a + /// comprehension's first iterable there — then no longer sees what the class body + /// bound, and would fail or quietly read a module-level name of the same name. + /// such a chain is refused rather than lowered + fn refuse_class_body_names(&mut self, expr: &Expr, head: &Expr) { + struct Names<'a>(Vec<&'a ExprName>); + impl<'a> Visitor<'a> for Names<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr { + self.0.push(name); + } + walk_expr(self, expr); + } + } + let mut names = Names(Vec::new()); + trailer_operands(expr, head, &mut |operand| names.visit_expr(operand)); + // a reference ty did not place may be one, so it is refused too + if let Some(name) = names + .0 + .into_iter() + .find(|name| self.types.reads_class_scope_binding(name) != Some(false)) + { + let line = self.source[..usize::from(expr.range().start())] + .matches('\n') + .count() + + 1; + self.errors.push(format!( + "the `?.` chain on line {line} cannot be lowered: it is in a comprehension's \ + first iterable in a class body, where python refuses `:=`, and the \ + comprehension that holds its value instead would not see the class body's \ + `{}`", + name.id + )); + } + } + /// visit the sub-expressions of the trailers between `head` and `expr` — the parts that - /// pass through as source and so can carry chains of their own. `head` itself is + /// pass through as source and so can carry chains of their own. `head`'s own links are /// rendered from its expansion, not walked - fn visit_trailers(&mut self, expr: &Expr, head: &Expr) { - if expr.range() == head.range() { - return; - } - match expr { - Expr::Attribute(attribute) => self.visit_trailers(&attribute.value, head), - Expr::Call(call) => { - self.visit_trailers(&call.func, head); - for arg in &*call.arguments.args { - self.visit_expr(arg); - } - for keyword in &*call.arguments.keywords { - self.visit_expr(&keyword.value); - } + fn visit_trailers<'a>(&mut self, expr: &'a Expr, head: &'a Expr) { + trailer_operands(expr, head, &mut |operand| self.visit_expr(operand)); + } +} + +/// calls `visit` on each operand of the trailers between `head` and `expr`: the +/// arguments of a call, the index of a subscript +fn trailer_operands<'a>(expr: &'a Expr, head: &'a Expr, visit: &mut impl FnMut(&'a Expr)) { + if expr.range() == head.range() { + return; + } + match expr { + Expr::Attribute(attribute) => trailer_operands(&attribute.value, head, visit), + Expr::Call(call) => { + trailer_operands(&call.func, head, visit); + for arg in &*call.arguments.args { + visit(arg); } - Expr::Subscript(subscript) => { - self.visit_trailers(&subscript.value, head); - self.visit_expr(&subscript.slice); + for keyword in &*call.arguments.keywords { + visit(&keyword.value); } - _ => {} } + Expr::Subscript(subscript) => { + trailer_operands(&subscript.value, head, visit); + visit(&subscript.slice); + } + _ => {} } } @@ -236,11 +304,24 @@ impl<'ast> Visitor<'ast> for NoneChain<'_> { if let Some((head, form, guards, base)) = chain_head(expr, self.source, self.types) { let temp = pick_temp_var(WrittenNames::new(self.source), self.types, expr); let trailers = TextRange::new(head.range().end(), expr.range().end()); + let walrus = self.regions.allows(expr.range()); + if !walrus { + self.refuse_class_body_names(expr, head); + } let mut fragments = vec![Fragment::Lit("(".to_owned())]; - fragments.extend(build_expansion(&guards, &form, &temp, base)); - fragments.push(Fragment::Src(trailers)); + fragments.extend(build_expansion( + &guards, + &form, + &temp, + base.range(), + trailers, + walrus, + )); fragments.push(Fragment::Lit(")".to_owned())); self.template_edits.push((expr.range(), fragments)); + // the base passes through as source, so a chain or a `??` inside it + // lowers on its own + self.visit_expr(base); self.visit_trailers(expr, head); return; } @@ -436,6 +517,60 @@ mod tests { check("x = z[a?.b]\n", "x = z[(None if a is None else a.b)]\n"); } + /// a chain's base passes through as source, so a chain inside it, behind a call, + /// lowers on its own + #[test] + fn chain_through_a_call() { + check( + "x = a?.b()?.c\n", + "x = (None if (__by_t_0__ := (None if a is None else a.b())) is None else __by_t_0__.c)\n", + ); + } + + /// python refuses `:=` in a comprehension's iterable, so the guards are the clauses + /// of a comprehension there, binding the temporary as its variable + #[test] + fn chain_in_a_comprehension_iterable() { + check( + "x = [i for i in f()?.b?.c(1)]\n", + "x = [i for i in (([__by_t_0__.c(1) for __by_t_0__ in (f(),) if __by_t_0__ is not None for __by_t_0__ in (__by_t_0__.b,) if __by_t_0__ is not None] or [None])[0])]\n", + ); + check( + "x = [i for i in a?.b?.c]\n", + "x = [i for i in (None if a is None else ([__by_t_0__.c for __by_t_0__ in (a.b,) if __by_t_0__ is not None] or [None])[0])]\n", + ); + } + + /// moving the trailers into that comprehension would take them out of a class + /// body's scope, so a name the class body binds there is refused + #[test] + fn class_body_name_in_a_moved_trailer_is_refused() { + let err = transpile( + "class C:\n k = 1\n x = [i for i in f()?.g(k)]\n", + &Config::test_default(), + ) + .unwrap_err(); + assert!( + err.contains("the `?.` chain on line 3 cannot be lowered") && err.contains("`k`"), + "got: {err}" + ); + } + + /// a builtin, a module-level name and an enclosing function's local resolve the + /// same from the comprehension, so a chain reading them is lowered + #[test] + fn a_name_the_class_body_does_not_bind_is_lowered() { + for source in [ + "class C:\n x = [i for i in f()?.g(len)]\n", + "k = 1\nclass C:\n x = [i for i in f()?.g(k)]\n", + "def outer():\n k = 3\n class C:\n x = [i for i in f()?.g(k)]\n return C\n", + "class C:\n x = [i for i in f()?.g(k)]\n k = 1\n", + ] { + transpile(source, &Config::test_default()) + .unwrap_or_else(|err| panic!("refused {source:?}: {err}")); + } + } + #[test] fn python_unchanged() { unchanged("x = None if a is None else a.b\n"); diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index 847bb19895..5d52f0fec3 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -56,7 +56,9 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{self as ast, CmpOp, Expr, PySourceType, Stmt}; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange}; -use ty_python_semantic::{ArgVariance, ParametricIsPlan, ProtocolMemberCheck, TargetSpelling}; +use ty_python_semantic::{ + ArgVariance, ParametricIsPlan, ProtocolMemberCheck, RuntimeSpelling, TargetSpelling, +}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::repeated_underscore::WrittenNames; @@ -101,9 +103,13 @@ fn variance_code(variance: ArgVariance) -> u8 { /// validates the same structural claim. each entry is a kind-tagged tuple: /// `(\"attr\", name, type, variance)` or `(\"method\", name, [(type, variance), /// …], return_or_None)` -fn protocol_members_literal(checks: &[ProtocolMemberCheck]) -> String { - let type_variance = |(expected, variance): &(String, ArgVariance)| { - format!("({expected}, {})", variance_code(*variance)) +fn protocol_members_literal(written: WrittenNames, checks: &[ProtocolMemberCheck]) -> String { + let type_variance = |(expected, variance): &(RuntimeSpelling, ArgVariance)| { + format!( + "({}, {})", + written.spelled(expected), + variance_code(*variance) + ) }; let entries = checks .iter() @@ -113,7 +119,8 @@ fn protocol_members_literal(checks: &[ProtocolMemberCheck]) -> String { expected, variance, } => format!( - "(\"attr\", {name:?}, {expected}, {})", + "(\"attr\", {name:?}, {}, {})", + written.spelled(expected), variance_code(*variance) ), ProtocolMemberCheck::Method { name, params, ret } => { @@ -269,7 +276,7 @@ fn fragments_for_plan( Fragment::Lit(call.to_owned()), value_ref(), Fragment::Lit(", ".to_owned()), - target_fragment(&target, arm), + target_fragment(written, &target, arm), ]; frags.push(Fragment::Lit(format!(", {})", variance_tuple(&codes)))); frags @@ -277,7 +284,7 @@ fn fragments_for_plan( ParametricIsPlan::ProtocolStructural(checks) => { needs.protocol_runtime = true; needs.references_value = true; - let members = protocol_members_literal(&checks); + let members = protocol_members_literal(written, &checks); vec![ Fragment::Lit("_by_protocol_is(".to_owned()), value_ref(), @@ -291,7 +298,9 @@ fn fragments_for_plan( // `isinstance`: a conforming type is not a subclass. the registry // answers first, and a value nothing registered falls back to carrying // the requirements - ParametricIsPlan::Conformance { target, members } => { + ParametricIsPlan::Conformance { + target, members, .. + } => { needs.references_value = true; needs.conformance_runtime = true; let mut spelled = String::from(", ("); @@ -303,7 +312,7 @@ fn fragments_for_plan( Fragment::Lit("_by_conforms(".to_owned()), value_ref(), Fragment::Lit(", ".to_owned()), - target_fragment(&target, arm), + target_fragment(written, &target, arm), Fragment::Lit(format!("{spelled})")), ] } @@ -337,7 +346,7 @@ fn fragments_for_plan( value_ref(), Fragment::Lit(", ".to_owned()), ]; - frags.push(target_fragment(&target, arm)); + frags.push(target_fragment(written, &target, arm)); frags.push(Fragment::Lit(")".to_owned())); frags } @@ -366,7 +375,7 @@ fn fragments_for_plan( Fragment::Lit(format!(") is {} and ", written.builtin(&class))), value_ref(), Fragment::Lit(" == ".to_owned()), - target_fragment(&value, arm), + target_fragment(written, &value, arm), Fragment::Lit(")".to_owned()), ] } @@ -375,7 +384,7 @@ fn fragments_for_plan( vec![ value_ref(), Fragment::Lit(" is ".to_owned()), - target_fragment(&target, arm), + target_fragment(written, &target, arm), ] } ParametricIsPlan::Subclass(target) => { @@ -391,7 +400,7 @@ fn fragments_for_plan( ))); frags.push(value_ref()); frags.push(Fragment::Lit(", ".to_owned())); - frags.push(target_fragment(&target, arm)); + frags.push(target_fragment(written, &target, arm)); frags.push(Fragment::Lit("))".to_owned())); frags } @@ -436,10 +445,10 @@ fn fragments_for_plan( /// how a plan's target is written: its own spelling, or the source the target /// was written as when the plan carries none -fn target_fragment(target: &TargetSpelling, arm: &Expr) -> Fragment { +fn target_fragment(written: WrittenNames, target: &TargetSpelling, arm: &Expr) -> Fragment { match target { TargetSpelling::Written => Fragment::Src(arm.range()), - TargetSpelling::Rebuilt(text) => Fragment::Lit(text.clone()), + TargetSpelling::Rebuilt(spelling) => Fragment::Lit(written.spelled(spelling)), } } @@ -740,12 +749,21 @@ impl TypeAwarePass for ParametricIsPass<'_> { Some(super::ast_driver::Lowering::ParametricIs) } - /// the type it tests against is printed as the runtime probe it builds, literals and - /// the `None` of an optional included + /// the runtime probe is built from the checker's reading of the type it tests against, + /// and the type is left out where the checker settles the test without it, so it writes + /// these lowerings of the type as written itself — literals, the `None` of an optional + /// and the emitted name of a `private` class among them. where the probe passes the + /// type as written through, the lowerings inside pass through with it fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { &[ + super::ast_driver::Lowering::AnonNamedTuple, + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::JustFloat, super::ast_driver::Lowering::LiteralType, super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::Typeof, + super::ast_driver::Lowering::VisibilityRename, ] } diff --git a/crates/by_transforms/src/transforms/propagate.rs b/crates/by_transforms/src/transforms/propagate.rs index a4d98daeae..3465b6ebaf 100644 --- a/crates/by_transforms/src/transforms/propagate.rs +++ b/crates/by_transforms/src/transforms/propagate.rs @@ -28,7 +28,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{PassContext, TypeAwarePass}; use super::repeated_underscore::WrittenNames; -use super::source_util::temporary_name; +use super::source_util::{standalone_expression_text, temporary_name}; use crate::type_info::{AbsentTest, TypeInfo}; /// The guard condition that detects the "absent" value for a given operand @@ -194,8 +194,9 @@ impl<'ast> Visitor<'ast> for Propagate<'_> { let temp = temporary_name(self.written, "prop", self.counter); self.counter += 1; let cond = absent_condition(absent, &temp, self.written); + let operand = standalone_expression_text(self.source, unary.operand.range()); ( - format!("{temp} = {operand_src}\n{indent}if {cond}: return {temp}\n{indent}"), + format!("{temp} = {operand}\n{indent}if {cond}: return {temp}\n{indent}"), format!("{temp}{unwrap}"), ) }; diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index 4c2e0c3d29..30d1d7710a 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -43,6 +43,7 @@ use crate::type_info::TypeInfo; use super::ast_driver::{PassContext, TypeAwarePass}; use super::callable::CallableSyntax; +use super::quotes::WrittenQuotes; use super::repeated_underscore::{WrittenNames, protocol_method_receiver}; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping}; @@ -78,6 +79,7 @@ impl Shape { pub(crate) struct ProtocolTypePass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: crate::Config, } @@ -85,11 +87,13 @@ impl<'src> ProtocolTypePass<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: crate::Config, ) -> Self { Self { source, written, + quotes, config, } } @@ -98,6 +102,8 @@ impl<'src> ProtocolTypePass<'src> { struct ProtocolTypeLowering<'src> { source: &'src str, written: WrittenNames<'src>, + /// where the member types are written as strings + quotes: &'src WrittenQuotes, /// One lowerer drives every member type in the run, so its imports and the /// `_Callable_*` classes it synthesizes can be collected at the end. The /// `callable` pass deliberately does not descend into an inline protocol, @@ -119,6 +125,7 @@ impl<'src> ProtocolTypeLowering<'src> { fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, claimed: &'src [TextRange], config: &Config, @@ -126,7 +133,8 @@ impl<'src> ProtocolTypeLowering<'src> { Self { source, written, - callable: CallableSyntax::new(source, written, config) + quotes, + callable: CallableSyntax::new(source, written, quotes, config) .with_types(types) .with_claimed_ranges(claimed), edits: Vec::new(), @@ -192,7 +200,7 @@ impl<'src> ProtocolTypeLowering<'src> { let line = format!( "def {name}({params}) -> {returns}: ...", name = method.name.id, - returns = quote_forward_ref(&returns), + returns = self.quotes.quote(&returns), ); Some(self.apply_typevar_renames(method.range(), &line)) } @@ -205,7 +213,7 @@ impl<'src> ProtocolTypeLowering<'src> { let Some(name) = named.target.as_name_expr() else { continue; }; - let ty = quote_forward_ref(&self.render_type(&named.value)); + let ty = self.quotes.quote(&self.render_type(&named.value)); // a member that only has to be readable is a getter-only property, // which is how a protocol class body says read-only members.push(if named.is_let { @@ -318,11 +326,6 @@ impl TypeExprVisitor for ProtocolTypeLowering<'_> { } } -/// Wrap a rendered member type in a forward-reference string. -fn quote_forward_ref(ty: &str) -> String { - format!("\"{}\"", ty.replace('\\', "\\\\").replace('"', "\\\"")) -} - /// Lowers inline protocols that appear in a value position (`x = protocol(a: /// int)`). `protocol(...)` is never valid python wherever it appears, so it has /// to be replaced everywhere, not just in the annotation slots the type-position @@ -376,12 +379,13 @@ impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for TypevarScopeWalker<'_> { fn lower<'src>( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, claimed: &'src [TextRange], stmts: &[Stmt], config: &crate::Config, ) -> ProtocolTypeLowering<'src> { - let mut inner = ProtocolTypeLowering::new(source, written, types, claimed, config); + let mut inner = ProtocolTypeLowering::new(source, written, quotes, types, claimed, config); let polyfilled = super::generics::PolyfilledTypeParams::decide(source, written, stmts, config); { let mut walker = TypevarScopeWalker { @@ -410,11 +414,12 @@ fn lower<'src>( pub(crate) fn cleanup( source: &str, written: WrittenNames, + quotes: &WrittenQuotes, types: &dyn TypeInfo, stmts: &[Stmt], config: &crate::Config, ) -> Result, String)>, String> { - let mut inner = lower(source, written, types, &[], stmts, config); + let mut inner = lower(source, written, quotes, types, &[], stmts, config); if let Some(error) = inner.errors.first() { return Err(error.clone()); } @@ -479,6 +484,7 @@ impl TypeAwarePass for ProtocolTypePass<'_> { let mut inner = lower( self.source, self.written, + self.quotes, types, &claimed, stmts, diff --git a/crates/by_transforms/src/transforms/python_scopes.rs b/crates/by_transforms/src/transforms/python_scopes.rs new file mode 100644 index 0000000000..f60147477a --- /dev/null +++ b/crates/by_transforms/src/transforms/python_scopes.rs @@ -0,0 +1,298 @@ +//! the scope python evaluates each part of a module in, as a walk over its syntax tree +//! +//! a definition's body is a scope of its own, and the rest of the definition is not: +//! a decorator, a parameter's default and a base class are evaluated where the +//! definition is written. a comprehension's first iterable is evaluated where the +//! comprehension is written, and everything else in it in its own scope. an annotation +//! is evaluated in a scope of its own from 3.14, and the type parameters of a generic +//! definition, with everything that can name them — a generic class's bases, a generic +//! function's annotations, a `type` alias's value — in one more +//! +//! both the lowerings that bind a temporary and the check that the emitted module +//! compiles need these rules, so they are stated once here + +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{AnyParameterRef, Comprehension, Expr, Parameters, Stmt}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Scope { + Class, + Function { + is_async: bool, + }, + Lambda, + /// a list, set or dict comprehension, or a generator expression + Comprehension { + generator: bool, + is_async: bool, + }, + /// an annotation + Annotation, + /// the type parameters of a generic definition + TypeParams, +} + +/// where the walk is: the scopes around the node, innermost last (the module is the +/// empty stack), and the constructs around it that python's rules ask about +#[derive(Default)] +pub(crate) struct ScopeState { + scopes: Vec, + /// how many comprehension iterables the node is in + iterables: usize, + /// whether the node is in a loop's body, short of a scope opened inside it + in_loop: bool, + /// whether the node is in a `try` statement + in_try: bool, +} + +impl ScopeState { + pub(crate) fn scopes(&self) -> &[Scope] { + &self.scopes + } + + pub(crate) fn in_iterable(&self) -> bool { + self.iterables > 0 + } + + pub(crate) fn in_loop(&self) -> bool { + self.in_loop + } + + pub(crate) fn in_try(&self) -> bool { + self.in_try + } +} + +/// what a walk does at each statement and expression, before it walks into it +pub(crate) trait ScopeHooks<'a> { + fn stmt(&mut self, _stmt: &'a Stmt, _state: &ScopeState) {} + fn expr(&mut self, _expr: &'a Expr, _state: &ScopeState) {} +} + +/// walks `suite`, calling `hooks` at every statement and expression with the scope +/// state it is evaluated in +pub(crate) fn walk<'a>(suite: &'a [Stmt], hooks: &mut impl ScopeHooks<'a>) { + let mut walker = Walker { + state: ScopeState::default(), + hooks, + }; + for stmt in suite { + walker.visit_stmt(stmt); + } +} + +struct Walker<'h, H> { + state: ScopeState, + hooks: &'h mut H, +} + +impl<'a, H: ScopeHooks<'a>> Walker<'_, H> { + fn in_scope(&mut self, scope: Scope, visit: impl FnOnce(&mut Self)) { + self.state.scopes.push(scope); + visit(self); + self.state.scopes.pop(); + } + + /// a body is a scope of its own, and a loop around the definition does not + /// reach into it + fn in_body(&mut self, scope: Scope, visit: impl FnOnce(&mut Self)) { + let in_loop = std::mem::replace(&mut self.state.in_loop, false); + self.in_scope(scope, visit); + self.state.in_loop = in_loop; + } + + fn in_loop_body(&mut self, visit: impl FnOnce(&mut Self)) { + let in_loop = std::mem::replace(&mut self.state.in_loop, true); + visit(self); + self.state.in_loop = in_loop; + } + + fn visit_iterable(&mut self, iterable: &'a Expr) { + self.state.iterables += 1; + self.visit_expr(iterable); + self.state.iterables -= 1; + } + + fn visit_defaults(&mut self, parameters: &'a Parameters) { + for default in parameters + .iter_non_variadic_params() + .filter_map(|parameter| parameter.default.as_deref()) + { + self.visit_expr(default); + } + } + + fn visit_parameter_annotations(&mut self, parameters: &'a Parameters) { + for parameter in parameters.iter().map(AnyParameterRef::as_parameter) { + self.visit_parameter(parameter); + } + } + + fn visit_comprehension_scope( + &mut self, + generators: &'a [Comprehension], + elements: &[&'a Expr], + generator: bool, + ) { + let Some((first, rest)) = generators.split_first() else { + return; + }; + self.visit_iterable(&first.iter); + let is_async = generators.iter().any(|generator| generator.is_async); + self.in_scope( + Scope::Comprehension { + generator, + is_async, + }, + |walker| { + walker.visit_expr(&first.target); + for condition in &first.ifs { + walker.visit_expr(condition); + } + for generator in rest { + walker.visit_expr(&generator.target); + walker.visit_iterable(&generator.iter); + for condition in &generator.ifs { + walker.visit_expr(condition); + } + } + for element in elements { + walker.visit_expr(element); + } + }, + ); + } +} + +impl<'a, H: ScopeHooks<'a>> Visitor<'a> for Walker<'_, H> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + self.hooks.stmt(stmt, &self.state); + match stmt { + Stmt::ClassDef(class) => { + for decorator in &class.decorator_list { + self.visit_decorator(decorator); + } + match &class.type_params { + Some(type_params) => self.in_scope(Scope::TypeParams, |walker| { + walker.visit_type_params(type_params); + if let Some(arguments) = &class.arguments { + walker.visit_arguments(arguments); + } + }), + None => { + if let Some(arguments) = &class.arguments { + self.visit_arguments(arguments); + } + } + } + self.in_body(Scope::Class, |walker| walker.visit_body(&class.body)); + } + Stmt::FunctionDef(function) => { + for decorator in &function.decorator_list { + self.visit_decorator(decorator); + } + self.visit_defaults(&function.parameters); + let annotations = |walker: &mut Self| { + walker.visit_parameter_annotations(&function.parameters); + if let Some(returns) = &function.returns { + walker.visit_annotation(returns); + } + }; + match &function.type_params { + Some(type_params) => self.in_scope(Scope::TypeParams, |walker| { + walker.visit_type_params(type_params); + annotations(walker); + }), + None => annotations(self), + } + let scope = Scope::Function { + is_async: function.is_async, + }; + self.in_body(scope, |walker| walker.visit_body(&function.body)); + } + Stmt::TypeAlias(alias) => { + self.visit_expr(&alias.name); + self.in_scope(Scope::TypeParams, |walker| { + if let Some(type_params) = &alias.type_params { + walker.visit_type_params(type_params); + } + walker.visit_expr(&alias.value); + for case in &alias.cases { + walker.visit_match_case(case); + } + }); + } + Stmt::For(for_stmt) => { + self.visit_expr(&for_stmt.iter); + self.visit_expr(&for_stmt.target); + if let Some(pattern) = &for_stmt.pattern { + self.visit_pattern(pattern); + } + self.in_loop_body(|walker| walker.visit_body(&for_stmt.body)); + self.visit_body(&for_stmt.orelse); + } + Stmt::While(while_stmt) => { + self.visit_expr(&while_stmt.test); + self.in_loop_body(|walker| walker.visit_body(&while_stmt.body)); + self.visit_body(&while_stmt.orelse); + } + Stmt::Try(_) => { + let in_try = std::mem::replace(&mut self.state.in_try, true); + walk_stmt(self, stmt); + self.state.in_try = in_try; + } + _ => walk_stmt(self, stmt), + } + } + + fn visit_annotation(&mut self, expr: &'a Expr) { + self.in_scope(Scope::Annotation, |walker| walker.visit_expr(expr)); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + self.hooks.expr(expr, &self.state); + match expr { + Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + self.visit_defaults(parameters); + self.visit_parameter_annotations(parameters); + } + if let Some(returns) = &lambda.returns { + self.visit_annotation(returns); + } + self.in_body(Scope::Lambda, |walker| walker.visit_expr(&lambda.body)); + } + Expr::ListComp(comprehension) => { + self.visit_comprehension_scope( + &comprehension.generators, + &[&comprehension.elt], + false, + ); + } + Expr::SetComp(comprehension) => { + self.visit_comprehension_scope( + &comprehension.generators, + &[&comprehension.elt], + false, + ); + } + Expr::Generator(comprehension) => { + self.visit_comprehension_scope( + &comprehension.generators, + &[&comprehension.elt], + true, + ); + } + Expr::DictComp(comprehension) => { + let elements: Vec<&Expr> = comprehension + .key + .as_deref() + .into_iter() + .chain([&*comprehension.value]) + .collect(); + self.visit_comprehension_scope(&comprehension.generators, &elements, false); + } + _ => walk_expr(self, expr), + } + } +} diff --git a/crates/by_transforms/src/transforms/quotes.rs b/crates/by_transforms/src/transforms/quotes.rs new file mode 100644 index 0000000000..715d197b6f --- /dev/null +++ b/crates/by_transforms/src/transforms/quotes.rs @@ -0,0 +1,63 @@ +//! the forward references the transpiler writes as strings +//! +//! python evaluates such a string against the module's namespace whenever something asks for +//! the type, so the lazy-import polyfill binds every name one reads where the name is +//! imported. writing a quote is a capability: only a holder of [`WrittenQuotes`] can write +//! one, as text ([`WrittenQuotes::quote`]) or as a passthrough of source the driver lowers in +//! place ([`WrittenQuotes::quote_span`]), and each is recorded as it is written. code that only +//! picks names never holds one + +use std::cell::RefCell; + +use ruff_text_size::TextRange; + +use super::ast_driver::Fragment; +use super::source_util::python_string_literal; + +/// every forward reference the module's lowerings write as a string, by the type expression +/// inside the quotes +#[derive(Debug, Default)] +pub(crate) struct WrittenQuotes { + texts: RefCell>, +} + +impl WrittenQuotes { + /// `ty`, a lowered type expression, written as the forward reference python reads lazily + pub(crate) fn quote(&self, ty: &str) -> String { + self.record(ty); + python_string_literal(ty) + } + + /// `range` written as a string, with whatever lowers inside it lowered too. the driver + /// records what it lowered to once it has written it + #[expect( + clippy::unused_self, + reason = "taking the registry is what makes writing a quote a capability of its holder" + )] + pub(crate) fn quote_span(&self, range: TextRange) -> Fragment { + Fragment::Quote(QuotedSpan { range }) + } + + /// record `ty`, the type expression a quote the driver wrote holds + pub(super) fn record(&self, ty: &str) { + self.texts.borrow_mut().push(ty.to_owned()); + } + + /// the type expressions the recorded quotes hold + pub(crate) fn texts(&self) -> Vec { + self.texts.borrow().clone() + } +} + +/// the source a [`Fragment::Quote`] writes as a string, which only +/// [`WrittenQuotes::quote_span`] makes +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct QuotedSpan { + range: TextRange, +} + +impl QuotedSpan { + pub(crate) fn range(self) -> TextRange { + self.range + } +} diff --git a/crates/by_transforms/src/transforms/raises_clause.rs b/crates/by_transforms/src/transforms/raises_clause.rs index 95fe886bc0..b0b1bbf4f5 100644 --- a/crates/by_transforms/src/transforms/raises_clause.rs +++ b/crates/by_transforms/src/transforms/raises_clause.rs @@ -25,6 +25,7 @@ use ruff_python_ast::{ModModule, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; /// Deletes every `raises` clause. @@ -85,12 +86,17 @@ fn clause_range(source: &str, function: &StmtFunctionDef) -> Option { /// Wraps each function with a runtime-testable `raises` clause in a guard. pub(crate) struct RaisesGuardPass<'src> { source: &'src str, + written: WrittenNames<'src>, enabled: bool, } impl<'src> RaisesGuardPass<'src> { - pub(crate) fn new(source: &'src str, enabled: bool) -> Self { - Self { source, enabled } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, enabled: bool) -> Self { + Self { + source, + written, + enabled, + } } } @@ -113,6 +119,7 @@ impl super::ast_driver::TypeAwarePass for RaisesGuardPass<'_> { for stmt in stmts { let mut collector = GuardCollector { source: self.source, + written: self.written, types, guards: &mut guards, }; @@ -135,6 +142,7 @@ impl super::ast_driver::TypeAwarePass for RaisesGuardPass<'_> { /// exactly as much in need of its contract as a top-level one. struct GuardCollector<'a> { source: &'a str, + written: WrittenNames<'a>, types: &'a dyn TypeInfo, guards: &'a mut Vec<(TextRange, String)>, } @@ -142,7 +150,7 @@ struct GuardCollector<'a> { impl<'ast> Visitor<'ast> for GuardCollector<'_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { if let Stmt::FunctionDef(function) = stmt - && let Some(guard) = guard_for(self.source, function, self.types) + && let Some(guard) = guard_for(self.source, self.written, function, self.types) { self.guards.push(guard); } @@ -161,6 +169,7 @@ impl<'ast> Visitor<'ast> for GuardCollector<'_> { /// there, so they stay above the decorator rather than between it and its `def`. fn guard_for( source: &str, + written: WrittenNames, function: &StmtFunctionDef, types: &dyn TypeInfo, ) -> Option<(TextRange, String)> { @@ -172,6 +181,7 @@ fn guard_for( let target = types.declared_raises_runtime_target(function)?; let (offset, indent) = def_keyword(source, function)?; let name = function.name.as_str(); + let ceiling = written.spelled(&target.ceiling); let arguments = match &target.resolved { Some(resolved) => { @@ -181,19 +191,19 @@ fn guard_for( .chain(&resolved.receiver) .map(String::as_str) .collect(); + let expression = written.spelled(&resolved.expression); let lambda = if parameters.is_empty() { - format!("lambda: {}", resolved.expression) + format!("lambda: {expression}") } else { - format!("lambda {}: {}", parameters.join(", "), resolved.expression) + format!("lambda {}: {expression}", parameters.join(", ")) }; format!( - "{}, \"{name}\", {lambda}, {}, {}", - target.ceiling, + "{ceiling}, \"{name}\", {lambda}, {}, {}", python_names(&resolved.own), python_names(&resolved.receiver), ) } - None => format!("{}, \"{name}\"", target.ceiling), + None => format!("{ceiling}, \"{name}\""), }; Some(( diff --git a/crates/by_transforms/src/transforms/reified_generic.rs b/crates/by_transforms/src/transforms/reified_generic.rs index ade4a6c1a3..e6fbeb6fac 100644 --- a/crates/by_transforms/src/transforms/reified_generic.rs +++ b/crates/by_transforms/src/transforms/reified_generic.rs @@ -54,6 +54,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::reified::reified_type_param_names; use super::ast_driver::{PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use super::source_util::{line_indent, line_start}; use crate::type_info::TypeInfo; @@ -147,6 +148,7 @@ impl<'ast> Visitor<'ast> for ReifiedGeneric<'_> { /// inserts the inferred `[...]` specialization at bare reified call sites struct SpecializationInjector<'ti> { types: &'ti dyn TypeInfo, + written: WrittenNames<'ti>, edits: Vec<(TextRange, String)>, } @@ -157,8 +159,10 @@ impl<'ast> Visitor<'ast> for SpecializationInjector<'_> { && !matches!(call.func.as_ref(), Expr::Subscript(_)) && let Some(specialization) = self.types.reified_call_specialization(call) { - self.edits - .push((TextRange::empty(call.func.range().end()), specialization)); + self.edits.push(( + TextRange::empty(call.func.range().end()), + self.written.spelled(&specialization), + )); } walk_expr(self, expr); } @@ -166,13 +170,19 @@ impl<'ast> Visitor<'ast> for SpecializationInjector<'_> { pub(crate) struct ReifiedGenericPass<'src> { source: &'src str, + written: WrittenNames<'src>, min_version: PythonVersion, } impl<'src> ReifiedGenericPass<'src> { - pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + min_version: PythonVersion, + ) -> Self { Self { source, + written, min_version, } } @@ -196,6 +206,7 @@ impl TypeAwarePass for ReifiedGenericPass<'_> { // call only when this injection exists, so the two stay in agreement let mut injector = SpecializationInjector { types, + written: self.written, edits: Vec::new(), }; for stmt in stmts { diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index 12d3c15a66..a4e87ee078 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -24,6 +24,7 @@ use ruff_python_ast::visitor::{ use ruff_python_ast::{Expr, ModModule, Parameter, Parameters, Stmt}; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange}; +use ty_python_semantic::RuntimeSpelling; use ty_python_semantic::types::repeated_underscore::{ self as decision, UnderscoreLowering, UnderscoreRefusal, callable_parameter_slots, lower_repeated_underscores, lowered_parameter_names, parameter_slots, @@ -97,7 +98,7 @@ pub(crate) fn collect(suite: &[Stmt], types: &dyn TypeInfo, slash: bool) -> Unde /// reads the same answer: [`WrittenNames::with_lowerings`] these, and every name and `/` /// it writes is the transpiler's pub fn repeated_underscore_lowerings( - model: &ty_python_semantic::SemanticModel<'_>, + model: &ty_python_semantic::LoweringModel<'_>, suite: &[Stmt], ) -> UnderscoreLowerings { collect(suite, model, true) @@ -567,7 +568,7 @@ impl<'src> WrittenNames<'src> { } /// [`Self::fresh`], also passing over every name `taken` answers for - fn fresh_outside(self, stem: &str, taken: impl Fn(&str) -> bool) -> String { + pub(crate) fn fresh_outside(self, stem: &str, taken: impl Fn(&str) -> bool) -> String { self.names .fresh_outside(stem, |name| self.lowering_binds(name) || taken(name)) } @@ -612,6 +613,12 @@ impl<'src> WrittenNames<'src> { } } + /// `spelling`, which the checker spelled for this module, as the module is emitted: each + /// module-level `private` symbol it reads under the name the symbol is emitted under + pub(crate) fn spelled(self, spelling: &RuntimeSpelling) -> String { + spelling.spelled(|symbol| self.module_private(symbol)) + } + /// the name a lowering writes `name`, which `module` exports, under — `typing`, /// `typing_extensions`, `collections.abc` and the like /// @@ -1093,6 +1100,16 @@ mod tests { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + fn refused(input: &str) -> String { transpile(input, &Config::test_default()).unwrap_err() } @@ -1295,9 +1312,9 @@ mod tests { /// that re-evaluates `f`'s mutable default there #[test] fn a_numbered_name_skips_a_name_the_module_binds() { - check( + check_beside_runtime( "_2: list[int] = [9]\n\ndef f(_: int, _: int, x: list[int] = _2) -> list[int]:\n return x\n\ndef g(_: int, _: int) -> list[int]:\n return _2\n", - "from typing import Any\n_MISSING: Any = object()\n_2: list[int] = [9]\n\ndef f(_: int, _3: int, /, x: list[int] = _MISSING) -> list[int]:\n if x is _MISSING:\n x = _2\n return x\n\ndef g(_: int, _3: int, /) -> list[int]:\n return _2\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"_2\", __name__, \"_MISSING\")\n_2: list[int] = [9]\n\ndef f(_: int, _3: int, /, x: list[int] = _MISSING) -> list[int]:\n if x is _MISSING:\n x = _2\n return x\n\ndef g(_: int, _3: int, /) -> list[int]:\n return _2\n", ); } diff --git a/crates/by_transforms/src/transforms/rerender.rs b/crates/by_transforms/src/transforms/rerender.rs index 8a563e9174..c4e47db861 100644 --- a/crates/by_transforms/src/transforms/rerender.rs +++ b/crates/by_transforms/src/transforms/rerender.rs @@ -1848,9 +1848,9 @@ mod tests { def f(_: list[int] = [], _: list[int] = []) -> int: return len(_) "}); - assert!(out.contains("if _ is _MISSING:"), "got:\n{out}"); + assert!(out.contains("if _ is _MISSING"), "got:\n{out}"); assert!( - out.contains("if _2 is _MISSING:\n _2 = []"), + out.contains("if _2 is _MISSING2:\n _2 = []"), "got:\n{out}" ); } @@ -2018,6 +2018,7 @@ mod tests { .map(|fragment| match fragment { Fragment::Lit(text) => text.clone(), Fragment::Src(range) => source[*range].to_string(), + Fragment::Quote(quoted) => source[quoted.range()].to_string(), }) .collect() }) @@ -2071,6 +2072,7 @@ mod tests { .map(|fragment| match fragment { Fragment::Lit(text) => text.clone(), Fragment::Src(range) => source[*range].to_string(), + Fragment::Quote(quoted) => source[quoted.range()].to_string(), }) .collect() }) diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index a8f83f7346..93555ec2c7 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -34,6 +34,13 @@ //! no check. A check whose target is exactly `type(None)` is dropped too: //! validating a `None` result guards no data flowing onward. //! +//! A check is made only where every name its target reads holds the class the +//! checker resolved. In a function, which runs whenever it is called, a name +//! the module binds to one class and nothing else — a class declared further +//! down — is looked up as the check runs instead: +//! `_soundness_once_bound(("Cut",), expr, _soundness_check, lambda: Cut)` makes +//! the check once `Cut` is bound and passes the value unchecked before. +//! //! When the target is a *user-defined* generic specialization (`A[int]`), the //! check deepens to `_soundness_parametric(expr, A[int], variances)`, which //! validates the base class *and* — when the value carries its reified @@ -60,22 +67,19 @@ //! //! Known gaps, deliberate for now: no checks on `await` results, unpacking //! targets, `*args`/`**kwargs` spreads, or arguments to non-function callees -//! (class constructors, overloaded functions); a check argument naming a class -//! defined later in the module can raise `NameError` if the checked line runs -//! at import time before the class body. +//! (class constructors, overloaded functions). use std::cell::Cell; use std::collections::HashMap; use ruff_python_ast::name::Name; use ruff_python_ast::visitor::transformer::{self, Transformer}; -use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{ - AtomicNodeIndex, Comprehension, Expr, ExprCall, ExprName, HasNodeIndex, Parameter, Stmt, - StmtFunctionDef, UnaryOp, + AtomicNodeIndex, Expr, ExprCall, ExprName, HasNodeIndex, Stmt, StmtFunctionDef, }; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::types::soundness::CheckTarget; +use ty_python_semantic::types::soundness_sites::SoundnessSite; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::parametric_is::variance_tuple; @@ -83,7 +87,10 @@ use super::repeated_underscore::WrittenNames; use super::source_util::{PrologueStatement, body_prologue, needs_parentheses_as_argument}; use crate::Config; use crate::config::SoundnessPositions; -use crate::type_info::{SoundnessCheck, TypeInfo}; +use crate::type_info::{SiteCheck, SoundnessCheck, TypeInfo}; + +/// the helper a check is made through when its target reads a name that may not be bound yet +const ONCE_BOUND: &str = crate::runtime::SOUNDNESS_ONCE_BOUND.name(); // deep check for a user-generic-specialized target: validates the base class // always, and the reified type arguments when the value carries them @@ -92,326 +99,63 @@ use crate::type_info::{SoundnessCheck, TypeInfo}; // leaving the base `isinstance` as the guarantee. reuses `_parametric_is` // (and its `_parametric_is_sub`) from `PARAMETRIC_IS_RUNTIME` -/// which parameter an argument binds to, for the `arguments` gate -enum ArgSlot<'a> { - Positional(usize), - Keyword(&'a str), -} - -/// one check the pass decided on, before anything is written for it +/// one check the pass makes, before anything is written for it: a site the checker's +/// [`soundness_sites`](ty_python_semantic::types::soundness_sites::soundness_sites) +/// decided on, with each checked parameter named as python binds it enum Site<'ast> { /// the value `expr` produces is checked as it is produced - Value { - expr: &'ast Expr, - plan: SoundnessCheck, - }, + Value { expr: &'ast Expr, check: SiteCheck }, /// each element drawn from `iterable` is checked as it is drawn Elements { iterable: &'ast Expr, is_async: bool, - plan: SoundnessCheck, + check: SiteCheck, }, /// `function`'s own parameters are checked where its body begins Parameters { function: &'ast StmtFunctionDef, /// each checked parameter by the name python binds it to, which for a repeated /// `_` is not the one the source spells - guards: Vec<(Name, SoundnessCheck)>, + guards: Vec<(Name, SiteCheck)>, }, } -/// where the checks go, decided once over a syntax tree, and what is not a place -/// for one -struct Soundness<'a, 'ast> { - types: &'a dyn TypeInfo, +/// the checks `suite` makes under `positions`, as the checker decides them. `written` is +/// the module as written, which a repeated `_` parameter's name is numbered around +fn sites<'ast>( + types: &dyn TypeInfo, + written: WrittenNames, + suite: &'ast [Stmt], positions: SoundnessPositions, - /// the module as written, which a repeated `_` parameter's name is numbered around - written: WrittenNames<'a>, - sites: Vec>, - /// gated expressions already covered by a wrap on an enclosing `!` - /// force-unwrap — wrapping them directly would splice the check's second - /// argument into `_force_unwrap`'s call parens - consumed: Vec, - /// declared-return-type check plans of the enclosing functions (innermost - /// last); `None` when a function has no annotation or an uncheckable one - return_targets: Vec>, -} - -impl<'a, 'ast> Soundness<'a, 'ast> { - fn new( - types: &'a dyn TypeInfo, - positions: SoundnessPositions, - written: WrittenNames<'a>, - ) -> Self { - Self { - types, - positions, - written, - sites: Vec::new(), - consumed: Vec::new(), - return_targets: Vec::new(), - } - } - - /// whether `expr` is a value this pass classifies as resting on an - /// unverified claim: a call whose result type came from a typevar - /// solution, or an element read out of a specialized container. this is - /// the position-agnostic *classification*; the direct-wrap sites also - /// require the corresponding position to be enabled ([`Self::gated_enabled`]) - fn gated(&self, expr: &Expr) -> bool { - match expr { - Expr::Call(call) => self.types.call_result_is_typevar_derived(&call.func), - Expr::Subscript(subscript) => { - subscript.ctx.is_load() - && self.types.is_specialized_generic_instance(&subscript.value) - } - _ => false, - } - } - - /// [`Self::gated`] restricted to the enabled positions: a `Call` needs - /// `generic_calls`, a `Subscript` needs `projections` - fn gated_enabled(&self, expr: &Expr) -> bool { - match expr { - Expr::Call(_) => self.positions.generic_calls && self.gated(expr), - Expr::Subscript(_) => self.positions.projections && self.gated(expr), - _ => false, - } - } - - /// whether `value` is "unhelpful" — its own inferred type doesn't back a - /// standalone check, so a surrounding annotation (assignment / return / - /// parameter) must supply the target: a plain `Any`, or a gated value - /// whose type is unresolved (a typevar solved only by that context) - fn value_needs_context_target(&self, value: &Expr) -> bool { - self.types.is_any(value) || (self.gated(value) && self.check_plan(value).is_none()) - } - - /// the check plan for `expr`'s inferred type, with the pure-`None` - /// isinstance noise case dropped (validating a `None` result guards nothing) - fn check_plan(&self, expr: &Expr) -> Option { - self.types - .soundness_check_plan(expr) - .filter(|plan| !matches!(plan, SoundnessCheck::Isinstance(CheckTarget::NoneType))) - } - - /// [`Self::check_plan`] for a parameter, read off the parameter rather than off - /// its annotation — which is the only way to see a type the source stated with a - /// default instead of with an annotation - fn parameter_plan(&self, parameter: &Parameter) -> Option { - self.types - .parameter_check_plan(parameter) - .filter(|plan| !matches!(plan, SoundnessCheck::Isinstance(CheckTarget::NoneType))) - } - - /// check the value `expr` produces against `plan` - fn check_value(&mut self, expr: &'ast Expr, plan: SoundnessCheck) { - self.sites.push(Site::Value { expr, plan }); - } - - /// check each element drawn from the iterable of a `for` / comprehension - /// clause when the iterable carries a generic specialization and the element - /// (loop target) type is checkable - fn check_iteration(&mut self, iterable: &'ast Expr, target_expr: &Expr, is_async: bool) { - if !self.types.is_specialized_generic_instance(iterable) { - return; - } - let Some(plan) = self.check_plan(target_expr) else { - return; - }; - self.sites.push(Site::Elements { - iterable, - is_async, - plan, - }); - } - - /// check each argument of `call` whose own type is unhelpful against its - /// matched parameter's annotation. positional mapping stops at the first - /// starred spread (positions past it are unknown); `**kwargs` spreads are - /// skipped - fn check_call_arguments(&mut self, call: &'ast ExprCall) { - let callee = call.func.as_ref(); - for (index, arg) in call.arguments.args.iter().enumerate() { - if arg.is_starred_expr() { - break; - } - self.maybe_check_argument(callee, arg, &ArgSlot::Positional(index)); - } - for keyword in &call.arguments.keywords { - if let Some(name) = &keyword.arg { - self.maybe_check_argument(callee, &keyword.value, &ArgSlot::Keyword(name.as_str())); - } - } - } - - fn maybe_check_argument(&mut self, callee: &Expr, arg: &'ast Expr, slot: &ArgSlot<'_>) { - if self.consumed.contains(&arg.range()) || !self.value_needs_context_target(arg) { - return; - } - let plan = match *slot { - ArgSlot::Positional(index) => self.types.call_positional_param_plan(callee, index), - ArgSlot::Keyword(name) => self.types.call_keyword_param_plan(callee, name), - }; - if let Some(plan) = plan { - self.check_value(arg, plan); - } - } - - /// the plan to validate a returned `value` against — the enclosing - /// function's declared return plan — but only when `value`'s own type - /// is unhelpful (else `generic_calls`/`projections` already covers it) - fn return_plan(&self, value: &Expr) -> Option { - let plan = self.return_targets.last()?.clone()?; - self.value_needs_context_target(value).then_some(plan) - } - - /// check each checkable parameter of `func` where its body begins — the - /// `parameters` position, defending the contract against callers the checker - /// never saw. variadic (`*args` / `**kwargs`) parameters are skipped, and so is - /// any parameter whose source states no type: an unannotated one with no - /// default states nothing, and `x=None` says the argument may be left out rather - /// than that `None` belongs there - /// - /// the plan is read off the *parameter*, not off its annotation, because a - /// default is a written type too. `def f(safe='/')` says `safe` is a `str` - /// — the native backend lays the parameter out at that bound and checks it - /// at the boundary, and asking the annotation node left the interpreted twin - /// silently more permissive than its own compiled form - fn check_parameters(&mut self, func: &'ast StmtFunctionDef) { - let params = &func.parameters; - let mut guards = Vec::new(); - for pwd in params - .posonlyargs - .iter() - .chain(¶ms.args) - .chain(¶ms.kwonlyargs) - { - let parameter = &pwd.parameter; - if let Some(plan) = self.parameter_plan(parameter) { - guards.push(( - crate::python_parameter_name(params, parameter, self.written), - plan, - )); - } - } - if !guards.is_empty() { - self.sites.push(Site::Parameters { - function: func, - guards, - }); - } - } -} - -impl<'ast> Visitor<'ast> for Soundness<'_, 'ast> { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - match stmt { - Stmt::FunctionDef(func) => { - if self.positions.parameters { - self.check_parameters(func); - } - // track the declared return plan so nested `return`s validate - // against the right function's annotation - let plan = func - .returns - .as_deref() - .and_then(|returns| self.check_plan(returns)); - self.return_targets.push(plan); - walk_stmt(self, stmt); - self.return_targets.pop(); - return; - } - Stmt::For(for_stmt) => { - if self.positions.iterations { - self.check_iteration(&for_stmt.iter, &for_stmt.target, for_stmt.is_async); - } - } - Stmt::AnnAssign(ann) => { - // the annotation node's stored type is the declared type; - // it backs two checks the value's own type can't: - // - an explicit `Any` flowing into the declared binding - // - a gated value whose typevar solution came from this very - // declaration (`a: str = t()` stores `Unknown` for `t()` — - // the context, not the expression, carries the claim) - if self.positions.assignments - && let Some(value) = &ann.value - && self.value_needs_context_target(value) - // a field-specifier RHS (`a: int = Field()`) is modelled as - // the field type but is a descriptor object at runtime, so a - // soundness check against the annotation would always fail - && !self.types.is_field_specifier(value) - && let Some(plan) = self.check_plan(&ann.annotation) - { - self.check_value(value, plan); - } - } - Stmt::Return(ret) => { - if self.positions.returns - && let Some(value) = &ret.value - && let Some(plan) = self.return_plan(value) - { - self.check_value(value, plan); - } - } - // a type-alias value is a type expression; nothing in it executes - Stmt::TypeAlias(_) => return, - _ => {} - } - walk_stmt(self, stmt); - } - - // annotations are type positions — never wrap inside them - fn visit_annotation(&mut self, _expr: &'ast Expr) {} - - fn visit_comprehension(&mut self, comprehension: &'ast Comprehension) { - if self.positions.iterations { - self.check_iteration( - &comprehension.iter, - &comprehension.target, - comprehension.is_async, - ); - } - ruff_python_ast::visitor::walk_comprehension(self, comprehension); - } - - fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::UnaryOp(unary) = expr - && unary.op == UnaryOp::Force - { - // the force lowering inserts `_force_unwrap(` at the operand's - // first byte; a check wrapped around the bare operand would land - // its second argument inside that call. wrap the whole `expr!` - // instead (the force edits materialize inside the passthrough) - // and consume the operand — and any inner `!` layers — so they - // aren't wrapped again - let already_consumed = self.consumed.contains(&expr.range()); - let mut operand = unary.operand.as_ref(); - while let Expr::UnaryOp(inner) = operand - && inner.op == UnaryOp::Force - { - self.consumed.push(operand.range()); - operand = &inner.operand; - } - if self.gated_enabled(operand) { - self.consumed.push(operand.range()); - if !already_consumed && let Some(plan) = self.check_plan(expr) { - self.check_value(expr, plan); - } - } - } else if self.gated_enabled(expr) - && !self.consumed.contains(&expr.range()) - && let Some(plan) = self.check_plan(expr) - { - self.check_value(expr, plan); - } - if self.positions.arguments - && let Expr::Call(call) = expr - { - self.check_call_arguments(call); - } - walk_expr(self, expr); - } +) -> Vec> { + types + .soundness_sites(suite, positions) + .into_iter() + .map(|site| match site { + SoundnessSite::Value { expr, check } => Site::Value { expr, check }, + SoundnessSite::Elements { + iterable, + is_async, + check, + } => Site::Elements { + iterable, + is_async, + check, + }, + SoundnessSite::Parameters { function, guards } => Site::Parameters { + function, + guards: guards + .into_iter() + .map(|(parameter, check)| { + ( + crate::python_parameter_name(&function.parameters, parameter, written), + check, + ) + }) + .collect(), + }, + }) + .collect() } /// a syntax node, as the checks a module makes are looked up by @@ -443,25 +187,25 @@ impl SiteKey { /// same values against the same types, under the same `--soundness` positions #[derive(Debug, Default)] pub struct SoundnessSites { - values: HashMap, - elements: HashMap, - parameters: HashMap>, + values: HashMap, + elements: HashMap, + parameters: HashMap>, } impl SoundnessSites { /// the check made on the value `expr` produces, as it is produced - pub fn value(&self, expr: &Expr) -> Option<&SoundnessCheck> { + pub fn value(&self, expr: &Expr) -> Option<&SiteCheck> { self.values.get(&SiteKey::of(expr)?) } /// the check made on each element drawn from `iterable`, as it is drawn - pub fn elements(&self, iterable: &Expr) -> Option<&SoundnessCheck> { + pub fn elements(&self, iterable: &Expr) -> Option<&SiteCheck> { self.elements.get(&SiteKey::of(iterable)?) } /// the checks made on `function`'s own parameters where its body begins, by the /// name python binds each parameter to, in the order the parameters are written - pub fn parameters(&self, function: &StmtFunctionDef) -> &[(String, SoundnessCheck)] { + pub fn parameters(&self, function: &StmtFunctionDef) -> &[(String, SiteCheck)] { SiteKey::of(function) .and_then(|key| self.parameters.get(&key)) .map_or(&[], Vec::as_slice) @@ -471,34 +215,31 @@ impl SoundnessSites { /// decide every soundness check `suite` makes under `positions`. `written` is the /// module `suite` is parsed from, as it was written pub fn soundness_sites( - model: &ty_python_semantic::SemanticModel<'_>, + model: &ty_python_semantic::LoweringModel<'_>, written: WrittenNames, suite: &[Stmt], positions: SoundnessPositions, ) -> SoundnessSites { let mut sites = SoundnessSites::default(); - if !positions.any() { - return sites; - } - let mut walker = Soundness::new(model, positions, written); - walker.visit_body(suite); - for site in walker.sites { + for site in self::sites(model, written, suite, positions) { match site { - Site::Value { expr, plan } => { + Site::Value { expr, check } => { if let Some(key) = SiteKey::of(expr) { - sites.values.insert(key, plan); + sites.values.insert(key, check); } } - Site::Elements { iterable, plan, .. } => { + Site::Elements { + iterable, check, .. + } => { if let Some(key) = SiteKey::of(iterable) { - sites.elements.insert(key, plan); + sites.elements.insert(key, check); } } Site::Parameters { function, guards } => { if let Some(key) = SiteKey::of(function) { let guards = guards .into_iter() - .map(|(name, plan)| (name.to_string(), plan)) + .map(|(name, check)| (name.to_string(), check)) .collect(); sites.parameters.insert(key, guards); } @@ -520,12 +261,14 @@ impl PrologueStatement for EntryGuard { } /// what the checks a pass decided on are written as -#[derive(Default)] #[expect( clippy::struct_excessive_bools, reason = "independent runtime-helper usage flags, not a state machine" )] -struct Rendered { +struct Rendered<'src> { + /// the module the checks are written into, whose `private` symbols a check + /// names as the module is emitted + written: WrittenNames<'src>, /// each value passed through a call: the span it is at, the text opening the call /// and the text closing it wraps: Vec<(TextRange, String, String)>, @@ -538,15 +281,38 @@ struct Rendered { used_aiter: bool, used_iter_p: bool, used_aiter_p: bool, + /// any check made once the names it reads are bound — pulls in + /// `_soundness_once_bound` + used_once_bound: bool, /// any deep parametric check emitted — pulls in `_soundness_parametric` /// and the `_parametric_is` probe it reuses used_parametric: bool, } -impl Rendered { - /// wrap `expr` in `helper(, )`. `trailing` - /// carries its own leading `, ` (e.g. `", str"` or `", A[int], (0,)"`) - fn wrap_call(&mut self, expr: &Expr, helper: &str, trailing: &str) { +impl<'src> Rendered<'src> { + fn new(written: WrittenNames<'src>) -> Self { + Self { + written, + wraps: Vec::new(), + guards: Vec::new(), + unanchored: Vec::new(), + used_iter: false, + used_aiter: false, + used_iter_p: false, + used_aiter_p: false, + used_once_bound: false, + used_parametric: false, + } + } + + /// `isinstance`'s second argument for `target` + fn target(&self, target: &CheckTarget) -> String { + self.written.spelled(&target.spelling()) + } + + /// wrap `expr` in `helper()`. `leading` carries its own + /// trailing `, `, and `trailing` its own leading `, ` (e.g. `", str"`) + fn wrap_call(&mut self, expr: &Expr, helper: &str, leading: &str, trailing: &str) { let (open, close) = if needs_parentheses_as_argument(expr) { ("(", ")") } else { @@ -554,78 +320,90 @@ impl Rendered { }; self.wraps.push(( expr.range(), - format!("{helper}({open}"), + format!("{helper}({leading}{open}"), format!("{close}{trailing})"), )); } - /// wrap `expr` in the scalar check (`_soundness_check` / - /// `_soundness_parametric`) named by `plan` - fn wrap_check(&mut self, expr: &Expr, plan: &SoundnessCheck) { - match plan { - SoundnessCheck::Isinstance(target) => { - self.wrap_call(expr, "_soundness_check", &format!(", {target}")); - } + /// the helper a check against `check`'s target is made with, and the text its value + /// argument is preceded and followed by. `helpers` names the plain helper and the + /// parametric one, which also takes the variances + fn checking( + &mut self, + check: &SiteCheck, + helpers: [&'static str; 2], + ) -> (String, String, String) { + let (helper, target, rest) = match &check.plan { + SoundnessCheck::Isinstance(target) => (helpers[0], self.target(target), String::new()), SoundnessCheck::Parametric { alias, variances } => { self.used_parametric = true; - self.wrap_call( - expr, - "_soundness_parametric", - &format!(", {alias}, {}", variance_tuple(variances)), - ); + ( + helpers[1], + self.written.spelled(alias), + format!(", {}", variance_tuple(variances)), + ) } + }; + if check.once_bound.is_empty() { + return ( + helper.to_owned(), + String::new(), + format!(", {target}{rest}"), + ); } + // the target is read through a lambda, once every name it reads is bound + self.used_once_bound = true; + let names: Vec = check + .once_bound + .iter() + .map(|read| format!("\"{}\"", self.written.spelled(read))) + .collect(); + ( + ONCE_BOUND.to_owned(), + format!("({},), ", names.join(", ")), + format!(", {helper}, lambda: {target}{rest}"), + ) + } + + /// wrap `expr` in the scalar check (`_soundness_check` / + /// `_soundness_parametric`) named by `check` + fn wrap_check(&mut self, expr: &Expr, check: &SiteCheck) { + let (helper, leading, trailing) = + self.checking(check, ["_soundness_check", "_soundness_parametric"]); + self.wrap_call(expr, &helper, &leading, &trailing); } /// wrap an iterable in a validating generator whose form (shallow vs /// parametric) follows the element's check plan - fn wrap_iteration(&mut self, iterable: &Expr, is_async: bool, plan: &SoundnessCheck) { - let (helper, trailing) = match plan { - SoundnessCheck::Isinstance(target) => { - let helper = if is_async { - self.used_aiter = true; - "_soundness_aiter" - } else { - self.used_iter = true; - "_soundness_iter" - }; - (helper, format!(", {target}")) - } - SoundnessCheck::Parametric { alias, variances } => { - self.used_parametric = true; - let helper = if is_async { - self.used_aiter_p = true; - "_soundness_aiter_p" - } else { - self.used_iter_p = true; - "_soundness_iter_p" - }; - (helper, format!(", {alias}, {}", variance_tuple(variances))) - } + fn wrap_iteration(&mut self, iterable: &Expr, is_async: bool, check: &SiteCheck) { + let helpers = if is_async { + ["_soundness_aiter", "_soundness_aiter_p"] + } else { + ["_soundness_iter", "_soundness_iter_p"] }; - self.wrap_call(iterable, helper, &trailing); + match (&check.plan, is_async) { + (SoundnessCheck::Isinstance(_), false) => self.used_iter = true, + (SoundnessCheck::Isinstance(_), true) => self.used_aiter = true, + (SoundnessCheck::Parametric { .. }, false) => self.used_iter_p = true, + (SoundnessCheck::Parametric { .. }, true) => self.used_aiter_p = true, + } + let (helper, leading, trailing) = self.checking(check, helpers); + self.wrap_call(iterable, &helper, &leading, &trailing); } /// the guard statements that validate each of `checks`, in order - fn guard_stmts(&mut self, checks: &[(Name, SoundnessCheck)]) -> Vec { + fn guard_stmts(&mut self, checks: &[(Name, SiteCheck)]) -> Vec { checks .iter() - .map(|(name, plan)| self.guard_stmt(name, plan)) + .map(|(name, check)| self.guard_stmt(name, check)) .collect() } - /// the guard statement that validates parameter `name` against `plan` - fn guard_stmt(&mut self, name: &str, plan: &SoundnessCheck) -> String { - match plan { - SoundnessCheck::Isinstance(target) => format!("_soundness_check({name}, {target})"), - SoundnessCheck::Parametric { alias, variances } => { - self.used_parametric = true; - format!( - "_soundness_parametric({name}, {alias}, {})", - variance_tuple(variances) - ) - } - } + /// the guard statement that validates parameter `name` against `check` + fn guard_stmt(&mut self, name: &str, check: &SiteCheck) -> String { + let (helper, leading, trailing) = + self.checking(check, ["_soundness_check", "_soundness_parametric"]); + format!("{helper}({leading}{name}{trailing})") } /// insert the entry guards at the top of `func`'s body, after any docstring @@ -633,7 +411,7 @@ impl Rendered { &mut self, source: &str, func: &StmtFunctionDef, - checks: &[(Name, SoundnessCheck)], + checks: &[(Name, SiteCheck)], ) { let guards: Vec = self .guard_stmts(checks) @@ -679,24 +457,28 @@ impl TypeAwarePass for SoundnessPass<'_> { if !self.positions.any() { return; } - let mut inner = Rendered::default(); + let mut inner = Rendered::new(self.written); let mut rerendered = Vec::new(); for (idx, stmt) in stmts.iter().enumerate() { - let mut walker = Soundness::new(types, self.positions, self.written); - walker.visit_stmt(stmt); + let sites = sites( + types, + self.written, + std::slice::from_ref(stmt), + self.positions, + ); // an AST-mutation pass rewrote this statement, and an edit at a node it // replaced would be dropped: its checks go into the tree it is printed from if ctx.changed.contains(&idx) { - let mut rendered = Rendered::default(); + let mut rendered = Rendered::new(self.written); let mut checks = Vec::new(); - for site in &walker.sites { + for site in &sites { match site { - Site::Value { expr, plan } => rendered.wrap_check(expr, plan), + Site::Value { expr, check } => rendered.wrap_check(expr, check), Site::Elements { iterable, is_async, - plan, - } => rendered.wrap_iteration(iterable, *is_async, plan), + check, + } => rendered.wrap_iteration(iterable, *is_async, check), Site::Parameters { function, guards } => { checks.push(RerenderedCheck::Guards { function: function.range(), @@ -716,20 +498,21 @@ impl TypeAwarePass for SoundnessPass<'_> { inner.used_aiter |= rendered.used_aiter; inner.used_iter_p |= rendered.used_iter_p; inner.used_aiter_p |= rendered.used_aiter_p; + inner.used_once_bound |= rendered.used_once_bound; inner.used_parametric |= rendered.used_parametric; if !checks.is_empty() { rerendered.push((idx, checks)); } continue; } - for site in &walker.sites { + for site in &sites { match site { - Site::Value { expr, plan } => inner.wrap_check(expr, plan), + Site::Value { expr, check } => inner.wrap_check(expr, check), Site::Elements { iterable, is_async, - plan, - } => inner.wrap_iteration(iterable, *is_async, plan), + check, + } => inner.wrap_iteration(iterable, *is_async, check), Site::Parameters { function, guards } => { inner.insert_param_guards(self.source, function, guards); } @@ -764,6 +547,9 @@ impl TypeAwarePass for SoundnessPass<'_> { if inner.used_aiter_p { ctx.runtime.insert(crate::runtime::SOUNDNESS_AITER_P); } + if inner.used_once_bound { + ctx.runtime.insert(crate::runtime::SOUNDNESS_ONCE_BOUND); + } ctx.template_edits .extend(inner.wraps.into_iter().map(|(range, before, after)| { ( @@ -935,6 +721,8 @@ impl Transformer for Placing { #[cfg(test)] mod tests { + use indoc::indoc; + use crate::{Config, transpile}; fn soundness_config() -> Config { @@ -1151,6 +939,130 @@ mod tests { ); } + /// a function may be called before or after the module reaches a class declared below + /// it, so a check in one looks the class up as it runs. a check at the module's own level + /// runs in order, where the class is not there yet, and is left out + #[test] + fn a_class_declared_below_is_looked_up_where_the_check_runs() { + let out = check_with( + indoc! {" + def t[T]() -> T: ... + + class Holder: + later: list[Later] + + def first(self) -> Later: + return self.later[0] + + def each(self) -> None: + for later in self.later: + pass + + def take(value: Later) -> None: ... + + early: Later = t() + + class Later: ... + "}, + crate::SoundnessPositions::all(), + ); + assert!( + out.contains( + r#"return _soundness_once_bound(("Later",), self.later[0], _soundness_check, lambda: Later)"# + ), + "got:\n{out}" + ); + assert!( + out.contains( + r#"for later in _soundness_once_bound(("Later",), self.later, _soundness_iter, lambda: Later):"# + ), + "got:\n{out}" + ); + assert!( + out.contains( + r#"_soundness_once_bound(("Later",), value, _soundness_check, lambda: Later)"# + ), + "got:\n{out}" + ); + assert!(out.contains(r#"early: "Later" = t()"#), "got:\n{out}"); + assert!(out.contains("def _soundness_once_bound"), "got:\n{out}"); + } + + /// the lookup finds whatever the name holds, so it is made only where every binding of + /// the name binds the one class: not where the module binds it to something else first, + /// binds a second class under it, binds it only for the checker, unbinds it, or rebinds + /// it from a function through `global` + #[test] + fn a_name_bound_to_anything_but_the_class_is_not_looked_up() { + let out = check(indoc! {" + from typing import TYPE_CHECKING + + class Other: ... + + def t[T]() -> T: ... + + def first() -> Later: + return t() + + def second() -> Twice: + return t() + + def third() -> Checked: + return t() + + def fourth() -> Gone: + return t() + + def fifth() -> Rebound: + return t() + + def rebind() -> None: + global Rebound + Rebound = None + + Later = None + + class Later: ... + + Twice = Other + + class Twice: ... + + if TYPE_CHECKING: + from decimal import Decimal as Checked + else: + class Checked: ... + + class Gone: ... + + del Gone + + class Rebound: ... + "}); + assert!(!out.contains("_soundness_once_bound("), "got:\n{out}"); + assert!(!out.contains("_soundness_check(t()"), "got:\n{out}"); + } + + /// a `private` class is emitted under another name, and the lookup is keyed by the name + /// the module emits it under + #[test] + fn a_private_class_is_looked_up_by_the_name_it_is_emitted_under() { + let out = check(indoc! {" + def t[T]() -> T: ... + + def make() -> Kept: + return t() + + private class Kept: ... + "}); + assert!( + out.contains( + r#"_soundness_once_bound(("_Kept",), t(), _soundness_check, lambda: _Kept)"# + ), + "got:\n{out}" + ); + } + #[test] fn shadowed_builtin_skips_check() { let out = check("str = 1\ndef t[T]() -> T: ...\ndef f():\n a: str = t()\n"); diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index 95baf7453a..0c26b544a7 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -12,6 +12,37 @@ use ruff_python_ast::{ use ruff_python_trivia::SimpleTokenizer; use ruff_text_size::{Ranged, TextRange, TextSize}; +/// the expression at `range` as fragments that stand on their own where a line break ends +/// the statement: the value of an assignment a lowering writes, the test of an `if` it +/// writes, the subject of a `match` it writes +/// +/// an expression's range stops inside any parentheses the source grouped it with, and in +/// them it may run over several lines (`(\n a\n .b()\n)`). re-emitted bare, its +/// second line ends the statement early or starts one of its own: `x = a` above `.b()` is +/// refused, and `x = [1]` above `+ [2]` runs `+[2]` as a statement of its own. so an +/// expression written over more than one line gets parentheses of its own, which python +/// accepts around any expression +pub(crate) fn standalone_expression(source: &str, range: TextRange) -> Vec { + if source[range].contains(['\n', '\r']) { + vec![ + Fragment::Lit("(".to_owned()), + Fragment::Src(range), + Fragment::Lit(")".to_owned()), + ] + } else { + vec![Fragment::Src(range)] + } +} + +/// [`standalone_expression`] as text, for a pass that writes its edits as text +pub(crate) fn standalone_expression_text(source: &str, range: TextRange) -> String { + if source[range].contains(['\n', '\r']) { + format!("({})", &source[range]) + } else { + source[range].to_owned() + } +} + /// Whether `expr`, written as it is, has to be parenthesized to stand as one argument /// of a call among others: a tuple or generator expression written without its own /// parentheses (`for x in a, b:`), or a `yield`, which a call does not accept bare diff --git a/crates/by_transforms/src/transforms/statement_expression.rs b/crates/by_transforms/src/transforms/statement_expression.rs index b8a8a7348e..aa4a9eaf0c 100644 --- a/crates/by_transforms/src/transforms/statement_expression.rs +++ b/crates/by_transforms/src/transforms/statement_expression.rs @@ -56,7 +56,8 @@ use std::collections::HashSet; use super::ast_driver::{AstPass, Fragment, PassContext}; use super::repeated_underscore::WrittenNames; use super::source_util::{ - line_indent, line_start, parenthesized_value_range, temporary_name, value_separator_start, + line_indent, line_start, parenthesized_value_range, standalone_expression, temporary_name, + value_separator_start, }; pub(crate) struct StatementExpressionPass<'src> { @@ -175,7 +176,7 @@ impl Lower<'_> { if let Expr::Statement(statement) = tail { self.emit_root(statement, stmt, tail, &temp, &indent, &mut fragments); } else { - Self::emit_value(tail, &temp, &indent, &mut fragments); + Self::emit_value(self.source, tail, &temp, &indent, &mut fragments); // the replaced range starts after the line's existing indentation, // so the first emitted line must not repeat it if let Some(Fragment::Lit(first)) = fragments.first_mut() @@ -293,7 +294,7 @@ impl Lower<'_> { } /// Emits the block that computes `expr` into `temp` at `indent`. - fn emit_value(expr: &Expr, temp: &str, indent: &str, out: &mut Vec) { + fn emit_value(source: &str, expr: &Expr, temp: &str, indent: &str, out: &mut Vec) { match expr { // only the diverging forms reach here: a form with a suite is always // the whole value expression and took [`Self::emit_root`] @@ -314,26 +315,26 @@ impl Lower<'_> { out.push(Fragment::Lit(format!("\n{inner}{guard}{temp}:\n"))); inner.push_str(" "); } - Self::emit_value(value, temp, &inner, out); + Self::emit_value(source, value, temp, &inner, out); } } Expr::If(if_expr) => { let inner = format!("{indent} "); out.push(Fragment::Lit(format!("{indent}if "))); - out.push(Fragment::Src(if_expr.test.range())); + out.extend(standalone_expression(source, if_expr.test.range())); out.push(Fragment::Lit(":\n".to_string())); - Self::emit_value(&if_expr.body, temp, &inner, out); + Self::emit_value(source, &if_expr.body, temp, &inner, out); out.push(Fragment::Lit(format!("\n{indent}else:\n"))); - Self::emit_value(&if_expr.orelse, temp, &inner, out); + Self::emit_value(source, &if_expr.orelse, temp, &inner, out); } Expr::BinOp(bin_op) if matches!(bin_op.op, Operator::Coalesce) => { let inner = format!("{indent} "); - Self::emit_value(&bin_op.left, temp, indent, out); + Self::emit_value(source, &bin_op.left, temp, indent, out); out.push(Fragment::Lit(format!("\n{indent}if {temp} is None:\n"))); - Self::emit_value(&bin_op.right, temp, &inner, out); + Self::emit_value(source, &bin_op.right, temp, &inner, out); } Expr::Named(named) => { - Self::emit_value(&named.value, temp, indent, out); + Self::emit_value(source, &named.value, temp, indent, out); out.push(Fragment::Lit(format!("\n{indent}"))); out.push(Fragment::Src(named.target.range())); out.push(Fragment::Lit(format!(" = {temp}"))); @@ -341,7 +342,7 @@ impl Lower<'_> { // an ordinary operand: evaluate it into the temporary _ => { out.push(Fragment::Lit(format!("{indent}{temp} = "))); - out.push(Fragment::Src(expr.range())); + out.extend(standalone_expression(source, expr.range())); } } } diff --git a/crates/by_transforms/src/transforms/template_type.rs b/crates/by_transforms/src/transforms/template_type.rs index 07038f4ae3..a6780c97a5 100644 --- a/crates/by_transforms/src/transforms/template_type.rs +++ b/crates/by_transforms/src/transforms/template_type.rs @@ -45,6 +45,10 @@ pub(crate) struct TemplateTypePass<'src> { } impl TypeAwarePass for TemplateTypePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::TemplateType) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = TemplateType { types, diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index fe904f472d..9cded327ba 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -22,8 +22,8 @@ //! 1. `AnnAssign` annotation //! 2. function parameter annotations (regular, vararg, kwarg, lambda params) //! 3. function return annotation -//! 4. `type X = T` RHS / `X: TypeAlias = T` (the `TypeAlias` form is reached -//! via `AnnAssign`, no special case needed) +//! 4. `type X = T` RHS / `X: TypeAlias = T` (the value of the `TypeAlias` form, +//! as ty reads the annotation) //! 5. `TypeParam` bound (PEP 695 + the basedpython type mapping `T in (…)`) //! 6. `TypeParam` default //! 7. value-position type applications (`list[T]`, `dict[K, V]` outside an @@ -145,8 +145,17 @@ pub(crate) fn walk_type_positions_skipping( /// callers that already know the expression is in a type position (e.g. /// `generics.rs` re-rewriting the body of a polyfilled type alias) pub(crate) fn walk_one_type_expr(expr: &Expr, visitor: &mut dyn TypeExprVisitor) { + walk_one_type_expr_with(expr, None, visitor); +} + +/// [`walk_one_type_expr`], with `types` to tell which value is `Callable` +pub(crate) fn walk_one_type_expr_with( + expr: &Expr, + types: Option<&dyn TypeInfo>, + visitor: &mut dyn TypeExprVisitor, +) { let mut walker = TypePosWalker { - types: None, + types, claimed: &[], visitor, }; @@ -220,7 +229,7 @@ impl TypePosWalker<'_> { } return; } - if name_matches(&s.value, "Callable") { + if self.is_callable(&s.value) { self.descend_callable(&s.slice); return; } @@ -263,18 +272,38 @@ impl TypePosWalker<'_> { self.visit_type_expr(slice, TypePos::Nested); } + /// whether `value` is `Callable`: the checker's answer where the walk has one, and the + /// spelling where it has none + fn is_callable(&self, value: &Expr) -> bool { + match self.types { + Some(types) => types.is_callable_form(value), + None => name_matches(value, "Callable"), + } + } + fn descend_callable(&mut self, slice: &Expr) { // `Callable[[P1, P2], R]` — slice is an unparenthesized 2-tuple // `(List[P1, P2], R)`. fall back to generic descent for any other // shape (e.g. `Callable[..., R]`, `Callable[P, R]` with ParamSpec) if let Expr::Tuple(t) = slice { if !t.parenthesized && t.elts.len() == 2 { - if let Expr::List(pl) = &t.elts[0] { - for p in &pl.elts { - self.visit_type_expr(p, TypePos::CallableParam); + match &t.elts[0] { + Expr::List(pl) => { + for p in &pl.elts { + self.visit_type_expr(p, TypePos::CallableParam); + } } - } else { - self.visit_type_expr(&t.elts[0], TypePos::Nested); + // basedpython: `Callable[(int, name: str), R]` writes the parameter list + // as a tuple, which lowers to python's list. each parameter's type is a + // type position of its own; a variadic one has no place in the list + Expr::Tuple(pt) if pt.parenthesized => { + for p in &pt.elts { + if let Some(ty) = callable_parameter_type(p) { + self.visit_type_expr(ty, TypePos::CallableParam); + } + } + } + params => self.visit_type_expr(params, TypePos::Nested), } self.visit_type_expr(&t.elts[1], TypePos::Nested); return; @@ -357,7 +386,15 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { declaration_annotation_type(&a.annotation).unwrap_or(&a.annotation); self.visit_root(annotation, RootKind::Annotation); if let Some(v) = &a.value { - self.visit_expr(v); + // the value of `X: TypeAlias = …` is a type expression of its own + if self + .types + .is_some_and(|types| types.is_type_alias_annotation(annotation)) + { + self.visit_root(v, RootKind::Evaluated); + } else { + self.visit_expr(v); + } } } Stmt::TypeAlias(a) => { @@ -475,6 +512,17 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { } } +/// the type of one parameter of a `Callable` parameter list written as a tuple, or `None` for +/// a variadic one, which python's list form has no place for: `int` for `int` and `name: int` +pub(crate) fn callable_parameter_type(parameter: &Expr) -> Option<&Expr> { + match parameter { + Expr::Named(named) if matches!(named.target.as_ref(), Expr::Starred(_)) => None, + Expr::Named(named) => Some(&named.value), + Expr::Starred(_) => None, + parameter => Some(parameter), + } +} + fn name_matches(expr: &Expr, ident: &str) -> bool { match expr { Expr::Name(n) => n.id.as_str() == ident, diff --git a/crates/by_transforms/src/transforms/type_reification.rs b/crates/by_transforms/src/transforms/type_reification.rs index 9e123ab8eb..badd81be1d 100644 --- a/crates/by_transforms/src/transforms/type_reification.rs +++ b/crates/by_transforms/src/transforms/type_reification.rs @@ -78,6 +78,7 @@ use ruff_python_ast::{self as ast, Expr, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use crate::type_info::{TypeInfo, trailing_name}; /// dunder assignments whose values static readers (linters, type checkers, @@ -105,6 +106,7 @@ fn is_version_info(expr: &Expr) -> bool { struct Reifier<'ti> { types: &'ti dyn TypeInfo, + written: WrittenNames<'ti>, edits: Vec<(TextRange, String)>, } @@ -146,7 +148,7 @@ impl<'ast> Visitor<'ast> for Reifier<'_> { { self.edits.push(( TextRange::empty(call.func.range().end()), - format!("[{arguments}]"), + format!("[{}]", self.written.spelled(&arguments)), )); } walk_expr(self, expr); @@ -174,17 +176,21 @@ impl<'ast> Visitor<'ast> for Reifier<'_> { } } -pub(crate) struct TypeReificationPass { +pub(crate) struct TypeReificationPass<'src> { + written: WrittenNames<'src>, min_version: PythonVersion, } -impl TypeReificationPass { - pub(crate) fn new(min_version: PythonVersion) -> Self { - Self { min_version } +impl<'src> TypeReificationPass<'src> { + pub(crate) fn new(written: WrittenNames<'src>, min_version: PythonVersion) -> Self { + Self { + written, + min_version, + } } } -impl TypeAwarePass for TypeReificationPass { +impl TypeAwarePass for TypeReificationPass<'_> { // the specialization is spelled out so the value a call constructs // carries it as the program runs fn runtime_only(&self) -> bool { @@ -197,6 +203,7 @@ impl TypeAwarePass for TypeReificationPass { } let mut reifier = Reifier { types, + written: self.written, edits: Vec::new(), }; for stmt in stmts { @@ -311,6 +318,46 @@ mod tests { ); } + /// `float` in basedpython is just `float`, so it is what a construction is specialized + /// with, not python's `int | float`. a covariant argument goes through the promotion + /// that would apply python's special case + #[test] + fn a_float_argument_specializes_as_just_float() { + let out = out_at( + indoc! {" + class A[out T]: + def __init__(self, t: T): ... + def f(y: float): + a = A(1.5) + b = A(y) + "}, + PythonVersion::PY312, + ); + assert!(out.contains("a = A[float](1.5)"), "got: {out}"); + assert!(out.contains("b = A[float](y)"), "got: {out}"); + } + + /// backlog N15 #1, constructor calls still widen `float` in `.by`: pinned so a fix shows up + /// here. the checker solves an invariant parameter from a `float` variable through python's + /// numeric special case, so the construction is typed `A[int | float]` and specialized that + /// way. the literal is right + #[test] + fn an_invariant_float_argument_still_specializes_as_int_or_float() { + let out = out_at( + indoc! {" + class A[T]: + def __init__(self, t: T): + self.t = t + def f(y: float): + a = A(1.5) + b = A(y) + "}, + PythonVersion::PY312, + ); + assert!(out.contains("a = A[float](1.5)"), "got: {out}"); + assert!(out.contains("b = A[int | float](y)"), "got: {out}"); + } + #[test] fn generic_builtin_the_runtime_never_subscripts_stays_bare() { // a `zip` object is generic to the type checker, but no release of @@ -477,7 +524,7 @@ mod tests { "}); assert!( out.contains("def f(x=_MISSING):"), - "default must stay a bare sentinel: {out}" + "default must stay a bare marker: {out}" ); assert!( !out.contains("](_MISSING"), @@ -606,6 +653,238 @@ mod tests { ); } + /// a class imported only under `if TYPE_CHECKING:` is a name the running program does + /// not have, so a specialization naming it is left out, as an annotated assignment and as + /// a write to a declared attribute + #[test] + fn a_class_imported_only_for_the_checker_is_left_out() { + let out = out(indoc! {" + from typing import TYPE_CHECKING + if TYPE_CHECKING: + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + + def __init__(self) -> None: + self.entries = set() + kept: set[Decimal] = set() + + totals: set[Decimal] = set() + "}); + assert!(!out.contains("set[Decimal]()"), "got: {out}"); + assert_eq!(out.matches("= set()").count(), 3, "got: {out}"); + } + + /// a block on the constant under another name, or read off the module, is a type-checking + /// block all the same, while a block on a condition that holds whenever the program runs + /// is not + #[test] + fn a_class_imported_under_an_aliased_type_checking_block_is_left_out() { + let out = out(indoc! {" + import typing as t + from typing import TYPE_CHECKING as TC + + CHECKING = TC + ALWAYS = True + + if CHECKING: + from decimal import Decimal + if t.TYPE_CHECKING: + from fractions import Fraction + if ALWAYS: + from ipaddress import IPv4Address + + class Ledger: + entries: set[Decimal] + fractions: set[Fraction] + addresses: set[IPv4Address] + + def __init__(self) -> None: + self.entries = set() + self.fractions = set() + self.addresses = set() + "}); + assert!(out.contains("self.entries = set()"), "got: {out}"); + assert!(out.contains("self.fractions = set()"), "got: {out}"); + assert!( + out.contains("self.addresses = set[IPv4Address]()"), + "got: {out}" + ); + } + + /// a condition that reads the constant in any way — here a module's own `TYPE_CHECKING`, + /// which the checker reads as `True` as it does `typing`'s, inside an `or` — may be one the + /// program does not take + #[test] + fn a_class_past_a_condition_that_reads_type_checking_is_left_out() { + let out = out(indoc! {" + import os + + TYPE_CHECKING = False + + if TYPE_CHECKING or os.environ.get('NEVER'): + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + + def __init__(self) -> None: + self.entries = set() + "}); + assert!(out.contains("self.entries = set()"), "got: {out}"); + } + + /// what the running program binds above the statement a read is in is spelled, and what + /// it binds only further down is not: a method's body runs when it is called, which can be + /// before the module reaches a class declared below it (`Ledger()` above `class Later`) + #[test] + fn a_class_declared_above_is_spelled_in_a_method_and_one_below_is_not() { + let out = out(indoc! {" + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + later: set[Later] + + def __init__(self) -> None: + self.entries = set() + self.later = set() + kept: set[Decimal] = set() + + class Later: ... + "}); + assert!(out.contains("self.entries = set[Decimal]()"), "got: {out}"); + assert!(out.contains("self.later = set()"), "got: {out}"); + assert!( + out.contains("kept: set[Decimal] = set[Decimal]()"), + "got: {out}" + ); + } + + /// the module's own statements run in order: one above a class's declaration runs before + /// the class exists, one below it after + #[test] + fn a_class_is_spelled_below_its_declaration_and_not_above() { + let out = out(indoc! {" + early: set[Later] = set() + + class Later: ... + + late: set[Later] = set() + "}); + assert!(out.contains("early: \"set[Later]\" = set()"), "got: {out}"); + assert!( + out.contains("late: set[Later] = set[Later]()"), + "got: {out}" + ); + } + + /// the checker resolves a module-level name from every binding the module makes, so a + /// statement that runs while an earlier binding of the name still holds — the module binds + /// it again further down — may read another class than the one the checker resolved + #[test] + fn a_name_bound_again_further_down_is_left_out_above_it() { + let out = out(indoc! {" + from decimal import Decimal + + kept: set[Decimal] = set() + + from decimal import Decimal + "}); + assert!(out.contains("kept: set[Decimal] = set()"), "got: {out}"); + } + + /// a class the module's global scope does not bind, as its own class, has no spelling at + /// all: a type parameter, a class local to a function, a class nested in another + #[test] + fn a_class_the_module_does_not_bind_is_left_out() { + let out = out(indoc! {" + class Box[T]: + items: set[T] + + def __init__(self): + self.items = set() + + def make(): + class Local: ... + kept: set[Local] = set() + return kept + + class Outer: + class Inner: ... + inners: set[Inner] = set() + "}); + assert!(out.contains("self.items = set()"), "got: {out}"); + assert!(out.contains("kept: set[Local] = set()"), "got: {out}"); + assert!(out.contains("inners: set[Inner] = set()"), "got: {out}"); + } + + /// a `del` unbinds a name by the end of the module, and a binding made in one branch of a + /// conditional expression leaves it unbound on the other + #[test] + fn a_name_the_module_may_have_unbound_is_left_out() { + let out = out(indoc! {" + import os + from decimal import Decimal + from fractions import Fraction as _F + + entries: set[Decimal] = set() + del Decimal + + _ = (Fraction := _F) if os.environ.get(\"UNSET\") else None + fractions: set[Fraction] = set() + "}); + assert!(!out.contains("set[Decimal]()"), "got: {out}"); + assert!(!out.contains("set[Fraction]()"), "got: {out}"); + assert_eq!(out.matches("= set()").count(), 2, "got: {out}"); + } + + /// a function that rebinds a module-level name through `global` can run at any time, so + /// the name may hold whatever it bound + #[test] + fn a_name_a_function_rebinds_through_global_is_left_out() { + let out = out(indoc! {" + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + + def __init__(self) -> None: + self.entries = set() + + def forget() -> None: + global Decimal + del Decimal + "}); + assert!(out.contains("self.entries = set()"), "got: {out}"); + } + + /// a module the checker finds never ends, here one that stops with `sys.exit()`, has no + /// end where every binding is made; a name only statements at its top bind, above the + /// read, is bound wherever the read runs all the same + #[test] + fn a_class_bound_above_is_spelled_in_a_module_that_never_ends() { + let out = out(indoc! {" + import sys + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + + def __init__(self) -> None: + self.entries = set() + + totals: set[Decimal] = set() + sys.exit(0) + "}); + assert!(out.contains("self.entries = set[Decimal]()"), "got: {out}"); + assert!( + out.contains("totals: set[Decimal] = set[Decimal]()"), + "got: {out}" + ); + } + #[test] fn inert_below_39() { let out = out_at("xs = [1, 2]\n", PythonVersion::PY38); diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index 8e8dae60d4..6a009bad82 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -27,7 +27,7 @@ //! b: _TypedDict_ //! ``` //! -//! field types are emitted as forward references — see [`quote_type`] +//! field types are emitted as forward references — see `Shape::class_def` use std::collections::BTreeSet; use std::fmt::Write as _; @@ -43,6 +43,7 @@ use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_posit use super::ast_driver::{AstPass, PassContext}; use super::callable::CallableSyntax; +use super::quotes::WrittenQuotes; use super::repeated_underscore::WrittenNames; use crate::config::Config; use crate::type_info::TypeInfo; @@ -66,9 +67,15 @@ impl Shape { /// the class declaring this shape as `name`, `typing_extensions.TypedDict` written as /// `typed_dict` - fn class_def(&self, name: &str, typed_dict: &str) -> String { + /// + /// every field type is written as a string. the hoisted classes are emitted in the + /// preamble, ahead of everything the module defines, and a shape can name a class declared + /// later or a type parameter that only exists inside the generic class the literal was + /// written in. neither resolves where the class lands, and `TypedDict` evaluates its + /// class-body annotations eagerly, so every field is a forward reference + fn class_def(&self, name: &str, typed_dict: &str, quotes: &WrittenQuotes) -> String { let bases = match &self.extra_items { - Some(ty) => format!("{typed_dict}, extra_items={}", quote_type(ty)), + Some(ty) => format!("{typed_dict}, extra_items={}", quotes.quote(ty)), None => format!("{typed_dict}, closed=True"), }; let mut out = format!("class {name}({bases}):\n"); @@ -82,38 +89,17 @@ impl Shape { // attribute annotations. Such typed dicts are still valid // (TypedDict's functional form supports them), but we leave // them un-rewritten and let the user fall back to that form. - let _ = writeln!(out, " {field_name}: {}", quote_type(field_type)); + let _ = writeln!(out, " {field_name}: {}", quotes.quote(field_type)); } out } } -/// Wrap a rendered field type in a string annotation. -/// -/// The hoisted classes are emitted in the preamble, ahead of everything the -/// module defines, and a shape can name a class declared later or a type -/// parameter that only exists inside the generic class the literal was written -/// in. Neither resolves where the class lands, and `TypedDict` evaluates its -/// class-body annotations eagerly, so every field is a forward reference. -fn quote_type(type_source: &str) -> String { - let mut out = String::with_capacity(type_source.len() + 2); - out.push('"'); - for c in type_source.chars() { - match c { - '\\' => out.push_str("\\\\"), - '"' => out.push_str("\\\""), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - _ => out.push(c), - } - } - out.push('"'); - out -} - pub(crate) struct TypedDictLiteral<'src> { source: &'src str, written: WrittenNames<'src>, + /// where the field types are written as strings + quotes: &'src WrittenQuotes, edits: Vec, errors: Vec, /// Insertion-ordered map of shape → synthesized class name so emitted @@ -139,13 +125,15 @@ impl<'src> TypedDictLiteral<'src> { fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: &Config, types: &'src dyn TypeInfo, ) -> Self { Self { source, written, - callable: CallableSyntax::new(source, written, config).with_types(types), + quotes, + callable: CallableSyntax::new(source, written, quotes, config).with_types(types), edits: Vec::new(), errors: Vec::new(), shapes: IndexMap::new(), @@ -160,7 +148,7 @@ impl<'src> TypedDictLiteral<'src> { let mut out = String::new(); let typed_dict = self.written.imported("typing_extensions", "TypedDict"); for (shape, name) in &self.shapes { - out.push_str(&shape.class_def(name, &typed_dict)); + out.push_str(&shape.class_def(name, &typed_dict, self.quotes)); out.push('\n'); } out @@ -484,6 +472,10 @@ fn flatten_bitand(expr: &Expr) -> Vec<&Expr> { } impl AstPass for TypedDictLiteralPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::TypedDictLiteral) + } + /// a typed dict type it replaces moves into a class, whose field types the shared /// type-expression lowerer prints, a `typeof` among them fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { @@ -499,7 +491,13 @@ impl AstPass for TypedDictLiteralPass<'_> { } fn run(&self, _module: &mut ModModule, ctx: &mut PassContext) { - let mut inner = TypedDictLiteral::new(self.source, self.written, &self.config, self.types); + let mut inner = TypedDictLiteral::new( + self.source, + self.written, + self.quotes, + &self.config, + self.types, + ); // the module as parsed, not the one the passes before this rewrote: the type // information the field lowerer asks is about those nodes, and the edits are keyed // on the source ranges both share @@ -551,6 +549,7 @@ impl AstPass for TypedDictLiteralPass<'_> { pub(crate) struct TypedDictLiteralPass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, /// the module as parsed, which is what `types` answers for suite: &'src [Stmt], @@ -561,6 +560,7 @@ impl<'src> TypedDictLiteralPass<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, suite: &'src [Stmt], types: &'src dyn TypeInfo, @@ -568,6 +568,7 @@ impl<'src> TypedDictLiteralPass<'src> { Self { source, written, + quotes, config, suite, types, diff --git a/crates/by_transforms/src/transforms/unique_loop_bindings.rs b/crates/by_transforms/src/transforms/unique_loop_bindings.rs index 2249b480e3..c142ddd5e8 100644 --- a/crates/by_transforms/src/transforms/unique_loop_bindings.rs +++ b/crates/by_transforms/src/transforms/unique_loop_bindings.rs @@ -574,6 +574,16 @@ mod tests { ); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + /// the decorated form emits the rebind runtime; the tests below assert /// everything else. exactly the runtime's text is removed, so whatever else /// the preamble holds is still asserted @@ -646,7 +656,7 @@ mod tests { /// of a loop binding there is a closure read and has to be rebound with it #[test] fn a_default_reading_the_target_is_rebound() { - check_body( + check_beside_runtime( indoc! {" def build(): for i in items: @@ -655,7 +665,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default, _by_loop_bind + _MISSING: Any = _by_default(\"i\", __name__, \"_MISSING\") def build(): for i in items: @_by_loop_bind(i=i) @@ -818,7 +829,7 @@ mod tests { /// binding the rebind froze #[test] fn a_closure_in_a_parameter_default_is_left_to_the_body_guard() { - check_body( + check_beside_runtime( indoc! {" def build(): for i in items: @@ -827,7 +838,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default, _by_loop_bind + _MISSING: Any = _by_default(\"lambda: i\", __name__, \"_MISSING\") def build(): for i in items: @_by_loop_bind(i=i) diff --git a/crates/by_transforms/src/transforms/visibility_rename.rs b/crates/by_transforms/src/transforms/visibility_rename.rs index bcc7e7a27b..0c5bd82f5a 100644 --- a/crates/by_transforms/src/transforms/visibility_rename.rs +++ b/crates/by_transforms/src/transforms/visibility_rename.rs @@ -104,6 +104,31 @@ impl<'a> StatementVisitor<'a> for RenamedDeclarations { } } +/// the name the reference `name` is written under, when a visibility keyword renamed what +/// it reads: a module-level `private` symbol, given the module's `module_private` names, or a +/// bare name in a class body that is one of the class's own restricted members +/// +/// a reference is renamed only when it reads the renamed binding — a parameter, a local or a +/// class attribute that shares the name is a different binding, and renaming it would split +/// it from its uses — so ty is asked where each resolves +pub(crate) fn renamed_reference( + types: &dyn TypeInfo, + written: WrittenNames, + module_private: &HashSet, + name: &ast::ExprName, +) -> Option { + // a name the parser synthesized has no source to rename + if name.ctx == ExprContext::Invalid || name.range.is_empty() { + return None; + } + if module_private.contains(name.id.as_str()) + && types.resolves_to_module_scope(name) == Some(true) + { + return Some(written.module_private(name.id.as_str())); + } + types.class_body_member_name(name) +} + /// the kind of scope a statement is directly in #[derive(Clone, Copy, PartialEq, Eq)] enum LexicalScope { @@ -361,25 +386,12 @@ impl<'ast> Visitor<'ast> for Renamer<'_> { } fn visit_expr(&mut self, expr: &'ast Expr) { - // a reference to a module-level private symbol is renamed only when it is - // one — a parameter, a local or a class attribute that shares the name is a - // different binding, and renaming it would split it from its uses — and a - // bare name in a class body when it is one of the class's own restricted - // members. ty is asked where each resolves if let Expr::Name(name) = expr - && name.ctx != ExprContext::Invalid - // a name the parser synthesized has no source to rename - && !name.range.is_empty() && !self.declarations.contains(&name.range) + && let Some(renamed) = + renamed_reference(self.types, self.written, &self.module_private, name) { - if self.module_private.contains(name.id.as_str()) - && self.types.resolves_to_module_scope(name) == Some(true) - { - self.edits - .push((name.range, self.written.module_private(name.id.as_str()))); - } else if let Some(renamed) = self.types.class_body_member_name(name) { - self.edits.push((name.range, renamed)); - } + self.edits.push((name.range, renamed)); } if let Expr::Attribute(attribute) = expr && let Some(mangled) = self.types.restricted_member_name(attribute) diff --git a/crates/by_transforms/src/transforms/walrus_scope.rs b/crates/by_transforms/src/transforms/walrus_scope.rs new file mode 100644 index 0000000000..a4579af30e --- /dev/null +++ b/crates/by_transforms/src/transforms/walrus_scope.rs @@ -0,0 +1,290 @@ +//! where a lowering may bind a temporary with `:=` +//! +//! a lowering that reads a value twice but may evaluate it only once — `a ?? b`, +//! `f()?.x` — binds it to a temporary with an assignment expression. python refuses +//! one: +//! +//! - anywhere inside a comprehension's iterable, at any depth, a lambda's body +//! included: `[x for x in (t := xs)]` +//! - directly in an annotation, from 3.14: `x: (t := int) = 1` +//! - directly in the scope of a generic definition's type parameters: a bound or a +//! default, a generic class's base, a generic function's annotation, a `type` +//! alias's value: `class C[T](base(t := 1)): ...` +//! - in a comprehension whose nearest enclosing scope, past comprehensions and +//! annotations, is a class body or type parameters' scope: +//! `class C: ys = [(t := x) for x in xs]` +//! +//! a lambda's body is a function scope, where none of this applies short of a +//! comprehension iterable. where python refuses `:=` the lowering writes its temporary +//! as a comprehension variable instead, which python accepts in any expression + +use std::collections::HashSet; + +use ruff_python_ast::{Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange}; + +use crate::transforms::python_scopes::{self, Scope, ScopeHooks, ScopeState}; + +/// the expressions of a module where python accepts an assignment expression +/// +/// what is recorded is where one is accepted, so a position the walk never reached +/// — or regions nobody collected — answers that it is refused, and a lowering falls +/// back to the form python accepts everywhere +#[derive(Default)] +pub(crate) struct WalrusRegions { + allowed: HashSet, + /// a range two expressions share is refused if either refuses it + refused: HashSet, +} + +impl WalrusRegions { + pub(crate) fn collect(stmts: &[Stmt]) -> Self { + let mut regions = WalrusRegions::default(); + python_scopes::walk(stmts, &mut regions); + regions + } + + /// whether an assignment expression written in place of the expression at `range` + /// is accepted + pub(crate) fn allows(&self, range: TextRange) -> bool { + self.allowed.contains(&range) && !self.refused.contains(&range) + } +} + +impl ScopeHooks<'_> for WalrusRegions { + fn expr(&mut self, expr: &Expr, state: &ScopeState) { + if walrus_allowed(state) { + self.allowed.insert(expr.range()); + } else { + self.refused.insert(expr.range()); + } + } +} + +fn walrus_allowed(state: &ScopeState) -> bool { + if state.in_iterable() { + return false; + } + match state.scopes().last() { + Some(Scope::Annotation | Scope::TypeParams) => false, + Some(Scope::Comprehension { .. }) => !matches!( + state + .scopes() + .iter() + .rev() + .find(|scope| !matches!(scope, Scope::Comprehension { .. } | Scope::Annotation)), + Some(Scope::Class | Scope::TypeParams) + ), + Some(Scope::Class | Scope::Function { .. } | Scope::Lambda) | None => true, + } +} + +#[cfg(test)] +mod tests { + use ruff_python_ast::visitor::{Visitor, walk_expr}; + use ruff_python_ast::{Expr, PySourceType}; + use ruff_python_parser::parse_unchecked_source; + use ruff_text_size::{Ranged, TextRange}; + + use super::WalrusRegions; + + /// each source with a name `hole`, and whether python 3.12, 3.13 and 3.14 all + /// compile `(t := f())` written in its place — on every version that compiles the + /// source at all. measured by compiling each one, with the walrus and with the + /// comprehension form the lowering writes instead, which compiles in all of them + const POSITIONS: &[(&str, bool)] = &[ + ("x = hole", true), + ("x = [hole for a in b]", true), + ("x = [a for a in b if hole]", true), + ("x = [a for a in hole]", false), + ("x = [a for c in d for a in hole]", false), + ("x = [a for a in (lambda: hole)()]", false), + ("x = [a for a in [hole for c in d]]", false), + ("x = {hole: 1 for a in b}", true), + ("x = {a: hole for a in b}", true), + ("x = {hole for a in b}", true), + ("x = (hole for a in b)", true), + ("x = (a for a in hole)", false), + ("x = lambda: hole", true), + ("x = lambda y=hole: y", true), + ("x = [(lambda: hole) for a in b]", true), + ("x = [(lambda y=hole: y) for a in b]", true), + ("def f():\n return [hole for a in b]", true), + ("def f():\n return [a for a in hole]", false), + ("def f(y=hole):\n pass", true), + ("def f(y: hole):\n pass", false), + ("def f() -> hole:\n pass", false), + ("def f(*y: hole):\n pass", false), + ("@dec(hole)\ndef f():\n pass", true), + ("class C:\n x = hole", true), + ("class C:\n x = [hole for a in b]", false), + ("class C:\n x = [a for a in b if hole]", false), + ("class C:\n x = [a for a in hole]", false), + ("class C:\n x = [[hole for a in b] for c in d]", false), + ("class C:\n x = [(lambda: hole) for a in b]", true), + ("class C:\n x = [(lambda y=hole: y) for a in b]", false), + ("class C:\n x = lambda: hole", true), + ("class C:\n def f(self, y=hole):\n pass", true), + ( + "class C:\n @dec(hole)\n def f(self):\n pass", + true, + ), + ("class C:\n x: hole = 1", false), + ("class C:\n x: int = hole", true), + ("class C(hole):\n pass", true), + ("class C(metaclass=hole):\n pass", true), + ("@dec(hole)\nclass C:\n pass", true), + ("class C[T](hole):\n pass", false), + ("class C[T](metaclass=hole):\n pass", false), + ("class C[T](b[[hole for a in c]]):\n pass", false), + ("class C[T](b[lambda: hole]):\n pass", true), + ("class C[T: hole]:\n pass", false), + ("class C[T = hole]:\n pass", false), + ("@dec(hole)\nclass C[T]:\n pass", true), + ("class C[T]:\n x = hole", true), + ("def f[T](y=hole):\n pass", true), + ("def f[T](y: hole):\n pass", false), + ("def f[T]() -> hole:\n pass", false), + ("def f[T: hole]():\n pass", false), + ("@dec(hole)\ndef f[T]():\n pass", true), + ("def f[T]():\n return hole", true), + ("type A = hole", false), + ("type A = b[[hole for a in c]]", false), + ("type A = b[lambda: hole]", true), + ("type A[T] = hole", false), + ("type A[T: hole] = int", false), + ("x: hole = 1", false), + ("x: int = hole", true), + ("x: b[[hole for a in c]] = 1", true), + ("x: b[lambda: hole] = 1", true), + ("x: b[[a for a in hole]] = 1", false), + ("def f():\n x: hole = 1", false), + ("def f():\n x: b[[hole for a in c]] = 1", true), + ("def f(y: b[lambda: hole]):\n pass", true), + ("for a in hole:\n pass", true), + ("with hole as a:\n pass", true), + ("x = f\"{hole}\"", true), + ("match hole:\n case _:\n pass", true), + ("match a:\n case _ if hole:\n pass", true), + ("assert hole", true), + ("x = [a for a in b if (lambda: hole)()]", true), + ("x = [[a for a in hole] for c in d]", false), + ("async def f():\n return [a for a in await hole]", false), + ("def f[T](y: b[[hole for a in c]]):\n pass", false), + ("def f[T](y: b[lambda: hole]):\n pass", true), + ("def f(y: b[[hole for a in c]]):\n pass", true), + ("class C[T]:\n x: hole = 1", false), + ("class C[T]:\n x: b[[hole for a in c]] = 1", false), + ("class C:\n x: b[[hole for a in c]] = 1", false), + ("class C:\n x = {a: hole for a in b}", false), + ("class C:\n x = (hole for a in b)", false), + ("class C:\n x = [a for a in b if (lambda: hole)()]", true), + ( + "class C:\n def f(self):\n return [hole for a in b]", + true, + ), + ( + "class C:\n class D:\n x = [hole for a in b]", + false, + ), + ( + "def f():\n class D:\n x = [hole for a in b]", + false, + ), + ( + "class C[T](b[(lambda: [hole for a in c])()]):\n pass", + true, + ), + ( + "class C[T](b[[(lambda: hole) for a in c]]):\n pass", + true, + ), + ("type A = b[[(lambda: hole) for a in c]]", true), + ("x: b[[a for a in c if hole]] = 1", true), + ("x: (lambda y=hole: y) = 1", false), + ("type A = (lambda y=hole: y)", false), + ("class C[T](b[lambda y=hole: y]):\n pass", false), + ("x = [lambda: [hole for a in b] for c in d]", true), + ( + "class C:\n x = [lambda: [hole for a in b] for c in d]", + true, + ), + ("def f(y=[hole for a in b]):\n pass", true), + ( + "class C:\n def f(self, y=[hole for a in b]):\n pass", + false, + ), + ("@dec([hole for a in b])\ndef f():\n pass", true), + ( + "class C:\n @dec([hole for a in b])\n def f(self):\n pass", + false, + ), + ( + "class C:\n def f(self, y: b[[hole for a in c]]):\n pass", + false, + ), + ( + "class C:\n def f(self) -> b[[hole for a in c]]:\n pass", + false, + ), + ( + "class C:\n def f(self, y: b[lambda: hole]):\n pass", + true, + ), + ("def f():\n x: b[[hole for a in c]] = 1", true), + ("class C:\n type A = b[lambda: hole]", true), + ("class C[T]:\n def f(self, y=hole):\n pass", true), + ("class C[T]:\n x = [hole for a in b]", false), + ("x = lambda: [hole for a in b]", true), + ("class C:\n x = lambda: [hole for a in b]", true), + ]; + + fn hole(source: &str) -> TextRange { + struct Hole(Option); + impl<'a> Visitor<'a> for Hole { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.id.as_str() == "hole" + { + self.0 = Some(name.range()); + } + walk_expr(self, expr); + } + } + let parsed = parse_unchecked_source(source, PySourceType::Python); + assert!( + parsed.errors().is_empty(), + "{source}: {:?}", + parsed.errors() + ); + let mut hole = Hole(None); + hole.visit_body(&parsed.syntax().body); + hole.0.unwrap_or_else(|| panic!("no `hole` in {source}")) + } + + /// a position nobody recorded is refused: regions left empty, or a range the + /// walk never reached + #[test] + fn an_unknown_position_is_refused() { + let source = "x = hole\n"; + assert!(!WalrusRegions::default().allows(hole(source))); + let parsed = parse_unchecked_source(source, PySourceType::Python); + let regions = WalrusRegions::collect(&parsed.syntax().body); + assert!(regions.allows(hole(source))); + // `x =` is no expression + assert!(!regions.allows(TextRange::new(0.into(), 3.into()))); + } + + #[test] + fn agrees_with_python_on_every_position() { + let mismatches: Vec = POSITIONS + .iter() + .filter_map(|&(source, python)| { + let parsed = parse_unchecked_source(source, PySourceType::Python); + let ours = WalrusRegions::collect(&parsed.syntax().body).allows(hole(source)); + (ours != python).then(|| format!("python {python}, ours {ours}: {source:?}")) + }) + .collect(); + assert!(mismatches.is_empty(), "{}", mismatches.join("\n")); + } +} diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 2205486c40..658c9989f0 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -5,13 +5,14 @@ use crate::transforms::trailing_lambda::RECEIVER_PARAMETER; use ruff_db::parsed::parsed_module; use ruff_python_ast::helpers::is_dotted_name; use ruff_python_ast::{ - Expr, ExprCall, ExprName, ExprRef, Parameter, ParameterWithDefault, Stmt, StmtClassDef, + Expr, ExprCall, ExprName, ExprRef, ExprTuple, ParameterWithDefault, Stmt, StmtClassDef, StmtFunctionDef, }; use ruff_python_parser::parse_expression; use ruff_text_size::{Ranged, TextRange}; use ty_python_core::scope::ScopeKind; use ty_python_core::{global_scope, place_table, semantic_index, use_def_map}; +use ty_python_semantic::RuntimeSpelling; use ty_python_semantic::types::call_type_forms::CallTypeForm; use ty_python_semantic::types::exceptions::RaisesRuntimeTarget; use ty_python_semantic::types::ide_support::BaseTypeArgument; @@ -20,7 +21,8 @@ use ty_python_semantic::types::{ DisplaySettings, DynamicType, KnownClass, KnownInstanceType, Type, UnpackedKwargs, character, }; use ty_python_semantic::{ - Db, HasType, ImplicitReceiverReference, ProgramEnvironment, SemanticModel, + AnonNamedTupleClass, AnonNamedTupleId, Db, HasType, ImplicitReceiverReference, LoweringModel, + ProgramEnvironment, SemanticModel, }; /// How the postfix `^` / `!` operators test the "absent" arm of an operand's @@ -79,26 +81,43 @@ pub(crate) struct SynthesizedType { /// binds further down, or the class being defined, or one nested in it. evaluated there it /// raises, so it is written as a string pub(crate) forward: bool, + /// the module-level `private` classes the text names, each written bare and read + /// nowhere in the text but as that class + pub(crate) module_private: Vec, } impl SynthesizedType { /// the type as the module `written` is of writes it, each of [`Self::typing_names`] under - /// the name the lowering imports it by + /// the name the lowering imports it by, and each of [`Self::module_private`] under the + /// name it is emitted as pub(crate) fn text_in(&self, written: WrittenNames) -> String { struct Roots<'a> { typing_names: &'a [&'static str], + module_private: &'a [String], written: WrittenNames<'a>, edits: Vec<(TextRange, String)>, } + impl Roots<'_> { + /// the name the module writes `name`, a root the text reads, under + fn local(&self, name: &str) -> Option { + match name { + _ if self.typing_names.contains(&name) => { + Some(self.written.imported("typing", name)) + } + _ if self.module_private.iter().any(|class| class == name) => { + Some(self.written.module_private(name)) + } + _ => None, + } + } + } impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for Roots<'_> { fn visit_expr(&mut self, expr: &'ast Expr) { if let Expr::Name(name) = expr - && self.typing_names.contains(&name.id.as_str()) + && let Some(local) = self.local(name.id.as_str()) + && local != name.id.as_str() { - let local = self.written.imported("typing", name.id.as_str()); - if local != name.id.as_str() { - self.edits.push((name.range, local)); - } + self.edits.push((name.range, local)); } ruff_python_ast::visitor::walk_expr(self, expr); } @@ -108,6 +127,7 @@ impl SynthesizedType { }; let mut roots = Roots { typing_names: &self.typing_names, + module_private: &self.module_private, written, edits: Vec::new(), }; @@ -164,6 +184,29 @@ impl SynthesizedType { } } +/// the anonymous named tuple class the checker reads a plain tuple literal as, which the +/// emitted python constructs around it +#[derive(Clone, Debug)] +pub(crate) struct AnonNamedTupleConstruction { + /// the checker's identity of the class, shared with every type form it reads as the + /// same type + pub(crate) id: AnonNamedTupleId, + /// each field's name and type, for a module that has no type form of the class to + /// emit it from + pub(crate) fields: Vec<(String, AnonNamedTupleField)>, +} + +/// the type of one field of an [`AnonNamedTupleConstruction`] +#[derive(Clone, Debug)] +pub(crate) enum AnonNamedTupleField { + /// a type the emitted python can name + Spelled(SynthesizedType), + /// another anonymous named tuple, emitted as a class of its own + NamedTuple(AnonNamedTupleConstruction), + /// a type the emitted python cannot name + Unspellable, +} + /// How an assignment inside a trailing-lambda block reaches an enclosing scope, /// so the block writes through instead of shadowing with a fresh local. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -220,6 +263,17 @@ pub(crate) trait TypeInfo { /// unresolved name should be treated as a runtime subscript, not a type) fn subscript_is_known_type_context(&self, value: &Expr) -> bool; + /// whether subscripting `value` specializes it, so the checker infers its slice as type + /// expressions — see [`SemanticModel::subscript_specializes`] + fn subscript_specializes(&self, value: &Expr) -> bool; + + /// whether `annotation` declares a PEP 613 type alias, whose value is a type expression + fn is_type_alias_annotation(&self, annotation: &Expr) -> bool; + + /// whether `expr` is `typing.Callable` or `collections.abc.Callable`, however it is spelled + /// — see [`SemanticModel::is_callable_form`] + fn is_callable_form(&self, expr: &Expr) -> bool; + /// the arguments of `call` that are type expressions, because the call is one of the /// typing constructs that spells a type through a call — `NewType("D", int)`, /// `TypeVar("T", bound=int)`, the functional `NamedTuple` and `TypedDict` @@ -284,14 +338,20 @@ pub(crate) trait TypeInfo { /// call form when a keyword-variadic pack contributes fields), or `None` /// when the call needs no injection or none is possible (ty reports the /// latter) - fn reified_call_specialization(&self, call: &ruff_python_ast::ExprCall) -> Option; + fn reified_call_specialization( + &self, + call: &ruff_python_ast::ExprCall, + ) -> Option; /// the comma-joined type arguments to inject at a bare constructor call /// of a generic class (`A(1)` → `"int"`), or `None` when the callee is /// not a generic class literal or the solved specialization has no /// runtime spelling — reification of constructors is best-effort, a /// missing spelling is never an error - fn constructor_specialization(&self, call: &ruff_python_ast::ExprCall) -> Option; + fn constructor_specialization( + &self, + call: &ruff_python_ast::ExprCall, + ) -> Option; /// when the parameter annotation `annotation` denotes a union of /// specializations of one erased origin (`list[int] | list[str]`), how the @@ -413,12 +473,26 @@ pub(crate) trait TypeInfo { /// `global`. `None` when ty did not index the reference and cannot place it fn resolves_to_module_scope(&self, reference: &ExprName) -> Option; + /// whether the name `reference` reads is read in a class body's own scope and may + /// find a binding the class body made, which the same read moved into a scope + /// nested in the class body would not. `None` when ty did not index the reference + fn reads_class_scope_binding(&self, reference: &ExprName) -> Option; + /// whether `name`, read in an annotation, is a forward reference: a name the /// program binds, but not by the point python evaluates the annotation where /// it is written, as it does before 3.14 without `from __future__ import /// annotations`. `None` when ty did not index the name fn is_forward_reference(&self, name: &ExprName) -> Option; + /// whether the checker reads the type expression `expr` as deferred though python evaluates + /// it where it stands, so that a name in it may be a forward reference — see + /// [`SemanticModel::is_read_deferred`] + fn is_read_deferred(&self, expr: &Expr) -> bool; + + /// whether every name `expr` reads holds, where it is written, the binding the checker + /// resolved when the program runs, so `expr` can be written into code that runs there + fn reads_names_bound_where_written(&self, expr: &Expr) -> bool; + /// how a bare name in a class body is emitted, when it names a member the /// class declares with a visibility keyword — `y = x + 1` after `private x = /// 1`. `None` for any other name @@ -515,6 +589,19 @@ pub(crate) trait TypeInfo { /// rather than as a global fn shares_a_cell_scope(&self, reference: &ExprName, anchor: &Expr) -> bool; + /// the checker's identity of the anonymous named tuple class a type form + /// `(name: T, ...)` declares + fn anon_named_tuple_id(&self, tuple: &ExprTuple) -> Option; + + /// the anonymous named tuple a plain tuple literal is built as: the checker reads + /// one standing where an anonymous named tuple is expected as an instance of it. + /// `None` for a literal that stays a tuple + fn anon_named_tuple_construction( + &self, + tuple: &ExprTuple, + min_version: ruff_python_ast::PythonVersion, + ) -> Option; + /// rendered inferred (literal-promoted) type of `expr`, or `None` when ty /// cannot resolve a type (unresolved import, parse error, etc.) or the type /// it resolves to has no python type-expression spelling the emitted file @@ -608,44 +695,13 @@ pub(crate) trait TypeInfo { /// stands for the enclosing class and can never bind to an optional fn optional_wraps(&self, expr: &Expr) -> bool; - /// whether a call through `callee` yields a result whose type was - /// derived by substituting typevars — a generic function's return, or a - /// method bound to a specialized generic instance. such results rest on - /// an assumption ty cannot verify, so the soundness pass validates them - fn call_result_is_typevar_derived(&self, callee: &Expr) -> bool; - - /// whether `expr`'s inferred type is an instance carrying a generic - /// specialization (`list[str]`, a `TypedDict`) — element projections out - /// of it consume an annotation-level claim - fn is_specialized_generic_instance(&self, expr: &Expr) -> bool; - - /// the runtime soundness check for `expr`'s inferred type — a shallow - /// `isinstance` target (`str`, `(int, type(None))`) or a deep parametric - /// check for a user-generic specialization (`A[int]` + variances). `None` - /// when the type has no faithful runtime test or its name doesn't resolve - /// at module scope - fn soundness_check_plan(&self, expr: &Expr) -> Option; - - /// the runtime soundness check for a *parameter*, planned against the type its - /// source states rather than the one its annotation node spells - /// - /// the two differ exactly where there is no annotation to spell. `def f(safe='/')` - /// opens a hole bounded by everything the function requires of `safe`, and a hole - /// has no runtime test — so asking about the annotation answered nothing, and the - /// interpreted leg wrote no check where the native backend was already enforcing - /// the bound at its boundary - fn parameter_check_plan(&self, parameter: &Parameter) -> Option; - - /// the soundness check for the parameter that the positional argument at - /// `index` binds to in a call through `callee` (see - /// [`soundness_check_plan`](TypeInfo::soundness_check_plan)). `None` when - /// the callee isn't a single-overload function/method, the parameter is - /// variadic/unannotated, or its type has no runtime test - fn call_positional_param_plan(&self, callee: &Expr, index: usize) -> Option; - - /// like [`call_positional_param_plan`](TypeInfo::call_positional_param_plan) - /// but for a keyword argument matched by `name` - fn call_keyword_param_plan(&self, callee: &Expr, name: &str) -> Option; + /// every runtime soundness check `suite` makes under `positions` — the checker's one + /// answer, which the native backend reads as well + fn soundness_sites<'ast>( + &self, + suite: &'ast [ruff_python_ast::Stmt], + positions: crate::SoundnessPositions, + ) -> Vec>; /// the runtime check for a type expression used as a checked cast target, /// or `None` when the type has no faithful runtime test. a user generic @@ -796,14 +852,6 @@ pub(crate) trait TypeInfo { /// written fn decides_receiver_binding(&self, decorator: &ruff_python_ast::Decorator) -> bool; - /// whether `expr` is a field-specifier call — `pydantic.Field(...)`, - /// `dataclasses.field(...)`, `attrs.field(...)`, and the like. the checker - /// models these as assignable to the field's declared type, but their - /// runtime value is a field-descriptor object (`FieldInfo`, `Field`), so a - /// soundness check against the field annotation would always fail at - /// runtime — the transpiler must not emit one - fn is_field_specifier(&self, expr: &Expr) -> bool; - /// basedpython: the python a static resource import lowers to, bound to /// `binding` — the document at `path` written out as classes, tuples and /// `Final` literals — or why the document cannot be read. @@ -816,6 +864,9 @@ pub(crate) trait TypeInfo { /// re-export of the ty-side check plan so transforms name a single type pub(crate) use ty_python_semantic::types::soundness::CheckKind as SoundnessCheck; +/// re-export of the ty-side check a soundness site makes, with the names it waits on +pub(crate) use ty_python_semantic::types::soundness_sites::SiteCheck; + /// re-export of the ty-side checked-cast plan (a superset of [`SoundnessCheck`] /// that also carries protocol-structural and unchecked cases) pub(crate) use ty_python_semantic::types::soundness::CastCheck; @@ -823,7 +874,7 @@ pub(crate) use ty_python_semantic::types::soundness::CastCheck; /// re-export of the ty-side framework role so transforms name a single type pub(crate) use ty_python_semantic::types::FrameworkRole; -impl TypeInfo for SemanticModel<'_> { +impl TypeInfo for LoweringModel<'_> { fn is_type_fn_application(&self, expr: &Expr) -> bool { let Expr::Subscript(subscript) = expr else { return false; @@ -867,6 +918,18 @@ impl TypeInfo for SemanticModel<'_> { } } + fn subscript_specializes(&self, value: &Expr) -> bool { + SemanticModel::subscript_specializes(self, value) + } + + fn is_type_alias_annotation(&self, annotation: &Expr) -> bool { + SemanticModel::is_type_alias_annotation(self, annotation) + } + + fn is_callable_form(&self, expr: &Expr) -> bool { + SemanticModel::is_callable_form(self, expr) + } + fn call_type_expression_arguments<'ast>(&self, call: &'ast ExprCall) -> Vec<&'ast Expr> { call.func .inferred_type(self) @@ -932,11 +995,17 @@ impl TypeInfo for SemanticModel<'_> { .is_some_and(|function| function.is_reified(self.db())) } - fn reified_call_specialization(&self, call: &ruff_python_ast::ExprCall) -> Option { - SemanticModel::reified_call_specialization(self, call) + fn reified_call_specialization( + &self, + call: &ruff_python_ast::ExprCall, + ) -> Option { + LoweringModel::reified_call_specialization(self, call) } - fn constructor_specialization(&self, call: &ruff_python_ast::ExprCall) -> Option { + fn constructor_specialization( + &self, + call: &ruff_python_ast::ExprCall, + ) -> Option { self.reified_constructor_type_arguments(call) } @@ -953,7 +1022,7 @@ impl TypeInfo for SemanticModel<'_> { lhs: &Expr, rhs: &Expr, ) -> Option { - SemanticModel::parametric_is_plan(self, lhs, rhs) + LoweringModel::parametric_is_plan(self, lhs, rhs) } fn is_plain_value(&self, expr: &Expr) -> bool { @@ -1187,10 +1256,18 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::resolves_to_module_scope(self, reference) } + fn reads_class_scope_binding(&self, reference: &ExprName) -> Option { + SemanticModel::reads_class_scope_binding(self, reference) + } + fn is_forward_reference(&self, name: &ExprName) -> Option { SemanticModel::is_forward_reference(self, name) } + fn is_read_deferred(&self, expr: &Expr) -> bool { + SemanticModel::is_read_deferred(self, expr) + } + fn class_body_member_name(&self, name: &ExprName) -> Option { SemanticModel::class_body_member_name(self, name) } @@ -1231,6 +1308,26 @@ impl TypeInfo for SemanticModel<'_> { }) } + fn anon_named_tuple_id(&self, tuple: &ExprTuple) -> Option { + self.anon_named_tuple_class(tuple).map(|class| class.id) + } + + fn anon_named_tuple_construction( + &self, + tuple: &ExprTuple, + min_version: ruff_python_ast::PythonVersion, + ) -> Option { + if tuple.is_anon_named_tuple + || tuple.is_anon_named_tuple_value + || tuple.has_parameter_shape() + || !tuple.ctx.is_load() + { + return None; + } + let class = self.anon_named_tuple_class(tuple)?; + Some(anon_named_tuple_construction(self, class, min_version)) + } + fn promoted_type_display( &self, expr: &Expr, @@ -1385,34 +1482,12 @@ impl TypeInfo for SemanticModel<'_> { ) } - fn call_result_is_typevar_derived(&self, callee: &Expr) -> bool { - callee.inferred_type(self).is_some_and(|ty| { - ty_python_semantic::types::soundness::call_result_is_typevar_derived( - self.db(), - &self.program_environment(), - ty, - ) - }) - } - - fn is_specialized_generic_instance(&self, expr: &Expr) -> bool { - expr.inferred_type(self).is_some_and(|ty| { - ty_python_semantic::types::soundness::is_specialized_generic_instance( - self.db(), - &self.program_environment(), - ty, - ) - }) - } - - fn soundness_check_plan(&self, expr: &Expr) -> Option { - let ty = expr.inferred_type(self)?; - ty_python_semantic::types::soundness::runtime_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ) + fn soundness_sites<'ast>( + &self, + suite: &'ast [ruff_python_ast::Stmt], + positions: crate::SoundnessPositions, + ) -> Vec> { + ty_python_semantic::types::soundness_sites::soundness_sites(self, suite, positions) } fn inherited_parameter_default(&self, parameter: &ParameterWithDefault) -> Option { @@ -1426,46 +1501,12 @@ impl TypeInfo for SemanticModel<'_> { ty_python_semantic::types::ide_support::repeated_underscore_lowering(self, function) } - fn parameter_check_plan(&self, parameter: &Parameter) -> Option { - let ty = parameter.inferred_type(self)?; - ty_python_semantic::types::soundness::parameter_runtime_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ) - } - - fn call_positional_param_plan(&self, callee: &Expr, index: usize) -> Option { - let ty = callee.inferred_type(self)?; - ty_python_semantic::types::soundness::parameter_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ty_python_semantic::types::soundness::ArgSelector::Positional(index), - ) - } - - fn call_keyword_param_plan(&self, callee: &Expr, name: &str) -> Option { - let ty = callee.inferred_type(self)?; - ty_python_semantic::types::soundness::parameter_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ty_python_semantic::types::soundness::ArgSelector::Keyword(name), - ) + fn reads_names_bound_where_written(&self, expr: &Expr) -> bool { + LoweringModel::reads_names_bound_where_written(self, expr) } fn cast_check_plan(&self, type_expr: &Expr) -> Option { - let ty = type_expr.inferred_type(self)?; - ty_python_semantic::types::soundness::cast_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ) + LoweringModel::checked_cast_plan(self, type_expr) } fn cast_is_redundant(&self, value: &Expr, target: &Expr) -> bool { @@ -1675,13 +1716,6 @@ impl TypeInfo for SemanticModel<'_> { .map(|rendered| rendered.source) .map_err(|error| error.to_string()) } - - fn is_field_specifier(&self, expr: &Expr) -> bool { - matches!( - expr.inferred_type(self), - Some(Type::KnownInstance(KnownInstanceType::Field(_))) - ) - } } /// Render a type as python source text. ty's own display is written for the reader of a @@ -1718,6 +1752,39 @@ fn display_for_python<'db>( .to_string() } +/// `class` with each field's type spelled for the emitted python +fn anon_named_tuple_construction<'db>( + model: &LoweringModel<'db>, + class: AnonNamedTupleClass<'db>, + min_version: ruff_python_ast::PythonVersion, +) -> AnonNamedTupleConstruction { + let db = model.db(); + let env = model.program_environment(); + let fields = class + .fields + .into_iter() + .map(|(name, ty)| { + let field = if let Some(nested) = AnonNamedTupleClass::of(db, &env, ty) { + AnonNamedTupleField::NamedTuple(anon_named_tuple_construction( + model, + nested, + min_version, + )) + } else { + spell_for_python(db, &env, model, ty, min_version, true, None).map_or( + AnonNamedTupleField::Unspellable, + AnonNamedTupleField::Spelled, + ) + }; + (name.to_string(), field) + }) + .collect(); + AnonNamedTupleConstruction { + id: class.id, + fields, + } +} + /// Spell `ty` as a type expression the *emitted* file can resolve, or `None` when it has /// no such spelling. /// @@ -1744,7 +1811,7 @@ fn display_for_python<'db>( fn spell_for_python<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, - model: &SemanticModel<'db>, + model: &LoweringModel<'db>, ty: Type<'db>, min_version: ruff_python_ast::PythonVersion, literal: bool, @@ -1783,6 +1850,7 @@ fn spell_for_python<'db>( modules, typing_names, forward, + module_private: spelling.module_private_classes, }; resolvable_in_output( model, @@ -1846,7 +1914,7 @@ fn typing_name(name: &str, literal: bool) -> Option<&'static str> { /// signature it annotates. Anything else would reach the output unresolved, so the spelling /// is refused rather than written with a name that is not there. fn resolvable_in_output( - model: &SemanticModel<'_>, + model: &LoweringModel<'_>, min_version: ruff_python_ast::PythonVersion, spelled: &SynthesizedType, bare_classes: &[String], diff --git a/crates/by_transforms/tests/anon_named_tuple_runtime.rs b/crates/by_transforms/tests/anon_named_tuple_runtime.rs index 126789e7de..aafef442e9 100644 --- a/crates/by_transforms/tests/anon_named_tuple_runtime.rs +++ b/crates/by_transforms/tests/anon_named_tuple_runtime.rs @@ -9,9 +9,14 @@ mod common; +use std::fs; +use std::path::{Path, PathBuf}; use std::process::Command; -use by_transforms::{Config, PythonVersion, transpile}; +use by_transforms::{Config, PythonVersion, SoundnessPositions, transpile, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; /// every field type here is basedpython surface syntax that has no python /// spelling, so an unlowered one is either a `NameError` at class construction @@ -70,21 +75,174 @@ assert b._replace(name="bob").name == "bob", "named spelling" print("ok") "#; +/// the checker reads a plain tuple literal standing where an anonymous named tuple is +/// expected as an instance of it, wherever the expectation comes from. each one used to +/// stay a plain tuple at runtime unless it was an annotated assignment, a `return` or an +/// element of a list or set literal, so `by check` was clean and every field access below +/// raised `AttributeError` +const POSITIONS_PROGRAM: &str = r#" +from typing import Iterator + +type P = (a: int, b: int) + + +def take(p: (a: int, b: int)) -> int: + return p.a + + +def keyword(*, p: (a: int, b: int)) -> int: + return p.b + + +def star(*ps: (a: int, b: int)) -> int: + return sum(p.a for p in ps) + + +def default(p: (a: int, b: int) = (5, 6)) -> int: + return p.a + + +def pairs() -> Iterator[(a: int, b: int)]: + yield (1, 2) + + +def through_alias() -> P: + return (9, 10) + + +class Holder: + init(let pair: (a: int, b: int)) + + +class Box: + var pair: (a: int, b: int) = (0, 0) + + +assert take((7, 8)) == 7, "argument" +assert keyword(p=(7, 8)) == 8, "keyword argument" +assert star((1, 2), (3, 4)) == 4, "*args" +assert default() == 5, "default" +assert next(pairs()).b == 2, "yield" +assert through_alias().a == 9, "return through an alias" +assert Holder((3, 4)).pair.b == 4, "init parameter" + +items: list[(a: int, b: int)] = [] +items.append((1, 2)) +assert items[0].a == 1, "method argument" + +by_name: dict[str, (a: int, b: int)] = {"k": (1, 2)} +assert by_name["k"].b == 2, "dict value" + +unique: set[(a: int, b: int)] = {(1, 2)} +assert next(iter(unique)).a == 1, "set element" + +nested: list[list[(a: int, b: int)]] = [[(1, 2)]] +assert nested[0][0].b == 2, "nested list" + +box = Box() +assert box.pair.a == 0, "class default" +box.pair = (3, 4) +assert box.pair.b == 4, "attribute write" +print("ok") +"#; + +/// a field that is itself an anonymous named tuple builds the inner literal too, a union +/// builds the one arm that accepts the literal, and a write through a union receiver whose +/// members would each build the same anonymous named tuple builds it +const NESTED_AND_UNION_PROGRAM: &str = r#" +def outer(p: (inner: (x: int, y: int), label: str)) -> int: + return p.inner.y + + +class Left: + var pair: (a: int, b: int) = (0, 0) + + +class Right: + var pair: (a: int, b: int) = (0, 0) + + +class Maybe: + var pair: (a: int, b: int) | None = None + + +def assign(target: Left | Right | Maybe) -> int: + target.pair = (1, 2) + return target.pair.b + + +assert outer(((1, 2), "a")) == 2, "nested field" + +maybe: (a: int) | None = (1,) +assert maybe.a == 1, "optional" + +either: (a: int) | (b: str) = ("x",) +assert either.b == "x", "the arm whose fields accept the elements" + +assert assign(Left()) == 2, "union receiver" +assert assign(Right()) == 2, "union receiver" +assert assign(Maybe()) == 2, "a member that may also hold None builds the same class" +print("ok") +"#; + +/// a value that already is the named tuple passes through untouched, and a literal where a +/// plain tuple is expected stays a plain tuple +const UNCHANGED_PROGRAM: &str = r#" +def same(p: (a: int, b: int)) -> (a: int, b: int): + return p + + +def kind(t: tuple[int, int]) -> str: + return type(t).__name__ + + +built: (a: int, b: int) = (1, 2) +assert same(built) === built, "an existing named tuple is not rebuilt" +assert kind((1, 2)) == "tuple", "a plain tuple stays a tuple" +assert kind(built) == type(built).__name__, "a named tuple passed as a tuple stays itself" +print("ok") +"#; + +/// a named field after more than one positional field, or after a modifier, makes the same +/// anonymous named tuple a named field right after the first does: the annotation is the class +/// a literal under it is built as, not a plain tuple +const NAMED_AFTER_POSITIONALS_PROGRAM: &str = r#" +def later(p: (int, str, name: bytes)) -> bytes: + return p.name + + +def borrowed(p: (int, local b: str)) -> str: + return p.b + + +assert later.__annotations__["p"]._fields == ("arg0", "arg1", "name"), "after two positional fields" +assert borrowed.__annotations__["p"]._fields == ("arg0", "b"), "after a modifier" +assert later((1, "a", b"x")) == b"x", "built" +assert borrowed((1, "y")) == "y", "built" +print("ok") +"#; + +fn run(program: &str) { + run_with( + program, + &Config { + min_version: PythonVersion::PY313, + ..Config::default() + }, + ); +} + #[expect( clippy::print_stderr, reason = "a skipped test must say why it skipped, or it reads as a pass" )] -fn run(program: &str) { +fn run_with(program: &str, config: &Config) { let Some(python) = common::python() else { eprintln!("skipping anon named tuple runtime test: no interpreter found"); return; }; - let config = Config { - min_version: PythonVersion::PY313, - ..Config::default() - }; - let transpiled = transpile(program, &config).expect("transpile should succeed"); + let transpiled = transpile(program, config).expect("transpile should succeed"); let output = Command::new(&python) .arg("-c") @@ -131,3 +289,136 @@ print("ok") fn a_hoisted_class_sees_the_runtime() { run(WRAPPED_OPTIONAL_PROGRAM); } + +#[test] +fn a_plain_tuple_is_built_wherever_one_is_expected() { + run(POSITIONS_PROGRAM); +} + +#[test] +fn a_nested_field_and_a_union_arm_are_built() { + run(NESTED_AND_UNION_PROGRAM); +} + +#[test] +fn a_named_field_anywhere_makes_an_anonymous_named_tuple() { + run(NAMED_AFTER_POSITIONALS_PROGRAM); +} + +#[test] +fn a_named_tuple_and_a_plain_tuple_are_left_as_they_are() { + run(UNCHANGED_PROGRAM); +} + +/// every runtime soundness check on, a parameter's entry check among them: each checks +/// the value against the class the literal was built as +#[test] +fn built_values_pass_every_soundness_check() { + for program in [ + POSITIONS_PROGRAM, + NESTED_AND_UNION_PROGRAM, + UNCHANGED_PROGRAM, + ] { + run_with( + program, + &Config { + min_version: PythonVersion::PY313, + soundness: SoundnessPositions::all(), + ..Config::default() + }, + ); + } +} + +/// transpile every one of `files` as one project, into a fresh directory under the cargo temp +/// dir, or the first transpile error +fn transpile_project(name: &str, files: &[(&str, &str)]) -> Result { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(name); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + for (path, _) in files { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = + transpile_typed(&db, file, &config, None).map_err(|error| error.message)?; + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + Ok(dir) +} + +/// a literal standing where another module's anonymous named tuple is expected is built in +/// this module, as a class declared from the checker's field types: the other module's class +/// is named by its own transpilation, which this one cannot see +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_literal_for_another_modules_named_tuple_is_built() { + let Some(python) = common::python() else { + eprintln!("skipping anon named tuple runtime test: no interpreter found"); + return; + }; + let dir = transpile_project( + "anon_named_tuple_cross_module", + &[ + ( + "/shapes.by", + "def take(p: (a: int, b: list[str])) -> str:\n return p.b[p.a]\n", + ), + ( + "/main.by", + "from shapes import take\n\nassert take((1, [\"x\", \"y\"])) == \"y\"\nprint(\"ok\")\n", + ), + ], + ) + .expect("transpile should succeed"); + let output = Command::new(&python) + .arg("main.py") + .current_dir(&dir) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- main.py ---\n{}", + String::from_utf8_lossy(&output.stderr), + fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +/// a field type the module building the literal has no python spelling for is refused, rather +/// than declared in a class that would not run +#[test] +fn a_field_the_module_cannot_spell_is_refused() { + let error = transpile_project( + "anon_named_tuple_unspellable_field", + &[ + ( + "/callbacks.by", + "def run(p: (f: (int) -> str)) -> str:\n return p.f(1)\n", + ), + ("/main.by", "from callbacks import run\n\nrun((str,))\n"), + ], + ) + .expect_err("the field type has no spelling"); + assert_eq!( + error, + "the tuple at offset 31 is built as an anonymous named tuple whose field `f` has no \ + type the emitted python can name; construct it explicitly, or give it a `NamedTuple` \ + class of its own" + ); +} diff --git a/crates/by_transforms/tests/attribute_write_runtime.rs b/crates/by_transforms/tests/attribute_write_runtime.rs new file mode 100644 index 0000000000..ddb2fc55be --- /dev/null +++ b/crates/by_transforms/tests/attribute_write_runtime.rs @@ -0,0 +1,59 @@ +//! Runtime test for a write to a declared attribute in a `.by` file. +//! +//! The value is read as what it is checked as: the declaration is its expected type, as it is +//! for an annotated assignment. A construction of a reified generic class is told the +//! specialization it builds from that, where without it the class could not say which one. + +mod common; + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +const PROGRAM: &str = r#" +class Box[reified T]: + def kind(self) -> str: + return T.__name__ + + +class Holder: + box: Box[int] + + def __init__(self): + self.box = Box() + + +holder = Holder() +assert holder.box.kind() == "int", "construction in a method" +holder.box = Box() +assert holder.box.kind() == "int", "construction through an instance" +print("ok") +"#; + +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +#[test] +fn a_reified_construction_takes_the_declared_specialization() { + let Some(python) = common::python() else { + eprintln!("skipping attribute write runtime test: no interpreter found"); + return; + }; + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/by_transforms/tests/destructure_runtime.rs b/crates/by_transforms/tests/destructure_runtime.rs index 312f01ab11..a1e670cfdc 100644 --- a/crates/by_transforms/tests/destructure_runtime.rs +++ b/crates/by_transforms/tests/destructure_runtime.rs @@ -159,24 +159,57 @@ assert not [name for name in vars(Colour) if "_by_" in name], ( print("ok") "#; -#[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] -fn destructuring_runs_correctly() { - let Some(python) = python() else { - eprintln!("skipping destructuring runtime test: no `python3` interpreter found"); - return; - }; +/// subjects and guards written over several lines inside parentheses, which are not +/// part of their ranges. the lowering moves each onto a line of a statement it writes — +/// an assignment, a `match` header, a `case` guard — where it still needs a grouping +const MULTI_LINE: &str = r#" +class Point: + __match_args__ = ("x", "y") + def __init__(self, x: int, y: int): + self.x = x + self.y = y + +def make() -> Point: + return Point(1, 2) + +def summed() -> int: + let Point(x, y) := ( + make + () + ) + return x + y + +assert summed() == 3, "a `let` subject over several lines" + +def classify(p: Point, q: int) -> str: + match ( + p if q >= 0 + else Point(0, 0) + ): + case Point(x, y) and object() if ( + x == 1 + and q > 0 + ): + return "first" + case _: + return "other" + +assert classify(Point(1, 2), 1) == "first", "a guard over several lines" +assert classify(Point(1, 2), 0) == "other" +assert classify(Point(0, 2), 1) == "other" + +print("ok") +"#; + +fn run(python: &str, program: &str) { let config = Config { min_version: PythonVersion::PY313, ..Config::default() }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(python) .arg("-c") .arg(&transpiled) .output() @@ -190,3 +223,32 @@ fn destructuring_runs_correctly() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn destructuring_runs_correctly() { + let Some(python) = python() else { + eprintln!("skipping destructuring runtime test: no `python3` interpreter found"); + return; + }; + + run(&python, PROGRAM); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_subject_written_over_several_lines_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!( + "skipping `a_subject_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" + ); + return; + }; + run(&python, MULTI_LINE); +} diff --git a/crates/by_transforms/tests/forward_reference_runtime.rs b/crates/by_transforms/tests/forward_reference_runtime.rs index 90cf287857..d4546f9530 100644 --- a/crates/by_transforms/tests/forward_reference_runtime.rs +++ b/crates/by_transforms/tests/forward_reference_runtime.rs @@ -1,4 +1,5 @@ -//! Runtime test for forward references in annotations python evaluates as the definition runs. +//! Runtime test for forward references in annotations python evaluates as the definition runs, +//! and in the types written outside an annotation that it evaluates where they stand //! //! basedpython resolves every annotation as deferred, so a signature may name a class defined //! further down, the class it sits in, or a name imported only for the checker. Before 3.14 @@ -6,8 +7,9 @@ //! forward reference raises `NameError` at import — which asserting on the lowered text cannot //! see. //! -//! Needs an interpreter older than 3.14. From 3.14 annotations are deferred natively, so there -//! is nothing to observe, and the test skips rather than pass without checking anything. +//! The annotation test needs an interpreter older than 3.14: from 3.14 annotations are deferred +//! natively, so there is nothing to observe, and it skips rather than pass without checking +//! anything //! //! Below 3.10 the output defers every annotation with `from __future__ import annotations` //! instead, and there a quote would reach `__annotations__` as a string inside a string, which @@ -15,7 +17,7 @@ use std::process::Command; -use by_transforms::{Config, PythonVersion, transpile}; +use by_transforms::{Config, PythonVersion, reverse_transpile, transpile}; mod common; use common::python; @@ -167,3 +169,174 @@ fn a_forward_reference_under_the_future_import_is_not_quoted() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); interpreters::ran(&interpreter, target); } + +/// a type written outside an annotation that python evaluates where it stands, naming classes +/// and an import bound only further down. the checker reads the type arguments as deferred, so +/// the transpiler quotes them, and a name a quote reads is bound where it is imported rather +/// than when the module first reads it, since a quote is evaluated against the module's +/// namespace +const OUTSIDE_AN_ANNOTATION: &str = r#" +from typing import Callable, NamedTuple, TypeVar, Union, get_args, get_type_hints + +items = list[Later]() +Alias = dict[str, Later] +Maybe = Union[Later, None] +Handler = Callable[[Later], Later] +T = TypeVar("T", bound=Later) +Pair = NamedTuple("Pair", [("first", Later)]) +Counts = list[OrderedDict] +Pairs = list[(Later, int)] +pair_map = dict[str, (Later, int)]() +P = TypeVar("P", bound=(Later, int)) +MaybeList = list[Later?] +maybes = list[Later?]() +HandlerList = list[(Later) -> None] +Method = list[int.() -> Later] +Fields = list[(first: Later, count: int)] +TupleParams = Callable[(Later, int), Later] +annotated: list[(first: Later, count: int)] = [] + + +class Base[S]: ... + + +class Bar(Base[Later]): ... + + +class PairBar(Base[(Later, int)]): ... + + +class MaybeBar(Base[Later?]): ... + + +class Later: ... + + +from collections import OrderedDict + + +# a builtin generic keeps a string argument as it is, where `typing` makes it a `ForwardRef` +def resolve(reference: object) -> object: + text = reference if isinstance(reference, str) else reference.__forward_arg__ + return eval(text, globals()) + + +assert resolve(get_args(Alias)[1]) == Later, get_args(Alias) +assert resolve(get_args(Maybe)[0]) == Later, get_args(Maybe) +assert resolve(get_args(Handler)[1]) == Later, get_args(Handler) +assert resolve(T.__bound__) == Later, T.__bound__ +assert resolve(get_args(Bar.__orig_bases__[0])[0]) == Later, Bar.__orig_bases__ +assert resolve(get_args(Counts)[0]) == OrderedDict, get_args(Counts) +assert resolve(get_args(Pairs)[0]) == tuple[Later, int], get_args(Pairs) +assert resolve(P.__bound__) == tuple[Later, int], P.__bound__ +assert resolve(get_args(PairBar.__orig_bases__[0])[0]) == tuple[Later, int], PairBar.__orig_bases__ +assert resolve(get_args(MaybeList)[0]) == Later | None, get_args(MaybeList) +assert resolve(get_args(MaybeBar.__orig_bases__[0])[0]) == Later | None, MaybeBar.__orig_bases__ +assert resolve(get_args(HandlerList)[0]) == Callable[[Later], None], get_args(HandlerList) +assert resolve(get_args(Method)[0]) == Callable[[int], Later], get_args(Method) +params, returns = get_args(TupleParams) +assert [resolve(params[0]), params[1], resolve(returns)] == [Later, int, Later], get_args(TupleParams) +fields = resolve(get_args(Fields)[0]) +assert get_type_hints(fields) == {"first": Later, "count": int}, get_type_hints(fields) +print("ok") +"#; + +#[test] +fn a_forward_reference_outside_an_annotation_resolves() { + let Some(python) = python() else { + return; + }; + let config = Config { + min_version: PythonVersion::PY310, + ..Config::default() + }; + let transpiled = transpile(OUTSIDE_AN_ANNOTATION, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +/// python that spells a forward reference outside an annotation as a string, in each place the +/// checker reads one as deferred: an alias's type argument, a union's, a callable's, an +/// optional's, a type variable's bound and a base's. the reverse transpile writes each as the +/// name it spells, and the forward transpile quotes it again, so python sees the same types +const QUOTED_OUTSIDE_AN_ANNOTATION: &str = r#" +from typing import Callable, Generic, List, Optional, TypeVar, Union, get_args, get_type_hints + +T = TypeVar("T") + + +class Foo(Generic[T]): + pass + + +Alias = List["Later"] +UAlias = Union["Later", int] +CAlias = Callable[["Later"], "Later"] +MaybeLater = Optional["Later"] +B = TypeVar("B", bound="Later") + + +class Bar(Foo["Later"]): + pass + + +class Later: + pass + + +def uses(a: Alias, u: UAlias, c: CAlias, m: MaybeLater, b: B) -> Bar: ... + + +print(get_args(Alias), get_args(UAlias), get_args(CAlias), get_args(MaybeLater)) +print(B.__bound__, Bar.__orig_bases__) +print(get_type_hints(uses)) +"#; + +/// what `program` prints on `python` +fn printed(python: &str, program: &str) -> String { + let output = Command::new(python) + .arg("-c") + .arg(program) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "program failed on {python}:\n--- stderr ---\n{}\n--- program ---\n{program}", + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +#[test] +fn a_forward_reference_outside_an_annotation_survives_the_round_trip() { + let Some(python) = python() else { + return; + }; + let config = Config { + min_version: PythonVersion::PY310, + ..Config::default() + }; + let by = reverse_transpile(QUOTED_OUTSIDE_AN_ANNOTATION, &config) + .expect("reverse transpile should succeed"); + assert!( + !by.contains("\"Later\""), + "a forward reference should be the name it spells:\n{by}" + ); + let back = transpile(&by, &config).expect("transpile should succeed"); + assert_eq!( + printed(&python, &back), + printed(&python, QUOTED_OUTSIDE_AN_ANNOTATION), + "--- basedpython ---\n{by}\n--- transpiled back ---\n{back}" + ); +} diff --git a/crates/by_transforms/tests/grapheme_runtime.rs b/crates/by_transforms/tests/grapheme_runtime.rs index 91534a0f5d..b3c3e32aba 100644 --- a/crates/by_transforms/tests/grapheme_runtime.rs +++ b/crates/by_transforms/tests/grapheme_runtime.rs @@ -73,8 +73,34 @@ assert type("a") is str, "a bare literal is still a plain str" print("ok") "#; -#[test] -fn grapheme_surface_runs_correctly() { +/// `first` and `last` bind a receiver that is more than a name to a temporary, which +/// python refuses to bind with `:=` in a comprehension's iterable or anywhere in a +/// comprehension in a class body +const BOUND_RECEIVER: &str = r#" +from ty_extensions import Character + +def greeting() -> str: + return "a🇺🇸é" + +assert [c for c in [greeting().first, greeting().last]] == ["a", "é"], "in an iterable" + +class Letters: + let firsts: list[Character?] = [greeting().first for _ in [0]] + +assert Letters.firsts == ["a"], "in a class body comprehension" + +print("ok") +"#; + +fn transpiled(program: &str) -> String { + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + transpile(program, &config).expect("transpile should succeed") +} + +fn run_with_regex(program: &str) { let Some(interpreter) = interpreters::oldest( PythonVersion::PY313, "import regex", @@ -82,12 +108,7 @@ fn grapheme_surface_runs_correctly() { ) else { return; }; - - let config = Config { - min_version: PythonVersion::PY313, - ..Config::default() - }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let transpiled = transpiled(program); let output = Command::new(&interpreter.command) .arg("-c") @@ -104,3 +125,39 @@ fn grapheme_surface_runs_correctly() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); interpreters::ran(&interpreter, PythonVersion::PY313); } + +#[test] +fn grapheme_surface_runs_correctly() { + run_with_regex(PROGRAM); +} + +#[test] +fn a_bound_receiver_runs_where_python_refuses_an_assignment_expression() { + run_with_regex(BOUND_RECEIVER); +} + +/// the same module compiles, `regex` or not: an assignment expression where python +/// refuses one is a `SyntaxError` before the helpers that need `regex` ever run +#[test] +fn a_bound_receiver_compiles_where_python_refuses_an_assignment_expression() { + let Some(interpreter) = interpreters::oldest(PythonVersion::PY313, "", "") else { + return; + }; + let transpiled = transpiled(BOUND_RECEIVER); + + let output = Command::new(&interpreter.command) + .args([ + "-c", + "import sys; compile(sys.argv[1], '', 'exec')", + &transpiled, + ]) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "the transpiled grapheme program does not compile on {interpreter}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + interpreters::ran(&interpreter, PythonVersion::PY313); +} diff --git a/crates/by_transforms/tests/if_let_runtime.rs b/crates/by_transforms/tests/if_let_runtime.rs index bd749aeaa7..9c02f09130 100644 --- a/crates/by_transforms/tests/if_let_runtime.rs +++ b/crates/by_transforms/tests/if_let_runtime.rs @@ -77,24 +77,42 @@ assert not [name for name in dict(globals()) if name.startswith("_by_if_let")], print("ok") "#; -#[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] -fn if_let_chains_run_correctly() { - let Some(python) = python() else { - eprintln!("skipping `if let` runtime test: no `python3` interpreter found"); - return; - }; +/// a subject and a condition written over several lines inside parentheses, which +/// are not part of their ranges. the lowering moves each into a statement header it +/// writes, where it still needs a grouping +const MULTI_LINE: &str = r#" +class Box: + def __init__(self, value: int | None): + self.value = value + +def unboxed(b: Box, fallback: bool) -> int: + if let int(x) := ( + b + .value + ): + return x + elif ( + fallback + and b.value is None + ): + return -1 + return 0 + +assert unboxed(Box(3), False) == 3, "a subject over several lines" +assert unboxed(Box(None), True) == -1, "a condition over several lines" +assert unboxed(Box(None), False) == 0 +print("ok") +"#; + +fn run(python: &str, program: &str) { let config = Config { min_version: PythonVersion::PY313, ..Config::default() }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(python) .arg("-c") .arg(&transpiled) .output() @@ -108,3 +126,32 @@ fn if_let_chains_run_correctly() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn if_let_chains_run_correctly() { + let Some(python) = python() else { + eprintln!("skipping `if let` runtime test: no `python3` interpreter found"); + return; + }; + + run(&python, PROGRAM); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_subject_written_over_several_lines_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!( + "skipping `a_subject_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" + ); + return; + }; + run(&python, MULTI_LINE); +} diff --git a/crates/by_transforms/tests/inferred_annotation_runtime.rs b/crates/by_transforms/tests/inferred_annotation_runtime.rs index b6cf2e695f..b0014ebbe5 100644 --- a/crates/by_transforms/tests/inferred_annotation_runtime.rs +++ b/crates/by_transforms/tests/inferred_annotation_runtime.rs @@ -57,8 +57,10 @@ fn run_at( ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); + // a deferred `Dec` is read through the module's `_LazyFrom` + let dec = if lazy_imports { "_by_lazy.Dec" } else { "Dec" }; assert!( - transpiled.contains(&format!("d: {annotation} = Dec(1)")), + transpiled.contains(&format!("d: {annotation} = {dec}(1)")), "the annotation names the module:\n{transpiled}" ); let output = Command::new(&interpreter.command) @@ -187,7 +189,13 @@ fn an_inferred_annotation_adds_no_import_cycle() { fs::write(dir.join(relative), transpiled).expect("write module"); } let m = fs::read_to_string(dir.join("m.py")).expect("read m"); - assert!(m.contains("x: \"x_mod.Thing\" = make()"), "{m}"); + // a deferred `make` is read through the module's `_LazyFrom` + let make = if lazy_imports { + "_by_lazy.make" + } else { + "make" + }; + assert!(m.contains(&format!("x: \"x_mod.Thing\" = {make}()")), "{m}"); let output = Command::new(&interpreter.command) .arg("main.py") .current_dir(&dir) diff --git a/crates/by_transforms/tests/lazy_import_runtime.rs b/crates/by_transforms/tests/lazy_import_runtime.rs index 3d817b983b..38fd171daf 100644 --- a/crates/by_transforms/tests/lazy_import_runtime.rs +++ b/crates/by_transforms/tests/lazy_import_runtime.rs @@ -1,70 +1,552 @@ -//! Runtime tests for the lazy-import polyfill (`--min-version` < 3.15, which -//! has no PEP 810 `lazy` keyword and so binds `from x import y` to a -//! `_LazyAttr` proxy). +//! Runtime tests for the lazy-import polyfill (`--min-version` < 3.15, which has no PEP 810 +//! `lazy` keyword). A `from x import y` there binds nothing where it is written: every read of +//! `y` is written as a read through the module's `_LazyFrom`, which resolves `x.y` on first use +//! and stores it in the module's namespace //! -//! These have to be *runtime* tests spanning two modules: the proxy only -//! misbehaves once a real interpreter applies an operator to it across a real -//! import, so neither the transform unit tests (which assert lowered text) nor -//! the mdtests (which assert types) can see it. The proxy previously forwarded -//! only `__getattr__` / `__call__` / `__getitem__`, which meant every -//! *unforwarded* dunder silently fell back to `object`'s version — `a == b` -//! compared proxy identity and answered `False` for equal values, and `a + 1` -//! raised `TypeError`. Python looks special methods up on the type and never -//! routes them through `__getattr__`, so that class of bug is invisible until -//! executed. +//! These have to be *runtime* tests spanning several modules: what a read evaluates to, what +//! another module importing the name sees, and what a patch of the name reaches only show once +//! a real interpreter runs the emitted python across a real import, so neither the transform +//! unit tests (which assert lowered text) nor the mdtests (which assert types) can see them //! -//! [`import_stays_lazy_until_first_use`] guards the other side of the trade: -//! the proxy exists to defer the imported module's execution, so a "fix" that -//! resolved eagerly would pass every forwarding assertion while quietly -//! destroying the feature. +//! [`import_stays_lazy_until_first_use`] guards the other side of the trade: the rewrite exists +//! to defer the imported module's execution, so a "fix" that imported eagerly would pass every +//! other assertion while quietly destroying the feature use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use by_transforms::{Config, transpile}; +use by_transforms::{Config, transpile, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; mod interpreters; use interpreters::Interpreter; -/// module whose values the importer exercises. every binding is a plain value -/// (not a callable), which is exactly the case the proxy used to break +/// a soundness check in a function on a class the module imports further down is made once +/// the import has run. the import is deferred, so the name is not in the module's namespace +/// then, and the check asks the module's `_LazyFrom` whether its statement has run +/// a forward reference the transpiler quotes in a type argument whose generic comes from a +/// sibling module. the quote reads `JSONDecoder` when something asks for the type, against the +/// module's namespace, so the import is bound where it is written however the module that +/// declares `Box` is resolved +const QUOTED_THROUGH_A_SIBLING: &[(&str, &str)] = &[ + ( + "/by_boxes_mod.by", + r#" +class Box[T]: + pass +"#, + ), + ( + "/main.by", + r#" +import typing +from json import JSONDecoder +from by_boxes_mod import Box + +Alias = Box[F | JSONDecoder] + +def h(x: Alias): + pass + +class F: + pass + +hints = typing.get_type_hints(h) +assert typing.get_args(hints["x"]) == (F | JSONDecoder,), hints +print("ok") +"#, + ), +]; + +/// the classes a lowering hoists ahead of the module write a field type naming one of the +/// module's imports as a string, which reads the import when something asks for the field's +/// type, so the import is bound where it is written +const HOISTED_FIELDS: &str = r#" +import typing +from json import JSONDecoder, JSONEncoder + +Pair = list[(first: JSONDecoder, count: int)] +Shaped = list[protocol(encoder: JSONEncoder)] + +def hints(alias: object) -> dict[str, object]: + return typing.get_type_hints(typing.get_args(alias)[0]) + +assert hints(Pair)["first"] == JSONDecoder, hints(Pair) +assert hints(Shaped)["encoder"] == JSONEncoder, hints(Shaped) +print("ok") +"#; + +const ONCE_BOUND: &[(&str, &str)] = &[ + ( + "/by_once_mod.by", + r#" +class Thing: + pass + +def things() -> list[Thing]: + wrong: dynamic = [1] + return wrong +"#, + ), + ( + "/main.by", + r#" +def first[T](xs: list[T]) -> T: + return xs[0] + +def use() -> object: + return first(things()) + +from by_once_mod import Thing, things + +try: + use() +except TypeError as error: + assert "type soundness violation" in str(error), str(error) +else: + raise AssertionError("the check is made once the import has run") +print("ok") +"#, + ), +]; + +/// `from . import x` names `x` in the package itself: a function its `__init__` defines, +/// or a submodule nothing has imported yet. python reads the attribute first and imports +/// the submodule only when there is none +const RELATIVE_PACKAGE: &str = r#" +def helper() -> str: + return "helper" +"#; + +const RELATIVE_SUBMODULE: &str = r#" +NAME = "other" +"#; + +const RELATIVE_USER: &str = r#" +from . import helper +from . import other + +def run() -> str: + return helper() + " " + other.NAME +"#; + +const RELATIVE_MAIN: &str = r#" +from by_relative_pkg.user import run + +assert run() == "helper other", run() +print("ok") +"#; + +/// a package whose `__init__` imports its own submodule, relatively and by its absolute name +/// +/// the submodule is read off the package, which is the module doing the reading, and has +/// to be imported as python's fromlist imports it +const INIT_SUBMODULE: &[(&str, &str)] = &[ + ( + "by_init_rel/__init__", + "from . import helper\n\ndef use() -> str:\n return helper.NAME\n", + ), + ("by_init_rel/helper", "NAME = \"relative\"\n"), + ( + "by_init_abs/__init__", + "from by_init_abs import helper\n\ndef use() -> str:\n return helper.NAME\n", + ), + ("by_init_abs/helper", "NAME = \"absolute\"\n"), + ( + "main", + r#" +import by_init_rel +import by_init_abs +from by_init_rel import helper as outside + +print(outside.NAME) +print(by_init_rel.use(), by_init_abs.use()) +"#, + ), +]; + +/// two modules re-exporting a name each takes from the other, which neither binds +const REEXPORT_CYCLE: &[(&str, &str)] = &[ + ("by_rx_a", "from by_rx_b import x\n"), + ("by_rx_b", "from by_rx_a import x\n"), + ( + "main", + r#" +try: + import by_rx_a + by_rx_a.x +except ImportError as error: + print(type(error).__name__) +"#, + ), +]; + +/// a name an import binds only on some paths reads the builtin of that name where the +/// import did not run, as python's global lookup does, and is a `NameError` where there is +/// no builtin +const BUILTIN_FALLBACK: &[(&str, &str)] = &[( + "main", + r#" +import sys + +if sys.version_info < (3, 0): + from itertools import izip as zip + +if "--pretty" in sys.argv: + from pprint import pprint as print + +if len(sys.argv) > 5: + from json import dumps + +print(list(zip([1, 2], [3, 4]))) +try: + dumps +except NameError as error: + print(str(error)) +"#, +)]; + +/// a reload runs a module again over the namespace it had, so its imports have to bind what +/// their modules hold now, not what the first run read +const RELOAD: &[(&str, &str)] = &[ + ("by_reload_src", "VALUE = 1\n"), + ("by_reload_user", "from by_reload_src import VALUE\n"), + ( + "main", + r#" +import importlib +import sys + +sys.dont_write_bytecode = True +import by_reload_src +import by_reload_user + +first = by_reload_user.VALUE +with open(by_reload_src.__file__, "w") as source: + source.write("VALUE = 2\n") +importlib.invalidate_caches() +importlib.reload(by_reload_src) +importlib.reload(by_reload_user) +print(first, by_reload_user.VALUE) +"#, + ), +]; + +/// a failing `from` import reports the error python reports: a submodule's own missing +/// dependency, an `AttributeError` the module's body raises, the name that is not there, and +/// the name a module still being imported has not bound yet +/// each case is a module of its own that imports and reads the name at its top level, where +/// python raises at the import and the polyfill at the read — both while the case module is +/// being imported, inside the `try` that catches it +const IMPORT_ERRORS: &[(&str, &str)] = &[ + ("by_err_pkg/__init__", ""), + ( + "by_err_pkg/broken", + "import by_not_installed_dependency\n\nX = 1\n", + ), + ( + "by_err_body", + "raise AttributeError(\"raised by the body\")\n", + ), + ( + "by_err_case_dependency", + "from by_err_pkg import broken\n\nbroken.X\n", + ), + ( + "by_err_case_body", + "from by_err_body import anything\n\nanything\n", + ), + ( + "by_err_case_name", + "from json import by_no_such_name\n\nby_no_such_name\n", + ), + ( + "by_err_case_submodule", + "from urllib import by_no_such_submodule\n\nby_no_such_submodule\n", + ), + // a module whose name is read while it is still being imported + ( + "by_err_case_circular", + "from by_err_circular_other import f\n\nf()\nx = 1\n", + ), + ( + "by_err_circular_other", + "from by_err_case_circular import x\n\ndef f() -> None:\n pass\n\nx\n", + ), + ( + "main", + r#" +import importlib + +for case in ("dependency", "body", "name", "submodule", "circular"): + try: + importlib.import_module("by_err_case_" + case) + except ImportError as error: + print(case, type(error).__name__, str(error), error.name, error.path) + print(getattr(error, "name_from", None)) + except AttributeError as error: + # the attribute read the polyfill resolves the name in gives the error the name it + # was read under, which an import raising it does not + print(case, type(error).__name__, str(error)) +"#, + ), +]; + +/// module whose values the importer exercises const VALUES_MODULE: &str = r#" num: int = 5 text: str = "hi" items: list[int] = [1, 2, 3] +ratio: float = 200.0 +nothing: int? = None class Point: def __init__(self, x: int): self.x = x + +ORIGIN = Point(0) + +def origin() -> Point: + return ORIGIN "#; -/// every operator here reaches the proxy as a dunder looked up on its *type*, -/// so each assertion fails unless the proxy forwards that specific method +/// an imported name is the object it names, so everything that asks *which* object it is — +/// identity, `type()`, `json`, a library deciding how to treat a value by what its type +/// defines — gets the answer the eager import gives const VALUES_MAIN: &str = r#" -from by_values_mod import num, text, items, Point - -assert num == 5, "eq" -assert not (num != 5), "ne" -assert num < 9 and num > 1 and num <= 5 and num >= 5, "ordering" -assert num + 1 == 6 and 1 + num == 6, "add / radd" -assert num * 2 == 10 and 10 - num == 5, "mul / rsub" -assert num % 3 == 2 and 2 ** num == 32, "mod / rpow" -assert -num == -5 and abs(-num) == 5, "unary" -assert text + "!" == "hi!" and "oh " + text == "oh hi", "concat / rconcat" -assert len(items) == 3, "len" -assert [i * 2 for i in items] == [2, 4, 6], "iter" -assert 2 in items, "contains" -assert items[0] == 1 and text[1] == "i", "getitem" -assert bool(num) and hash(num) == hash(5), "bool / hash" -assert str(text) == "hi" and repr(text) == "'hi'", "str / repr" -assert f"{num:03d}" == "005", "format" -assert int(num) == 5 and float(num) == 5.0, "int / float" -assert sorted([3, num, 1]) == [1, 3, 5], "sorting uses forwarded comparisons" -assert list(reversed(items)) == [3, 2, 1], "reversed" -assert isinstance(num, int), "isinstance via __class__" -assert isinstance(Point(1), Point), "isinstance against a lazily-imported class" -assert Point(7).x == 7, "call + attribute" +import json +import by_values_mod as values +from by_values_mod import num, text, items, ratio, nothing, Point, ORIGIN, origin + +def approx_like(expected: object) -> str: + # `pytest.approx` reads anything whose type has `__iter__` as a collection + return "collection" if hasattr(type(expected), "__iter__") else "scalar" + +assert origin() === ORIGIN, "identity across the module boundary" +assert ORIGIN === values.ORIGIN, "the name is the module's object" +assert type(num) === int and type(text) === str and type(ORIGIN) === Point, "type()" +assert json.dumps({"n": num, "t": text, "i": items}) == '{"n": 5, "t": "hi", "i": [1, 2, 3]}', "json" +assert approx_like(ratio) == "scalar" and approx_like(items) == "collection", "approx" +assert nothing === None, "identity with None" +assert num + 1 == 6 and 1 + num == 6 and text + "!" == "hi!" and len(items) == 3, "operators" +assert isinstance(Point(1), Point) and Point(7).x == 7, "a class constructs and tests" +print("ok") +"#; + +/// a module whose one value is told apart by identity +const TOKEN_MODULE: &str = r#" +class Token: + pass + +TOKEN = Token() +"#; + +/// every scope a read can reach the module's binding from +const SCOPES_MAIN: &str = r#" +import by_token_mod as m +from by_token_mod import TOKEN + +def nested() -> object: + def inner() -> object: + return TOKEN + return inner() + +class Body: + seen = TOKEN + +def default(value: object = TOKEN) -> object: + return value + +def reads_global() -> object: + global TOKEN + return TOKEN + +def outer() -> object: + TOKEN = "local" + class Inner: + seen = TOKEN + TOKEN = "class" + return Inner.seen + +later = lambda: TOKEN +listed = [TOKEN for _ in range(1)] +text = f"{TOKEN!r}" + +assert nested() === m.TOKEN, "a nested function" +assert Body.seen === m.TOKEN, "a class body" +assert default() === m.TOKEN, "a default value" +assert reads_global() === m.TOKEN, "a `global` read" +assert outer() === m.TOKEN, "a class body reads the module, not the function around it" +assert later() === m.TOKEN, "a lambda" +assert listed[0] === m.TOKEN, "a comprehension" +assert text == repr(m.TOKEN), "an f-string" +print("ok") +"#; + +/// a name bound again keeps its import as written, and every read finds what python's would +const REBOUND_MAIN: &str = r#" +import by_token_mod as m +from by_token_mod import TOKEN + +first = TOKEN + +def read() -> object: + return TOKEN + +def rebind() -> None: + global TOKEN + TOKEN = "rebound" + +assert first === m.TOKEN and read() === m.TOKEN, "before the rebinding, the import" +rebind() +assert read() == "rebound", "after it, the new binding" +print("ok") +"#; + +/// a read made before the import statement has run fails as python's does +const EARLY_MAIN: &str = r#" +def early() -> object: + return TOKEN + +try: + early() +except NameError as error: + assert str(error) == "name 'TOKEN' is not defined", str(error) +else: + raise AssertionError("a read before the import ran must raise") + +from by_token_mod import TOKEN +import by_token_mod as m + +assert early() === m.TOKEN, "and once it has run, reads the import" +print("ok") +"#; + +const REEXPORT_SOURCE: &str = r#" +class Marker: + pass + +VALUE = Marker() +"#; + +/// imports `VALUE` and never reads it, so only another module can +const REEXPORT_MIDDLE: &str = r#" +from by_reexport_src import VALUE +"#; + +const REEXPORT_LISTED: &str = r#" +from by_reexport_src import VALUE + +__all__ = ["VALUE"] +"#; + +const REEXPORT_STAR: &str = r#" +from by_reexport_listed import * + +def got() -> object: + return VALUE +"#; + +/// a module that imports a name passes it on, as python's would, through every way another +/// module asks for it. until something reads it, the name is not in the module's namespace +const REEXPORT_MAIN: &str = r#" +import by_reexport_src as src +import by_reexport_middle as middle +import by_reexport_star as star + +assert "VALUE" not in vars(middle), "an unread name is not in the namespace yet" +assert "VALUE" in dir(middle), "dir() lists it" +assert hasattr(middle, "VALUE") and middle.VALUE === src.VALUE, "an attribute read" +assert vars(middle)["VALUE"] === src.VALUE, "which leaves the object in the namespace" +assert not hasattr(middle, "NOT_THERE"), "a name the module never bound is still missing" +assert star.got() === src.VALUE, "a star import of a module with `__all__`" + +from by_reexport_middle import VALUE +assert VALUE === src.VALUE, "a `from` import of it" +print("ok") +"#; + +/// two modules importing each other's names: neither runs the other until a name is read +const CYCLE_A: &str = r#" +from by_cycle_b import pong + +def ping(n: int) -> int: + return 0 if n == 0 else pong(n - 1) +"#; + +const CYCLE_B: &str = r#" +from by_cycle_a import ping + +def pong(n: int) -> int: + return 1 if n == 0 else ping(n - 1) + +PARTNER = ping +"#; + +/// the cycle the other way round: one side's name is read in an annotation, which keeps that +/// import eager, and the other side's import of it is deferred +const CYCLE_C: &str = r#" +from by_cycle_d import Maker + +class Thing: + def maker(self) -> Maker: + return Maker() +"#; + +const CYCLE_D: &str = r#" +from by_cycle_c import Thing + +class Maker: + def thing(self) -> object: + return Thing() +"#; + +const CYCLE_MAIN: &str = r#" +import by_cycle_a as a +import by_cycle_b as b +from by_cycle_a import ping +from by_cycle_c import Thing + +assert ping(3) == 1, "a call crosses the cycle" +assert b.PARTNER === a.ping, "and the name is the other module's object" +assert isinstance(Thing().maker().thing(), Thing), "an eager side and a deferred side" +print("ok") +"#; + +const PATCH_DEPENDENCY: &str = r#" +def fetch() -> str: + return "real" +"#; + +const PATCH_SERVICE: &str = r#" +from by_patch_dep import fetch + +def run() -> str: + return fetch() +"#; + +/// a test patches a name where it is used, in the module that imported it. the patch writes +/// that module's namespace, which is where each of its reads looks first +const PATCH_MAIN: &str = r#" +from unittest import mock +import by_patch_svc as svc + +with mock.patch("by_patch_svc.fetch", lambda: "patched"): + assert svc.run() == "patched", "a patch made before the name was read" +assert svc.run() == "real", "and undone" + +with mock.patch.object(svc, "fetch", lambda: "object"): + assert svc.run() == "object", "a patch made after" +assert svc.run() == "real", "and undone" + +original = svc.fetch +setattr(svc, "fetch", lambda: "set") +assert svc.run() == "set", "what `monkeypatch.setattr` does" +setattr(svc, "fetch", original) +assert svc.run() == "real", "and undoes" print("ok") "#; @@ -105,12 +587,8 @@ made_there: Character = "a" assert made_here == made_there, "str equality holds regardless of class identity" assert isinstance(made_there, Character), "same-module Character" assert isinstance(made_here, Character), "Character identity must survive an import" -# `===` is identity here — in basedpython `is` means isinstance. and identity is -# read off `__class__` rather than `type()`: `made_here` is a lazily-imported -# proxy whose `type()` is `_LazyAttr` by design (a documented limit of the -# < 3.15 polyfill), while `__class__` is both forwarded by the proxy and what -# `isinstance` itself consults -assert made_here.__class__ === made_there.__class__, "one Character class per process" +# `===` is identity here — in basedpython `is` means isinstance +assert type(made_here) === type(made_there), "one Character class per process" assert made_here.__class__.__name__ == "Character", "still named Character" print("ok") "#; @@ -123,7 +601,7 @@ const SUBMODULE_MAIN: &str = r#" from urllib import parse from urllib import by_not_there as missing -assert parse.quote("a b") == "a%20b", "a submodule resolves through the proxy" +assert parse.quote("a b") == "a%20b", "a submodule resolves on first use" assert parse.urlparse("http://h/p").path == "/p", "and keeps working after the first use" try: @@ -200,11 +678,10 @@ print("ok") "#; /// the readers of an annotation, each asserting the answer python gives for the names it -/// finds. a proxy is not the object it stands for, and no forwarded operator makes it one: -/// `dataclasses` tells a `KW_ONLY` annotation from a field by identity, and a type hint is -/// compared by identity by whatever consumes it. so a name an annotation reads is imported -/// eagerly. `InitVar[int]` and `singledispatch`'s registration by annotation are readers -/// the proxy happened to satisfy, and are kept here as readers all the same +/// finds. an annotation is read back from the module's namespace — `typing.get_type_hints` +/// evaluates a string annotation against it — where a deferred name is not until something +/// reads it, and `dataclasses` tells a `KW_ONLY` annotation from a field by identity as the +/// class is made. so a name an annotation reads is imported eagerly const ANNOTATIONS_MAIN: &str = r#" from dataclasses import dataclass, fields, KW_ONLY, InitVar from fractions import Fraction @@ -254,9 +731,9 @@ assert typing.get_type_hints(half)["value"] === fractions.Fraction, "the string print("ok") "#; -/// a module that binds names of builtins the proxy reads when it resolves, formats and -/// iterates. the proxy is pasted into the module and reads the builtins through its -/// globals, so it reads them under names of its own +/// a module that binds names of builtins the polyfill reads when it resolves a name, and +/// the one the module's `_LazyFrom` is made with. the runtime is pasted into the module and +/// reads the builtins through its globals, so it reads them under names of its own const BUILTIN_NAMES_MAIN: &str = r#" from decimal import Decimal from math import pi @@ -271,10 +748,13 @@ def format(value: object, spec: str) -> str: def iter(o: object) -> int: return 0 -assert Decimal(1) + 1 == Decimal(2), "resolving the proxy calls `getattr`" -assert f"{pi:.2f}" == "3.14", "formatting the proxy calls `format`" -assert [c for c in digits][:2] == ["0", "1"], "iterating the proxy calls `iter`" -assert getattr(1, "x") == 42 and format(1, "") == "shadowed" and iter(1) == 0 +def globals() -> int: + return 7 + +assert Decimal(1) + 1 == Decimal(2), "resolving a name calls `getattr`" +assert f"{pi:.2f}" == "3.14", "formatting the value calls `format`" +assert [c for c in digits][:2] == ["0", "1"], "iterating the value calls `iter`" +assert getattr(1, "x") == 42 and format(1, "") == "shadowed" and iter(1) == 0 and globals() == 7 print("ok") "#; @@ -294,11 +774,55 @@ fn build_case(case: &str, sources: &[(&str, &str)]) -> PathBuf { for (stem, source) in sources { let transpiled = transpile(source, &Config::default()).expect("transpile should succeed"); - fs::write(dir.join(format!("{stem}.py")), transpiled).expect("write module"); + let path = dir.join(format!("{stem}.py")); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("create package dir"); + } + fs::write(path, transpiled).expect("write module"); } dir } +/// run `sources` both transpiled and as the python they are written in, asserting the two +/// print the same, and something. the sources are plain python, so python running them is +/// the reference +fn agrees_with_python(python: &Interpreter, case: &str, sources: &[(&str, &str)]) { + let transpiled = build_case(case, sources); + let reference = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{case}_python")); + let _ = fs::remove_dir_all(&reference); + for (stem, source) in sources { + let path = reference.join(format!("{stem}.py")); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("create package dir"); + } + fs::write(path, source).expect("write module"); + } + let run = |dir: &Path| { + let output = Command::new(&python.command) + .arg("main.py") + .current_dir(dir) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "{} failed:\n--- stdout ---\n{}\n--- stderr ---\n{}", + dir.display(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + // the two trees are in directories of their own, which a message naming a file says + String::from_utf8_lossy(&output.stdout).replace(&dir.display().to_string(), "") + }; + let expected = run(&reference); + assert!(!expected.trim().is_empty(), "the reference printed nothing"); + assert_eq!( + run(&transpiled), + expected, + "the transpiled program disagrees with python" + ); + interpreters::ran(python, Config::default().min_version); +} + /// run `main.py` in `dir`, asserting it exits cleanly and prints `ok` fn run_main(python: &Interpreter, dir: &Path) { let output = Command::new(&python.command) @@ -318,7 +842,7 @@ fn run_main(python: &Interpreter, dir: &Path) { } #[test] -fn proxy_forwards_value_operations() { +fn an_imported_value_is_the_object_itself() { let Some(python) = python() else { return; }; @@ -375,7 +899,7 @@ fn an_unlazifiable_name_in_a_multi_name_import_stays_bound() { } #[test] -fn a_submodule_resolves_through_the_proxy() { +fn a_submodule_resolves_on_first_use() { let Some(python) = python() else { return; }; @@ -405,10 +929,234 @@ fn a_string_annotation_resolves_to_the_imported_object() { } #[test] -fn the_proxy_reads_builtins_the_module_binds_for_itself() { +fn the_polyfill_reads_builtins_the_module_binds_for_itself() { let Some(python) = python() else { return; }; let dir = build_case("lazy_builtin_names", &[("main", BUILTIN_NAMES_MAIN)]); run_main(&python, &dir); } + +#[test] +fn a_read_in_every_scope_is_the_object_itself() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_scopes", + &[("by_token_mod", TOKEN_MODULE), ("main", SCOPES_MAIN)], + ); + run_main(&python, &dir); +} + +#[test] +fn a_name_bound_again_reads_as_python_does() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_rebound", + &[("by_token_mod", TOKEN_MODULE), ("main", REBOUND_MAIN)], + ); + run_main(&python, &dir); +} + +#[test] +fn a_read_before_the_import_runs_raises_python_s_name_error() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_early", + &[("by_token_mod", TOKEN_MODULE), ("main", EARLY_MAIN)], + ); + run_main(&python, &dir); +} + +#[test] +fn a_deferred_name_is_passed_on_to_other_modules() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_reexport", + &[ + ("by_reexport_src", REEXPORT_SOURCE), + ("by_reexport_middle", REEXPORT_MIDDLE), + ("by_reexport_listed", REEXPORT_LISTED), + ("by_reexport_star", REEXPORT_STAR), + ("main", REEXPORT_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn modules_importing_each_other_resolve_on_use() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_cycle", + &[ + ("by_cycle_a", CYCLE_A), + ("by_cycle_b", CYCLE_B), + ("by_cycle_c", CYCLE_C), + ("by_cycle_d", CYCLE_D), + ("main", CYCLE_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_patch_of_the_importing_module_reaches_its_reads() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_patch", + &[ + ("by_patch_dep", PATCH_DEPENDENCY), + ("by_patch_svc", PATCH_SERVICE), + ("main", PATCH_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_check_on_a_deferred_class_is_made_once_its_import_has_run() { + let Some(python) = python() else { + return; + }; + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in ONCE_BOUND { + db.write_file(path, source).expect("write file failed"); + } + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("lazy_once_bound"); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + for (path, _) in ONCE_BOUND { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = transpile_typed(&db, file, &Config::default(), None) + .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); + if path.ends_with("main.by") { + // the check this is about, on a name the polyfill defers + assert!( + transpiled.contains("_soundness_once_bound((\"Thing\",)") + && transpiled.contains("lambda: _by_lazy.Thing"), + "{transpiled}" + ); + } + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + run_main(&python, &dir); +} + +/// transpile each of `files` against a project db holding all of them, into `case`'s directory +fn build_project(case: &str, files: &[(&str, &str)]) -> PathBuf { + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(case); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + for (path, _) in files { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = transpile_typed(&db, file, &Config::default(), None) + .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + dir +} + +#[test] +fn a_hoisted_class_s_quoted_field_binds_what_it_reads() { + let Some(python) = python() else { + return; + }; + let dir = build_case("lazy_hoisted_fields", &[("main", HOISTED_FIELDS)]); + run_main(&python, &dir); +} + +#[test] +fn a_quote_in_a_type_argument_of_a_sibling_module_s_generic_binds_what_it_reads() { + let Some(python) = python() else { + return; + }; + let dir = build_project("lazy_quoted_through_a_sibling", QUOTED_THROUGH_A_SIBLING); + let main = fs::read_to_string(dir.join("main.py")).expect("read main"); + assert!(main.contains("Box[\"F | JSONDecoder\"]"), "{main}"); + run_main(&python, &dir); +} + +#[test] +fn a_relative_import_of_the_package_reads_its_attribute_or_its_submodule() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_relative", + &[ + ("by_relative_pkg/__init__", RELATIVE_PACKAGE), + ("by_relative_pkg/other", RELATIVE_SUBMODULE), + ("by_relative_pkg/user", RELATIVE_USER), + ("main", RELATIVE_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_package_importing_its_own_submodule_agrees_with_python() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_init_submodule", INIT_SUBMODULE); +} + +#[test] +fn a_re_export_cycle_is_python_s_import_error() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_reexport_cycle", REEXPORT_CYCLE); +} + +#[test] +fn a_name_whose_import_did_not_run_reads_the_builtin() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_builtin_fallback", BUILTIN_FALLBACK); +} + +#[test] +fn a_reloaded_module_binds_what_its_modules_hold_now() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_reload", RELOAD); +} + +#[test] +fn a_failing_import_reports_python_s_error() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_import_errors", IMPORT_ERRORS); +} diff --git a/crates/by_transforms/tests/literal_type_runtime.rs b/crates/by_transforms/tests/literal_type_runtime.rs new file mode 100644 index 0000000000..08f3671871 --- /dev/null +++ b/crates/by_transforms/tests/literal_type_runtime.rs @@ -0,0 +1,73 @@ +//! Runtime test for a string in a `.by` type expression, which is a literal type wherever a +//! type expression holds it: the checker reads `list["a"]` as a list of the string `"a"` +//! (`basedpython_literal_annotations.md`, "a string in a generic's type argument is a +//! Literal"). The emitted python has to say the same to whatever reads it back, or +//! `typing.get_type_hints` reads the bare string as a forward reference to a name `a` +//! +//! A forward reference the transpiler quotes itself is a string python has to read as a +//! name, so it stays one, with the literal types inside it lowered + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod interpreters; + +/// the shapes the mdtest reveals, read back through `typing` +const PROGRAM: &str = r#" +import typing +from typing import Callable, Literal, get_args, get_type_hints +from enum import Enum + +class Colour(Enum): + Red = 1 + +type Named = dict[str, "y"] +Alias = list["x"] + +def f(a: list["a"], b: dict[str, "k"], c: Callable[["p"], "r"], n: Named, al: Alias) -> None: + pass + +def later(x: list["q"] | Later) -> None: + pass + +class Later: + pass + +hints = get_type_hints(f) +assert get_args(hints["a"]) == (Literal["a"],), hints["a"] +assert get_args(hints["b"]) == (str, Literal["k"]), hints["b"] +assert get_args(hints["c"]) == ([Literal["p"]], Literal["r"]), hints["c"] +assert get_args(Named.__value__) == (str, Literal["y"]), Named.__value__ +assert get_args(Alias) == (Literal["x"],), Alias +quoted = get_type_hints(later)["x"] +assert get_args(quoted) == (list[Literal["q"]], Later), quoted +# a subscript that looks a member up by name holds no type at all +assert Colour["Red"] === Colour.Red +print("ok") +"#; + +/// from 3.12, where `type` is a statement of its own +#[test] +fn a_string_type_argument_reads_back_as_its_literal_type() { + let Some(interpreter) = interpreters::oldest(PythonVersion::PY312, "", "") else { + return; + }; + let config = Config { + min_version: PythonVersion::PY312, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "failed on {interpreter}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, PythonVersion::PY312); +} diff --git a/crates/by_transforms/tests/mutable_defaults_runtime.rs b/crates/by_transforms/tests/mutable_defaults_runtime.rs index 5fbefdc815..1c66cded4a 100644 --- a/crates/by_transforms/tests/mutable_defaults_runtime.rs +++ b/crates/by_transforms/tests/mutable_defaults_runtime.rs @@ -112,6 +112,226 @@ print("ok") /// second line it is an ordinary character python refuses to tokenize const BOM: &str = "\u{feff}\ndef fresh(items = []):\n items.append(1)\n return items\n\nassert fresh() == [1], \"the default is re-evaluated per call\"\n\nprint(\"ok\")\n"; +/// a default written over several lines inside parentheses, which are not part of its +/// range. moved into the body it still needs a grouping, or its second line is a +/// statement of its own: `+ [2]` runs as a unary plus and the default is just `[1]` +const MULTI_LINE: &str = r#" +def joined(xs: list[int] = ( + [1] + + [2] +)) -> list[int]: + return xs + +assert joined() == [1, 2], "the whole default is re-evaluated, not its first line" + +def keyword(*, xs: list[int] = ( + [1] + + [2] +)) -> list[int]: + return xs + +assert keyword() == [1, 2], "so is a keyword-only one" + +# an `init(...)` with no body of its own writes its guards with the body it makes +class Holder: + init(self, let xs: list[int] = ( + [1] + + [2] + )) + +assert Holder().xs == [1, 2], "so is one of a bodyless `init`" + +print("ok") +"#; + +/// a default that names a value stands in the signature as a marker whose `repr` is the +/// default as written — there is no value to put there, because the call reads the name as +/// it runs. handed back, as `apply_defaults` hands it, the marker means the argument was +/// not given +const NAME_DEFAULT: &str = r#" +import inspect + +EARLY = 2 + +def f(x: int = EARLY) -> int: + return x + +signature = inspect.signature(f) +assert repr(signature.parameters["x"].default) == "", repr(signature.parameters["x"].default) +assert str(signature) == "(x: int = ) -> int", str(signature) + +EARLY = 3 +bound = signature.bind() +bound.apply_defaults() +assert f(*bound.args, **bound.kwargs) == 3, "the default handed back is read as the call runs" +assert f() == 3, "and so is one left out" + +print("ok") +"#; + +/// a call is evaluated afresh by every call that leaves its parameter out, and by every +/// call handed the marker +const CALL_DEFAULT: &str = r#" +import inspect + +def make() -> list[int]: + return [] + +def f(items: list[int] = make()) -> list[int]: + items.append(1) + return items + +signature = inspect.signature(f) +assert repr(signature.parameters["items"].default) == "", repr(signature.parameters["items"].default) + +bound = signature.bind() +bound.apply_defaults() +assert f(*bound.args, **bound.kwargs) == [1], "the marker handed back is a fresh call" +assert f(*bound.args, **bound.kwargs) == [1], "every time" + +print("ok") +"#; + +/// a marker copied or pickled — a tool that keeps a signature's defaults may do either — +/// comes back as the very marker, and still means the argument was not given +const COPIED_MARKER: &str = r#" +import copy +import inspect +import pickle + +def f(items: list[int] = []) -> list[int]: + items.append(1) + return items + +default = inspect.signature(f).parameters["items"].default +assert copy.copy(default) === default, "a copy is the marker" +assert copy.deepcopy(default) === default, "so is a deep copy" +assert pickle.loads(pickle.dumps(default)) === default, "and a pickled one" +assert f(**{"items": pickle.loads(pickle.dumps(default))}) == [1], "which is the default" + +print("ok") +"#; + +/// each default has a marker of its own: one handed to another parameter is a value there, +/// as any other object would be +const ANOTHER_DEFAULTS_MARKER: &str = r#" +import inspect + +def f(items: list[int] = []) -> list[int]: + return items + +def g(value: object = []) -> object: + return value + +marker = inspect.signature(f).parameters["items"].default +assert g(**{"value": marker}) === marker, "another default's marker is a value" +assert g() == [], "and the parameter's own default still applies" + +print("ok") +"#; + +/// a marker shows its default on one line, without the comments written in it +const MULTI_LINE_MARKER: &str = r#" +import inspect + +def f(xs: list[int] = [ + 1, # one + 2, +], ys: list[int] = ( + [3] + + [4] +)) -> list[int]: + return xs + ys + +parameters = inspect.signature(f).parameters +assert repr(parameters["xs"].default) == "", repr(parameters["xs"].default) +assert repr(parameters["ys"].default) == "", repr(parameters["ys"].default) +assert f() == [1, 2, 3, 4] + +print("ok") +"#; + +/// a default that reads an earlier parameter has no value until the call that supplies +/// that parameter — the marker is all there is to show +const LATE_BOUND_DEFAULT: &str = r#" +import inspect + +def f(a: int, b: int = a + 1) -> int: + return b + +signature = inspect.signature(f) +assert repr(signature.parameters["b"].default) == "", repr(signature.parameters["b"].default) + +bound = signature.bind(5) +bound.apply_defaults() +assert f(*bound.args, **bound.kwargs) == 6, "the default is computed from the argument given" + +print("ok") +"#; + +/// the constructor an `init(let ...)` declares shows its defaults the same way +const INIT_FIELD_DEFAULT: &str = r#" +import inspect + +EARLY = 2 + +class Base: + init(let geysers: int = EARLY) + +default = inspect.signature(Base.__init__).parameters["geysers"].default +assert repr(default) == "", repr(default) + +signature = inspect.signature(Base) + +bound = signature.bind() +bound.apply_defaults() +assert Base(*bound.args, **bound.kwargs).geysers == 2, "the marker handed back is the default" + +EARLY = 5 +assert Base(*bound.args, **bound.kwargs).geysers == 5, "read as the call runs" +assert Base().geysers == 5, "and so is one left out" + +print("ok") +"#; + +/// a parameter python would reject — required, after a defaulted one — stands in the +/// signature with a marker that says it is required, and handed back it is the missing +/// argument it stands for +const REQUIRED_AFTER_DEFAULT: &str = r#" +import inspect + +def f(x: int = 1, a: int) -> int: + return x + a + +signature = inspect.signature(f) +assert repr(signature.parameters["a"].default) == "", repr(signature.parameters["a"].default) + +bound = signature.bind() +bound.apply_defaults() +try: + f(*bound.args, **bound.kwargs) +except TypeError as error: + assert "missing required argument: 'a'" in str(error), str(error) +else: + raise AssertionError("a required argument handed back as its marker was accepted") + +print("ok") +"#; + +/// a scalar default is python's own, so the signature holds its value +const SCALAR_DEFAULT: &str = r#" +import inspect + +def f(x: int = 3, s: str = "a") -> int: + return x + +parameters = inspect.signature(f).parameters +assert parameters["x"].default == 3, repr(parameters["x"].default) +assert parameters["s"].default == "a", repr(parameters["s"].default) + +print("ok") +"#; + fn run(python: &str, program: &str) { let config = Config { min_version: PythonVersion::PY313, @@ -157,6 +377,21 @@ fn an_extension_members_default_is_hoisted_below_the_preamble() { run(&python, EXTENSION); } +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_default_written_over_several_lines_survives_the_move_into_the_body() { + let Some(python) = python() else { + eprintln!( + "skipping `a_default_written_over_several_lines_survives_the_move_into_the_body`: no `python3` interpreter found" + ); + return; + }; + run(&python, MULTI_LINE); +} + /// both preamble paths have to leave the BOM at offset 0: the sentinel the /// transform emits, and the `from __future__ import annotations` line every /// target below 3.10 gets @@ -197,3 +432,75 @@ fn a_leading_bom_stays_ahead_of_the_preamble() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } } + +#[test] +fn a_name_default_shows_as_written_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, NAME_DEFAULT); +} + +#[test] +fn a_call_default_shows_as_written_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, CALL_DEFAULT); +} + +#[test] +fn a_late_bound_default_shows_as_written_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, LATE_BOUND_DEFAULT); +} + +#[test] +fn an_init_field_default_shows_as_written_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, INIT_FIELD_DEFAULT); +} + +#[test] +fn a_required_parameter_after_a_default_shows_as_required_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, REQUIRED_AFTER_DEFAULT); +} + +#[test] +fn a_scalar_default_shows_its_value_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, SCALAR_DEFAULT); +} + +#[test] +fn a_copied_or_pickled_marker_is_still_the_default() { + let Some(python) = python() else { + return; + }; + run(&python, COPIED_MARKER); +} + +#[test] +fn another_defaults_marker_is_a_value() { + let Some(python) = python() else { + return; + }; + run(&python, ANOTHER_DEFAULTS_MARKER); +} + +#[test] +fn a_marker_shows_its_default_on_one_line() { + let Some(python) = python() else { + return; + }; + run(&python, MULTI_LINE_MARKER); +} diff --git a/crates/by_transforms/tests/none_coalesce_runtime.rs b/crates/by_transforms/tests/none_coalesce_runtime.rs new file mode 100644 index 0000000000..62ac371c01 --- /dev/null +++ b/crates/by_transforms/tests/none_coalesce_runtime.rs @@ -0,0 +1,294 @@ +//! runtime test for the `??` (none-coalesce) lowering +//! +//! `a ?? b` lowers to a conditional expression that evaluates `a` once, binding it to +//! a temporary when it is more than a name, and evaluates `b` only when `a` is `None`. +//! the transform's unit tests assert on the lowered *text*; this test runs it, with +//! the `??` written where the lowering has to compose with what surrounds it: +//! +//! - inside an operand of another `??` — a receiver, an argument, a subscript, an +//! f-string — where it is lowered by an edit of its own +//! - as an operand of `and` / `or`, or the test or value of a conditional +//! expression, where python reads an unparenthesized conditional as something else +//! - where python refuses `:=`: a comprehension's iterable, and any part of a +//! comprehension in a class body +//! +//! no third-party packages are needed, so any `python3` will do; if none is found the +//! test skips rather than fails + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; +use common::python; + +/// a `??` inside an operand of another one +const NESTED: &str = r#" +calls: list[str] = [] + +def probe(tag: str, value: int?) -> int?: + calls.append(tag) + return value + +def lookup(a: dict[int, list[int]]?, k: int) -> list[int]: + return (a ?? {}).get(k) ?? [] + +assert lookup(None, 1) == [] and lookup({1: [2]}, 1) == [2] and lookup({}, 1) == [] + +def right(a: int?, b: int?) -> int: + return a ?? int(str(probe("b", b) ?? 4)) + +assert right(None, None) == 4 and right(1, None) == 1 + +def indexed(xs: list[int]?, i: int?) -> int: + return (xs ?? [9])[i ?? 0] + +assert indexed(None, None) == 9 and indexed([1, 2], 1) == 2 + +def add(a: int, b: int) -> int: + return a + b + +def argument(x: dict[int, int]?, y: int?) -> int: + return add((x ?? {}).get(1) ?? 0, b=y ?? 0) + +assert argument(None, None) == 0 and argument({1: 5}, 2) == 7 + +def formatted(x: int?, d: dict[int, int]?) -> str: + return f"{x ?? 1}-{(d ?? {}).get(1) ?? 2}" + +assert formatted(None, None) == "1-2" + +# a chain evaluates each left operand once, in order, and stops at the first that +# is not `None` +def chained(a: int?, b: int?, c: int?) -> int: + return probe("a", a) ?? probe("b", b) ?? probe("c", c) ?? 0 + +calls.clear() +assert chained(None, 2, 3) == 2 and calls == ["a", "b"] +calls.clear() +assert chained(None, None, None) == 0 and calls == ["a", "b", "c"] + +# a unary operator is a call of its operand's method +class Negated: + def __init__(self): + self.calls = 0 + + def __neg__(self) -> int?: + self.calls += 1 + return 5 + +negated = Negated() +assert (-negated ?? 3) == 5 and negated.calls == 1, "the left operand is evaluated once" + +print("ok") +"#; + +/// a `??` where python reads an unparenthesized conditional expression as something +/// else +const GROUPING: &str = r#" +def body(a: int?, c: bool) -> int: + return a ?? 1 if c else 2 + +assert body(None, True) == 1 +assert body(5, False) == 2, "the condition picks between the `??` and 2" + +def operand(a: int?, b: int) -> int: + return a ?? 0 or b + +assert operand(5, 9) == 5 +assert operand(0, 9) == 9, "`or` applies to the coalesced value" + +def test(a: bool?) -> int: + return 1 if a ?? True else 2 + +assert test(None) == 1 and test(False) == 2 + +def kept(d: dict[int, bool], ks: list[int]) -> list[int]: + return [k for k in ks if d.get(k) ?? True] + +assert kept({1: False}, [1, 2]) == [2] + +def unpacked(a: list[int]?, b: list[int], d: dict[str, int]?) -> tuple[list[int], dict[str, int]]: + return [*a ?? b], {**d ?? {"k": 1}} + +assert unpacked(None, [2], None) == ([2], {"k": 1}) + +print("ok") +"#; + +/// a `??` where python refuses `:=`, and next to it the places it accepts one +const BINDING: &str = r#" +from collections.abc import Callable + +table: dict[int, list[int]] = {1: [2]} +flags: dict[int, bool] = {1: False} +values: dict[int, int?] = {1: 5} + +def iterable(k: int) -> list[int]: + return [x for x in table.get(k) ?? []] + +assert iterable(1) == [2] and iterable(3) == [] + +def nested_iterable(ks: list[int]) -> list[int]: + return [x for k in ks for x in table.get(k) ?? [0]] + +assert nested_iterable([1, 3]) == [2, 0] + +def generator(k: int) -> int: + return sum(x for x in table.get(k) ?? []) + +assert generator(1) == 2 and generator(3) == 0 + +def displays(k: int) -> tuple[set[int], dict[int, int]]: + return {x for x in table.get(k) ?? []}, {x: x for x in table.get(k) ?? [7]} + +assert displays(3) == (set(), {7: 7}) + +def lambda_in_iterable() -> list[int]: + return [x for x in (lambda: table.get(3) ?? [1])()] + +assert lambda_in_iterable() == [1] + +def parts(ks: list[int]) -> list[int]: + return [values.get(k) ?? -1 for k in ks if flags.get(k) ?? True] + +assert parts([1, 2]) == [-1] + +def counted(ks: list[int]) -> int: + return sum(values.get(k) ?? 0 for k in ks) + +assert counted([1, 2]) == 5 + +class Holder: + let direct: int = values.get(2) ?? 3 + let element: list[int] = [values.get(k) ?? 0 for k in [1, 2]] + let condition: list[int] = [k for k in [1, 2] if flags.get(k) ?? True] + let from_iterable: list[int] = [x for x in table.get(9) ?? [direct]] + let in_lambda: list[int] = [(lambda: values.get(k) ?? 0)() for k in [1, 2]] + +assert Holder.direct == 3 +assert Holder.element == [5, 0] +assert Holder.condition == [2] +assert Holder.from_iterable == [3], "the fallback still sees the class body's names" +assert Holder.in_lambda == [5, 0] + +def scaled(n: int) -> Callable[[Callable[[], int]], Callable[[], int]]: + def wrap(fn: Callable[[], int]) -> Callable[[], int]: + return lambda: fn() * n + return wrap + +@scaled(values.get(2) ?? 10) +def decorated(x: int = values.get(2) ?? 1) -> int: + return x + +assert decorated() == 10 + +print("ok") +"#; + +/// a `??` in an annotation and in the scope of a generic definition's type parameters, +/// where python refuses `:=` too +const ANNOTATION_SCOPES: &str = r#" +from typing import Annotated + +table: dict[str, int] = {} + +class Base: + pass + +def base(n: int) -> type[Base]: + return Base + +class Generic[T](base(table.get("a") ?? 0)): + pass + +annotated: Annotated[int, table.get("b") ?? 1] = 1 + +def parameter(y: Annotated[int, table.get("c") ?? 2]): + pass + +type Alias = Annotated[int, table.get("d") ?? 3] + +assert issubclass(Generic, Base) and annotated == 1 + +print("ok") +"#; + +fn run(python: &str, program: &str) { + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + let output = Command::new(python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_coalesce_inside_an_operand_is_lowered() { + let Some(python) = python() else { + eprintln!( + "skipping `a_coalesce_inside_an_operand_is_lowered`: no `python3` interpreter found" + ); + return; + }; + run(&python, NESTED); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_coalesce_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!("skipping `a_coalesce_keeps_its_grouping`: no `python3` interpreter found"); + return; + }; + run(&python, GROUPING); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_coalesce_binds_its_value_where_python_refuses_an_assignment_expression() { + let Some(python) = python() else { + eprintln!( + "skipping `a_coalesce_binds_its_value_where_python_refuses_an_assignment_expression`: no `python3` interpreter found" + ); + return; + }; + run(&python, BINDING); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_coalesce_binds_its_value_in_an_annotation_and_a_type_parameter_scope() { + let Some(python) = python() else { + eprintln!( + "skipping `a_coalesce_binds_its_value_in_an_annotation_and_a_type_parameter_scope`: no `python3` interpreter found" + ); + return; + }; + run(&python, ANNOTATION_SCOPES); +} diff --git a/crates/by_transforms/tests/optional_chain_runtime.rs b/crates/by_transforms/tests/optional_chain_runtime.rs new file mode 100644 index 0000000000..50e3dc95ee --- /dev/null +++ b/crates/by_transforms/tests/optional_chain_runtime.rs @@ -0,0 +1,131 @@ +//! runtime test for the `?.` (optional chain) lowering +//! +//! `a?.b` lowers to a conditional expression that evaluates each receiver once, +//! binding a compound one to a temporary, and stops at the first `None`. the +//! transform's unit tests assert on the lowered *text*; this test runs it, with the +//! chain written where the lowering has to compose with what surrounds it: behind a +//! call on another chain, whose receiver it passes through as source, and where python +//! refuses the `:=` that binds the temporary — a comprehension's iterable, and any +//! part of a comprehension in a class body +//! +//! no third-party packages are needed, so any `python3` will do; if none is found the +//! test skips rather than fails + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; +use common::python; + +const CHAINS: &str = r#" +calls: list[str] = [] + +class Node: + def __init__(self, value: int, items: list[int]): + self.value = value + self.items = items + + def me(self) -> Node?: + calls.append("me") + return self + + def nothing(self) -> Node?: + return None + +def make(present: bool) -> Node?: + calls.append("make") + return Node(1, [1, 2]) if present else None + +# a chain behind a call on another chain +def through_a_call(n: Node?) -> int?: + return n?.me()?.value + +assert through_a_call(Node(3, [])) == 3 and through_a_call(None) is None +assert make(True)?.nothing()?.value is None + +# in a comprehension's iterable +def iterable(present: bool) -> list[int]: + return [x for x in make(present)?.items ?? []] + +calls.clear() +assert iterable(True) == [1, 2] and calls == ["make"], "the receiver is evaluated once" +assert iterable(False) == [] + +def nested_iterable() -> list[int]: + return [x for _ in [0] for x in make(True)?.me()?.items ?? []] + +calls.clear() +assert nested_iterable() == [1, 2] and calls == ["make", "me"] + +def in_a_display() -> list[list[int]?]: + return [y for y in [make(True)?.items, make(False)?.items]] + +assert in_a_display() == [[1, 2], None] + +class Holder: + let element: list[int?] = [make(present)?.value for present in [True, False]] + let condition: list[int] = [k for k in [1, 2] if make(k == 1)?.value] + +assert Holder.element == [1, None] +assert Holder.condition == [1] + +# from a class body comprehension's first iterable, the chain's call still reads a +# builtin and an enclosing function's local, which the comprehension it moves into sees +class Picker: + def __init__(self, items: list[int]): + self.items = items + + def pick(self, f: (list[int]) -> int) -> list[int]: + return [f(self.items)] + +def picker() -> Picker?: + return Picker([4, 5]) + +class Lengths: + let picked: list[int] = [n for n in picker()?.pick(len) ?? []] + +assert Lengths.picked == [2] + +def outer() -> list[int]: + def first(xs: list[int]) -> int: + return xs[0] + class Inner: + let picked: list[int] = [n for n in picker()?.pick(first) ?? []] + return Inner.picked + +assert outer() == [4] + +print("ok") +"#; + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn optional_chains_run_correctly() { + let Some(python) = python() else { + eprintln!("skipping optional chain runtime test: no `python3` interpreter found"); + return; + }; + + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(CHAINS, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled optional chain program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/by_transforms/tests/parametric_is_runtime.rs b/crates/by_transforms/tests/parametric_is_runtime.rs index 669147ee48..a82a1c7e71 100644 --- a/crates/by_transforms/tests/parametric_is_runtime.rs +++ b/crates/by_transforms/tests/parametric_is_runtime.rs @@ -142,6 +142,26 @@ assert (FeedBool() is Feed[int]) is False, "param contravariant: bool does not a assert (FeedBool() is Feed[bool]) is True, "param matches bool" assert (C() is Feed[bool]) is False, "value without the method fails" +# a default re-evaluated per call has no value before the call, so nothing at runtime +# records the type the checker reads off it — the marker standing in the signature +# is not that type. the check refuses to answer rather than answer from the marker +class FeedLate: + def f(self, other = C()): + pass + +def refused(test) -> bool: + try: + test() + except TypeError as error: + return "per call" in str(error) + return False + +def feeds_bool(value: object) -> bool: + return value is Feed[bool] + +assert refused(lambda: FeedLate() is Feed[C]), "the check refuses the type the default would give" +assert refused(lambda: feeds_bool(FeedLate())), "and one it would not" + # an impl with an extra *required* parameter the protocol doesn't supply can't # satisfy it — a protocol-shaped call would fail to provide that argument class ExtraRequired: @@ -150,6 +170,57 @@ class ExtraRequired: assert (ExtraRequired() is Feed[bool]) is False, "extra required parameter fails" +# so does a required one after a default, which python cannot declare and which stands +# in the signature as a marker in place of a default +class ExtraRequiredAfterDefault: + def f(self, other = True, extra): + pass + +assert feeds_bool(ExtraRequiredAfterDefault()) is False, "a required parameter after a default fails" + +# a required parameter after a default the protocol does supply is required, not a +# default evaluated per call: an unannotated one says nothing about its type, so the +# check answers `False` rather than refusing +class FeedSecond[in out T](Protocol): + def f(self, a: int, other: T): ... + +class ReqAfterDefault: + def f(self, a = 1, other): + pass + +def feeds_second_bool(value: object) -> bool: + return value is FeedSecond[bool] + +assert feeds_second_bool(ReqAfterDefault()) is False, "a required parameter is not a late-bound default" + +# a class of the program's own may carry every attribute a marker has, and a default +# that is one of its instances is still a value, read by its type: a marker is only the +# object its module binds under the name it carries. written in plain python, so the +# default stands in the signature as written +_plain = {"__name__": "__main__"} +exec(""" +class Tagged: + _by_marker = True + _by_name = "Tagged" + +class Forged: + _by_name = "Forged" + _by_source = None + +class TakesTagged: + def f(self, other=Tagged()): + pass + +class TakesForged: + def f(self, other: bool = True, extra=Forged()): + pass +""", _plain) +Tagged = _plain["Tagged"] +Forged = _plain["Forged"] + +assert feeds_bool(_plain["TakesTagged"]()) is False, "a value with a marker's attributes is a value" +assert feeds_bool(_plain["TakesForged"]()) is True, "and a default, not a required parameter's marker" + # a method return is covariant: an impl returning `bool` satisfies a protocol # asking for `int` (bool <: int), but not one asking for `str` class Get[in out T](Protocol): @@ -444,6 +515,199 @@ assert len(calls) == 1, "an effectful value is evaluated once" print("ok") "#; +/// basedpython whose module-level `assert`s run what the transpiler writes out from the +/// checker's spelling of a type that holds a `float`. `float` in basedpython is just `float`, +/// not python's `int | float`, so a type test's probe compares the arguments a `list[float]` +/// subclass records against `float`, and a reified type parameter is handed `float` itself +const JUST_FLOAT_PROGRAM: &str = r#" +# a type test names the alias, so the probe is rebuilt from the alias's value +type Floats = list[float] + +class FloatList(list[float]): ... +class IntList(list[int]): ... + +def floats(v: object) -> bool: + return v is Floats + +assert floats(FloatList()) is True, "a list subclass of float is a list[float]" +assert floats(IntList()) is False, "a list subclass of int is not" +assert (FloatList() cast? Floats) is not None, "a checked cast accepts a list subclass of float" +assert (IntList() cast? Floats) is None, "and refuses a list subclass of int" + +# a reified type parameter is handed `float` itself +class Tagged[reified T]: + def __init__(self, t: T): + self.t = t + + def argument(self) -> object: + return T + +def argument[reified T](x: T) -> object: + return T + +def from_variable(y: float) -> object: + return argument(y) + +def from_floats(ys: list[float]) -> object: + return argument(ys) + +def from_complex(c: complex) -> object: + return argument(c) + +class Made: ... + +assert argument(1.5) === float, f"a float literal reifies as {argument(1.5)}" +assert from_variable(1.5) === float, f"a float variable reifies as {from_variable(1.5)}" +assert from_floats([1.5]) == list[float], f"a list of floats reifies as {from_floats([1.5])}" +assert from_complex(1j) === complex, f"a complex variable reifies as {from_complex(1j)}" +# a constructed value is exactly a `Made`, and the argument is the class +assert argument(Made()) === Made, f"a constructed value reifies as {argument(Made())}" +assert Tagged(1.5).argument() === float, f"a float literal specializes as {Tagged(1.5).argument()}" + +print("ok") +"#; + +/// basedpython whose module-level `assert`s run checked casts and type tests against targets +/// another lowering rewrites where they are written — `float`, `int?`, a literal, a tuple +/// type, `typeof`, a callable type, `dynamic` and the rest. the check is built from the +/// checker's reading of the target, so it tests what the target means rather than how it +/// is spelled; and where the operand's type cannot hold the target, the checker settles the +/// check without the target at all +const LOWERED_TARGETS_PROGRAM: &str = r#" +y = 1 + +def raises(f: () -> object) -> bool: + try: + f() + except TypeError: + return True + return False + +# the operand's type cannot hold the target, so the checker settles the cast without the +# target: it never succeeds +def int_to_float(x: int) -> float: + return x cast! float + +def int_to_float_or_none(x: int) -> float?: + return x cast? float + +assert raises(lambda: int_to_float(3)), "an int is not a float" +assert int_to_float_or_none(3) is None, "an int is not a float" + +def str_to_one(x: str) -> object: + return x cast? 1 + +def str_to_pair(x: str) -> object: + return x cast? (int, str) + +def str_to_named(x: str) -> object: + return x cast? (a: int, b: str) + +def str_to_inf(x: str) -> object: + return x cast? float.inf + +def str_to_floats(x: str) -> object: + return x cast? tuple[float, ...] + +assert str_to_one("1") is None +assert str_to_pair("a") is None +assert str_to_named("a") is None +assert str_to_inf("inf") is None +assert str_to_floats("a") is None + +# the operand may hold the target, so the check is built from the checker's reading of it +def to_float(x: object) -> float: + return x cast! float + +assert to_float(3.5) == 3.5 +assert raises(lambda: to_float(3)), "an int is not a float" + +def to_optional(x: object) -> int?: + return x cast! int? + +assert to_optional(None) is None +assert to_optional(1) == 1 +assert raises(lambda: to_optional("a")) + +class FloatList(list[float]): ... +class IntList(list[int]): ... + +def to_pair(x: object) -> object: + return x cast? (int, str) + +def to_floats(x: object) -> object: + return x cast? list[float] + +def to_y(x: object) -> object: + return x cast? typeof y + +def to_y_plus_one(x: object) -> object: + return x cast? typeof y + 1 + +def to_float_class(x: object) -> object: + return x cast? type[float] + +def to_pattern(x: object) -> object: + return x cast? f"a{int}b" + +def to_dynamic_list(x: object) -> object: + return x cast? list[dynamic] + +def to_callbacks(x: object) -> object: + return x cast? list[(int) -> str] + +assert to_pair((1, "a")) is not None +assert to_pair([1, "a"]) is None +assert to_floats(FloatList()) is not None +assert to_floats(IntList()) is None +assert to_y(1) == 1 +assert to_y(2) is None +assert to_y(True) is None +assert to_y_plus_one(2) == 2 +assert to_y_plus_one(1) is None +assert to_float_class(float) === float +assert to_float_class(int) is None +assert to_pattern("a1b") == "a1b" +assert to_pattern("a1c") is None +assert to_dynamic_list([1]) == [1] +assert to_dynamic_list((1,)) is None +assert to_callbacks([str]) is not None +assert to_callbacks("s") is None + +# a type test is built the same way, and `float` is just `float` there too +def is_float(v: object) -> bool: + return v is float + +assert is_float(3.5) is True +assert is_float(3) is False, "an int is not a float" + +def is_floats(v: object) -> bool: + return v is list[float] + +def is_y(v: object) -> bool: + return v is typeof y + +def is_callbacks(v: object) -> bool: + return v is list[(int) -> str] + +def is_dynamic_list(v: object) -> bool: + return v is list[dynamic] + +def str_is_named(v: str) -> bool: + return v is (a: int, b: str) + +assert is_floats(FloatList()) is True +assert is_floats(IntList()) is False +assert is_y(1) is True +assert is_y(2) is False +assert is_callbacks([]) is False +assert is_dynamic_list([1]) is True +assert is_dynamic_list((1,)) is False +assert str_is_named("a") is False + +print("ok") +"#; + const PEP695_PROBE: &str = "type X[T] = T"; /// pep 696 defaults in native syntax plus the `has_default()` accessor the probe @@ -538,3 +802,30 @@ fn type_param_defaults_resolve_at_runtime() { }; run_program(&python, DEFAULTS_PROGRAM); } + +/// a reified class needs 3.13, so this is probed like [`DEFAULTS_PROGRAM`] +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_check_written_from_a_float_type_checks_just_float() { + let Some(python) = python_supporting(PEP696_PROBE) else { + eprintln!("skipping just-float runtime test: no 3.13+ interpreter found"); + return; + }; + run_program(&python, JUST_FLOAT_PROGRAM); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_target_another_lowering_rewrites_checks_correctly_at_runtime() { + let Some(python) = python_supporting(PEP695_PROBE) else { + eprintln!("skipping lowered-target runtime test: no PEP 695-capable interpreter found"); + return; + }; + run_program(&python, LOWERED_TARGETS_PROGRAM); +} diff --git a/crates/by_transforms/tests/statement_expression_runtime.rs b/crates/by_transforms/tests/statement_expression_runtime.rs new file mode 100644 index 0000000000..71bb3f93fa --- /dev/null +++ b/crates/by_transforms/tests/statement_expression_runtime.rs @@ -0,0 +1,199 @@ +//! runtime test for the early exits a statement expression lowers to, and the one +//! `^` lowers to +//! +//! `x = value or return`, `x = value ?? raise E()` and `x = a if c else raise E()` +//! lower to statements: each operand is evaluated into a temporary on a line of its +//! own, and the diverging branch becomes the body of an `if`. `x = value^` evaluates +//! its operand into a temporary the same way, ahead of a guard that returns it when +//! it is absent. the transform's unit +//! tests assert on the lowered *text*; this test runs it. an operand the source wrote +//! over several lines, inside parentheses that are not part of its own range, has to +//! keep a grouping when it moves onto its own line — without one, its second line +//! either fails to parse or parses as a statement of its own, and the program quietly +//! computes something else +//! +//! no third-party packages are needed, so any `python3` will do; if none is found the +//! test skips rather than fails + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; +use common::python; + +/// operands written over several lines, one per early-exit form +const MULTI_LINE_OPERANDS: &str = r#" +class Units: + def __init__(self, n: int): + self.n = n + + def of_type(self, *types: int) -> Units: + return self + + def excluding(self, n: int) -> Units: + return Units(self.n - n) + + def __bool__(self) -> bool: + return self.n > 0 + +# a parenthesised method chain, one link per line +def army(units: Units, types: list[int]) -> int: + var chosen = ( + units.of_type(*types) + .excluding(1) + .excluding(1) + ) or return -1 + return chosen.n + +assert army(Units(5), [1]) == 3, "a present chain is the value" +assert army(Units(2), [1]) == -1, "an empty chain returns" + +# a continuation line that reads as a statement on its own: `+ 1` is a unary plus +def doubled(a: int) -> int: + x = ( + a + + 1 + ) or return 0 + return x * 2 + +assert doubled(1) == 4, "the whole operand is the value, not its first line" +assert doubled(-1) == 0 + +# `and return` takes the same path +def zero(a: int) -> str: + x = ( + a + - 1 + ) and return "nonzero" + return f"zero {x}" + +assert zero(1) == "zero 0" and zero(3) == "nonzero" + +# `or raise` +def size(xs: list[int]) -> int: + n = (len(xs) + + 0) or raise ValueError("empty") + return n + +assert size([1, 2]) == 2 +try: + _ = size([]) +except ValueError: + pass +else: + raise AssertionError("an empty list raises") + +# `?? raise` with an operand that is a chain of attribute reads +class Node: + def __init__(self, value: int?): + self.value = value + self.inner = self + +def value_of(node: Node) -> int: + v = ( + node + .inner + .value + ) ?? raise KeyError("absent") + return v + +assert value_of(Node(4)) == 4 +try: + _ = value_of(Node(None)) +except KeyError: + pass +else: + raise AssertionError("an absent value raises") + +# a conditional expression whose test and value both span lines +def positive(a: int) -> int: + x = ( + a + + 1 + ) if ( + a + > 0 + ) else raise ValueError(a) + return x + +assert positive(1) == 2, "the value is the whole operand" +try: + _ = positive(-1) +except ValueError: + pass +else: + raise AssertionError("a failed test raises") + +print("ok") +"#; + +/// a `^` operand written over several lines +const PROPAGATE_MULTI_LINE: &str = r#" +class Node: + def __init__(self, value: int?): + self.value = value + self.inner = self + +def incremented(node: Node) -> int?: + x = ( + node + .inner + .value + )^ + return x + 1 + +assert incremented(Node(3)) == 4 +assert incremented(Node(None)) is None + +print("ok") +"#; + +fn run(python: &str, program: &str) { + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + let output = Command::new(python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn an_operand_written_over_several_lines_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!("skipping statement expression runtime test: no `python3` interpreter found"); + return; + }; + run(&python, MULTI_LINE_OPERANDS); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_propagated_operand_written_over_several_lines_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!( + "skipping `a_propagated_operand_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" + ); + return; + }; + run(&python, PROPAGATE_MULTI_LINE); +} diff --git a/crates/by_transforms/tests/stored_function_runtime.rs b/crates/by_transforms/tests/stored_function_runtime.rs new file mode 100644 index 0000000000..5d737ab96c --- /dev/null +++ b/crates/by_transforms/tests/stored_function_runtime.rs @@ -0,0 +1,86 @@ +//! Runtime test for a `.by` function read back out of where it was stored. +//! +//! A `.by` function can carry what a call has to act on — a `context` parameter the call fills, +//! reified type parameters the call specializes by subscript. The checker keeps the function's +//! own type when a list, a dict, a tuple or an attribute stores it, and the transpiler reads that +//! type to write the call: the filled `context` argument is only in the output because the +//! checker still sees the function. This test runs the result to see both arrive. + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod interpreters; + +const STORED: &str = r#" +def greet(a: int, context b: str) -> str: + return f"{b} {a}" + +def name[reified T]() -> str: + return T.__name__ + +class Holder: + def __init__(self): + self.greet = greet + self.name = name + +context salutation = "hello" + +greeters = [greet] +greeter_by_key = {"a": greet} +greeter_pair = (greet, 1) +held = Holder() +appended_greeters = [] +appended_greeters.append(greet) + +assert greeters[0](1) == "hello 1", greeters[0](1) +assert greeter_by_key["a"](2) == "hello 2", greeter_by_key["a"](2) +assert greeter_pair[0](3) == "hello 3", greeter_pair[0](3) +assert held.greet(4) == "hello 4", held.greet(4) +assert appended_greeters[0](5) == "hello 5", appended_greeters[0](5) + +names = [name] +name_by_key = {"a": name} +name_pair = (name, 1) +appended_names = [] +appended_names.append(name) + +assert names[0][int]() == "int", names[0][int]() +assert name_by_key["a"][str]() == "str", name_by_key["a"][str]() +assert name_pair[0][bytes]() == "bytes", name_pair[0][bytes]() +assert held.name[float]() == "float", held.name[float]() +assert appended_names[0][int]() == "int", appended_names[0][int]() + +print("ok") +"#; + +#[test] +fn a_stored_function_is_called_with_what_it_carries() { + // a reified generic is a pep 695 function. `oldest` says why when it skips + let Some(interpreter) = interpreters::oldest(PythonVersion::PY312, "", "") else { + return; + }; + let target = interpreter.version.min(PythonVersion::PY313); + let config = Config { + min_version: target, + ..Config::default() + }; + // `transpile` analyses the module with the checker, which is what fills the `context` + // arguments + let transpiled = transpile(STORED, &config).expect("transpile should succeed"); + + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "the transpiled program failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} diff --git a/crates/by_transforms/tests/type_checking_import_runtime.rs b/crates/by_transforms/tests/type_checking_import_runtime.rs new file mode 100644 index 0000000000..3b0cd2d0b3 --- /dev/null +++ b/crates/by_transforms/tests/type_checking_import_runtime.rs @@ -0,0 +1,376 @@ +//! Runtime test for a class imported only under `if TYPE_CHECKING:`, or bound only further +//! down the module. +//! +//! The checker resolves such a name, and the running program does not have it where the +//! lowered code reads it — under `TYPE_CHECKING` that is how an import cycle is broken. A +//! lowering that writes a type into code that runs (a specialization the checker solved, a +//! soundness check's target, a cast's type) must not write that name, or the module raises +//! `NameError` with `by check` clean, unless it looks the name up as it runs. + +mod common; + +use std::fs; +use std::path::Path; +use std::process::Command; + +use by_transforms::{Config, PythonVersion, SoundnessPositions, transpile, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; + +const PROGRAM: &str = r#" +from typing import TYPE_CHECKING +from decimal import Decimal as RuntimeDecimal + +if TYPE_CHECKING: + from decimal import Decimal + + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + kept: set[Decimal] = set() + assert kept == set() + + +def first(values: list[Decimal]) -> Decimal: + for value in values: + return value + return RuntimeDecimal(0) + + +def widen(value: Decimal) -> Decimal | int: + return value cast Decimal | int + + +totals: set[Decimal] = set() +ledger = Ledger() +ledger.entries = set() +assert ledger.entries == set() and totals == set() +assert first([RuntimeDecimal(1)]) == RuntimeDecimal(1) +assert widen(RuntimeDecimal(2)) == RuntimeDecimal(2) +print("ok") +"#; + +/// the same reads where the name is bound only under `TYPE_CHECKING` imported under another +/// name, only on a path the program does not take, or only further down the module +const ELSEWHERE_PROGRAM: &str = r#" +import os +from typing import TYPE_CHECKING as CHECKING +from decimal import Decimal as RuntimeDecimal + +if CHECKING: + from decimal import Decimal + +if os.environ.get("BASEDPYTHON_NEVER_SET"): + from fractions import Fraction + + +def first(values: list[Decimal]) -> Decimal: + for value in values: + return value + return RuntimeDecimal(0) + + +def first_fraction(values: list[Fraction]) -> object: + for value in values: + return value + return None + + +early: set[Later] = set() +seen: set[Fraction] = set() + + +class Later: ... + + +class Ledger: + entries: set[Decimal] + fractions: set[Fraction] + + def __init__(self): + self.entries = set() + self.fractions = set() + + +assert first([RuntimeDecimal(1)]) == RuntimeDecimal(1) +assert first_fraction([]) is None +assert early == set() and seen == set() +assert Ledger().entries == set() and Ledger().fractions == set() +print("ok") +"#; + +/// the same reads where the name is bound past a condition that reads `TYPE_CHECKING` in some +/// other way than as the whole test of an `if`, or where its value is chosen by the constant +const CONDITION_PROGRAM: &str = r#" +import typing +from typing import TYPE_CHECKING +from decimal import Decimal as RuntimeDecimal +from fractions import Fraction as RuntimeFraction + +match TYPE_CHECKING: + case True: + from decimal import Decimal as InMatch + case _: + pass + +if TYPE_CHECKING == True: + from decimal import Decimal as InComparison + +if bool(TYPE_CHECKING): + from decimal import Decimal as InCall + +if bool(typing.TYPE_CHECKING): + from decimal import Decimal as InAttribute + +if checking := TYPE_CHECKING: + from decimal import Decimal as InWalrus + +if checking: + from decimal import Decimal as InWalrusAlias + +while TYPE_CHECKING: + from decimal import Decimal as InWhile + break + +RUNNING = not TYPE_CHECKING +if not RUNNING: + from decimal import Decimal as InNegatedAlias + +combined = False +combined |= TYPE_CHECKING +if combined: + from decimal import Decimal as InAugmented + +Fraction = RuntimeFraction if TYPE_CHECKING else None + +_ = (InConditional := RuntimeDecimal) if TYPE_CHECKING else None + + +def parse(text: str) -> dynamic: + return RuntimeFraction(text) + + +class Ledger: + in_match: set[InMatch] + in_comparison: set[InComparison] + in_call: set[InCall] + in_attribute: set[InAttribute] + in_walrus: set[InWalrus] + in_walrus_alias: set[InWalrusAlias] + in_augmented: set[InAugmented] + in_while: set[InWhile] + in_negated_alias: set[InNegatedAlias] + in_conditional: set[InConditional] + + def __init__(self): + self.in_match = set() + self.in_comparison = set() + self.in_call = set() + self.in_attribute = set() + self.in_walrus = set() + self.in_walrus_alias = set() + self.in_augmented = set() + self.in_while = set() + self.in_negated_alias = set() + self.in_conditional = set() + in_ternary: Fraction = parse("1") + assert in_ternary == RuntimeFraction(1) + + +price: Fraction = parse("1/2") +assert price == RuntimeFraction(1, 2) +ledger = Ledger() +assert ledger.in_match == ledger.in_negated_alias == set() +print("ok") +"#; + +/// a check in a function on a class the module declares further down: the function can run +/// before the class exists, when the value passes unchecked, and after, when it is checked +const LATER_PROGRAM: &str = r#" +def loose(value: object) -> dynamic: + return value + + +def first(values: list[dynamic]) -> Later: + return values[0] + + +def make(value: object) -> Kept: + return loose(value) + + +assert first([1]) == 1 + + +class Later: + def __init__(self, size: int): + self.size = size + + +private class Kept: ... + + +class Holder: + later: list[Later] + + def __init__(self, later: list[dynamic]): + self.later = later + + def total(self) -> int: + total = 0 + for later in self.later: + total += later.size + return total + + +def refused(run) -> bool: + try: + run() + except TypeError: + return True + return False + + +assert Holder([Later(1), Later(2)]).total() == 3 +assert refused(lambda: Holder([1]).total()) +assert refused(lambda: first([1])) +assert first([Later(3)]).size == 3 +assert refused(lambda: make(1)) +assert make(Kept()) is not None +print("ok") +"#; + +#[test] +fn a_class_imported_only_for_the_checker_is_never_evaluated() { + run(PROGRAM); +} + +#[test] +fn a_name_bound_elsewhere_than_where_it_is_read_is_never_evaluated() { + run(ELSEWHERE_PROGRAM); +} + +#[test] +fn a_name_bound_past_a_condition_reading_type_checking_is_never_evaluated() { + run(CONDITION_PROGRAM); +} + +#[test] +fn a_class_declared_further_down_is_checked_once_it_is_bound() { + run(LATER_PROGRAM); +} + +/// a module importing a name another module binds one way for the checker and another way as +/// it runs reads what the program binds, which need not be the class the checker resolved +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_name_imported_from_a_type_checking_block_elsewhere_is_never_evaluated() { + let Some(python) = common::python() else { + eprintln!("skipping type checking import runtime test: no interpreter found"); + return; + }; + let files = [ + ( + "/kinds.by", + "\ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal +else: + Decimal = float + + +def parse(text: str) -> dynamic: + import decimal + + return decimal.Decimal(text) +", + ), + ( + "/main.by", + "\ +from kinds import Decimal, parse + + +def total() -> Decimal: + return parse(\"1.5\") + + +assert str(total()) == \"1.5\" +print(\"ok\") +", + ), + ]; + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("type_checking_import_elsewhere"); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + let config = Config { + min_version: PythonVersion::PY313, + soundness: SoundnessPositions::all(), + ..Config::default() + }; + for (path, _) in files { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = + transpile_typed(&db, file, &config, None).expect("transpile should succeed"); + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + let output = Command::new(&python) + .arg("main.py") + .current_dir(&dir) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- main.py ---\n{}", + String::from_utf8_lossy(&output.stderr), + fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run(program: &str) { + let Some(python) = common::python() else { + eprintln!("skipping type checking import runtime test: no interpreter found"); + return; + }; + let config = Config { + min_version: PythonVersion::PY313, + soundness: SoundnessPositions::all(), + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/by_transforms/tests/visibility_runtime.rs b/crates/by_transforms/tests/visibility_runtime.rs index 91699ac668..cafe837a3f 100644 --- a/crates/by_transforms/tests/visibility_runtime.rs +++ b/crates/by_transforms/tests/visibility_runtime.rs @@ -469,3 +469,282 @@ fn a_renamed_symbol_never_takes_a_name_the_module_already_has() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); interpreters::ran(&interpreter, target); } + +/// a module-level `private` class named by the checks a lowering writes from a type rather +/// than from the source: the soundness checks on a loop, an assignment, a return, an +/// argument and a projection — through a container, through a private type alias, from a +/// method of another class, and against a private generic's specialization, by a public +/// argument and by a private one +const SOUNDNESS_CHECKS: &str = r#" +from typing import Any + +private class Kept: + init(let n: int) + +private class Box[T]: + init(let v: T) + +private type Alias = Kept + +private var kept: list[Kept] = [Kept(1), Kept(2)] + +def dynamic(value: object) -> Any: + return value + +def looped() -> int: + var sum = 0 + for one in kept: + sum += one.n + return sum + +def assigned() -> int: + k: Kept = dynamic(Kept(3)) + return k.n + +def returned() -> Kept: + return dynamic(Kept(4)) + +def takes(k: Kept) -> int: + return k.n + +def argument() -> int: + return takes(dynamic(Kept(5))) + +def projected() -> int: + nested: dict[str, list[Kept]] = {"a": [Kept(6)]} + return nested["a"][0].n + +def aliased() -> int: + a: Alias = dynamic(Kept(7)) + return a.n + +class Reader: + def read(self) -> int: + var sum = 0 + for one in kept: + sum += one.n + return sum + +def boxed() -> int: + boxes: list[Box[int]] = [Box[int](8)] + var sum = 0 + for box in boxes: + sum += box.v + return sum + +def kept_boxes() -> int: + boxes: list[Box[Kept]] = [Box[Kept](Kept(9))] + var sum = 0 + for box in boxes: + sum += box.v.n + return sum + +def results() -> list[int]: + return [looped(), assigned(), returned().n, argument(), projected(), aliased(), Reader().read(), boxed(), kept_boxes()] +"#; + +const SOUNDNESS_CHECKS_IMPORTER: &str = r#" +import importlib +m = importlib.import_module("emitted") +assert m.results() == [3, 3, 4, 5, 6, 7, 3, 8, 9], m.results() +m.dynamic = lambda value: "not kept" +try: + m.returned() +except TypeError as error: + assert "_Kept" in str(error), error +else: + raise AssertionError("the return check let a `str` through") +assert not hasattr(m, "Kept") and not hasattr(m, "Box") +print("ok") +"#; + +#[test] +fn a_soundness_check_names_a_private_class_by_its_renamed_binding() { + // the program declares a generic class with pep 695 syntax + let Some(interpreter) = interpreters::oldest(PythonVersion::PY312, "", "") else { + return; + }; + let target = interpreter.version.min(PythonVersion::PY313); + let config = Config { + min_version: target, + ..Config::default() + }; + let transpiled = transpile(SOUNDNESS_CHECKS, &config).expect("transpile should succeed"); + + let dir = tempfile::tempdir().expect("failed to create a temp dir"); + std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); + + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(SOUNDNESS_CHECKS_IMPORTER) + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "importing the transpiled module failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} + +/// a module-level `private` class named by the other lowerings that write a type out at +/// runtime: an `is` type test against a class, a generic, a protocol's member and an enum +/// member, the checked casts, the specialization a reified generic and an erased union are +/// called with, the guard a `raises` clause is checked by, and the annotation inferred for +/// a class attribute, which python evaluates as the class body runs +const RUNTIME_TYPES: &str = r#" +from enum import Enum +from typing import Protocol + +private class Kept: + init(let n: int) + +private class Box[T]: + init(let v: T) + +private class Oops(Exception): + pass + +class Holder[T]: + init(let v: T) + + def kind(self) -> object: + return T + +def kind_of[T](value: T) -> object: + return T + +def is_kept(value: object) -> bool: + return value is Kept + +def is_int_box(value: object) -> bool: + return value is Box[int] + +private class HasKept(Protocol): + k: Kept + + def get(self, other: Kept) -> Kept: ... + +class Shaped: + k: Kept = Kept(6) + + def get(self, other: Kept) -> Kept: + return other + +def has_kept(value: object) -> bool: + return value is HasKept + +private class Color(Enum): + RED = 1 + BLUE = 2 + +def is_red(value: object) -> bool: + return value is Color.RED + +def count(values: list[Kept] | list[int]) -> int: + return len(values) + +def counted() -> int: + return count([Kept(7), Kept(8)]) + +def checked(value: object) -> int: + return (value cast! Kept).n + +def tried(value: object) -> Kept?: + return value cast? Kept + +def held() -> object: + return Holder(Kept(1)).kind() + +def called() -> object: + return kind_of(Kept(2)) + +def fails() -> int raises Oops: + raise Oops() + +def rethrow[reified E: Exception](error: E) raises E | Oops: + raise error + +def made() -> Kept: + return Kept(5) + +class Shelf: + var item = made() +"#; + +const RUNTIME_TYPES_IMPORTER: &str = r#" +import importlib +m = importlib.import_module("emitted") +kept = m._Kept(3) +assert m.is_kept(kept) and not m.is_kept(3) +assert m.is_int_box(m._Box[int](4)) and not m.is_int_box(m._Box[str]("4")) +assert m.has_kept(m.Shaped()) and not m.has_kept(object()) +assert m.is_red(m._Color.RED) and not m.is_red(m._Color.BLUE) +assert m.counted() == 2 +assert m.checked(kept) == 3 +try: + m.checked(3) +except TypeError: + pass +else: + raise AssertionError("`cast!` let an `int` through") +assert m.tried(kept) is kept and m.tried(3) is None +assert m.held() is m._Kept and m.called() is m._Kept +try: + m.fails() +except m._Oops: + pass +# the guard reads `E` off the specialization, and tests `(E, _Oops)` rather than the +# ceiling `Exception`, which would let the `KeyError` through +for raised in (ValueError("v"), m._Oops()): + try: + m.rethrow[ValueError](raised) + except type(raised): + pass +try: + m.rethrow[ValueError](KeyError("k")) +except AssertionError as error: + assert "rethrow raised KeyError" in str(error), error +else: + raise AssertionError("the `raises` guard let a `KeyError` through") +assert m.Shelf.__annotations__["item"] is m._Kept +print("ok") +"#; + +#[test] +fn a_type_written_out_at_runtime_names_a_private_class_by_its_renamed_binding() { + // the program declares a generic class with pep 695 syntax + let Some(interpreter) = interpreters::oldest(PythonVersion::PY312, "", "") else { + return; + }; + let target = interpreter.version.min(PythonVersion::PY313); + let config = Config { + min_version: target, + runtime_raises_checks: true, + ..Config::default() + }; + let transpiled = transpile(RUNTIME_TYPES, &config).expect("transpile should succeed"); + + let dir = tempfile::tempdir().expect("failed to create a temp dir"); + std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); + + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(RUNTIME_TYPES_IMPORTER) + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "importing the transpiled module failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} diff --git a/crates/ruff/tests/cli/snapshots/cli__lint__output_format_preview_sarif.snap b/crates/ruff/tests/cli/snapshots/cli__lint__output_format_preview_sarif.snap index f246ff25e2..dc14b90e0e 100644 --- a/crates/ruff/tests/cli/snapshots/cli__lint__output_format_preview_sarif.snap +++ b/crates/ruff/tests/cli/snapshots/cli__lint__output_format_preview_sarif.snap @@ -1,5 +1,6 @@ --- source: crates/ruff/tests/cli/lint.rs +assertion_line: 4424 info: program: buff args: @@ -139,7 +140,7 @@ exit_code: 1 }, { "fullDescription": { - "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" + "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## basedpython\na `.by` or `.byi` file is not checked. an import there can be used without its\nname ever being read: the `extension`s and conformances a module declares are\nonly applicable in the files that import it, and whether one of them supplies\nsomething a file relies on is a question about types, which a linter checking one\nfile at a time cannot answer. `by check` reports unused imports in `.by` files\ninstead, with this rule's semantics, as `unused-import`\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" }, "help": { "text": "`{name}` imported but unused; consider using `importlib.util.find_spec` to test for availability" diff --git a/crates/ruff/tests/cli/snapshots/cli__lint__output_format_sarif.snap b/crates/ruff/tests/cli/snapshots/cli__lint__output_format_sarif.snap index 2db1001143..3381523f0a 100644 --- a/crates/ruff/tests/cli/snapshots/cli__lint__output_format_sarif.snap +++ b/crates/ruff/tests/cli/snapshots/cli__lint__output_format_sarif.snap @@ -1,6 +1,6 @@ --- source: crates/ruff/tests/cli/lint.rs -assertion_line: 3663 +assertion_line: 4375 info: program: buff args: @@ -120,7 +120,7 @@ exit_code: 1 "rules": [ { "fullDescription": { - "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" + "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## basedpython\na `.by` or `.byi` file is not checked. an import there can be used without its\nname ever being read: the `extension`s and conformances a module declares are\nonly applicable in the files that import it, and whether one of them supplies\nsomething a file relies on is a question about types, which a linter checking one\nfile at a time cannot answer. `by check` reports unused imports in `.by` files\ninstead, with this rule's semantics, as `unused-import`\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" }, "help": { "text": "`{name}` imported but unused; consider using `importlib.util.find_spec` to test for availability" diff --git a/crates/ruff/tests/snapshots/integration_test__rule_f401.snap b/crates/ruff/tests/snapshots/integration_test__rule_f401.snap index 7eba551c4a..bdd939cb8c 100644 --- a/crates/ruff/tests/snapshots/integration_test__rule_f401.snap +++ b/crates/ruff/tests/snapshots/integration_test__rule_f401.snap @@ -1,7 +1,8 @@ --- source: crates/ruff/tests/integration_test.rs +assertion_line: 1009 info: - program: ruff + program: buff args: - rule - F401 @@ -123,6 +124,14 @@ else: print("numpy is not installed") ``` +## basedpython +a `.by` or `.byi` file is not checked. an import there can be used without its +name ever being read: the `extension`s and conformances a module declares are +only applicable in the files that import it, and whether one of them supplies +something a file relies on is a question about types, which a linter checking one +file at a time cannot answer. `by check` reports unused imports in `.by` files +instead, with this rule's semantics, as `unused-import` + ## Options - `lint.ignore-init-module-imports` - `lint.pyflakes.allowed-unused-imports` diff --git a/crates/ruff/tests/snapshots/integration_test__rule_f401_output_json.snap b/crates/ruff/tests/snapshots/integration_test__rule_f401_output_json.snap index 9795a7a608..5832be996d 100644 --- a/crates/ruff/tests/snapshots/integration_test__rule_f401_output_json.snap +++ b/crates/ruff/tests/snapshots/integration_test__rule_f401_output_json.snap @@ -1,7 +1,8 @@ --- source: crates/ruff/tests/integration_test.rs +assertion_line: 1040 info: - program: ruff + program: buff args: - rule - F401 @@ -23,7 +24,7 @@ exit_code: 0 ], "fix": "Fix is sometimes available.", "fix_availability": "Sometimes", - "explanation": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n", + "explanation": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## basedpython\na `.by` or `.byi` file is not checked. an import there can be used without its\nname ever being read: the `extension`s and conformances a module declares are\nonly applicable in the files that import it, and whether one of them supplies\nsomething a file relies on is a question about types, which a linter checking one\nfile at a time cannot answer. `by check` reports unused imports in `.by` files\ninstead, with this rule's semantics, as `unused-import`\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n", "preview": false, "status": { "Stable": { @@ -33,7 +34,7 @@ exit_code: 0 "category": "suspicious", "source_location": { "file": "", - "line": 145 + "line": 153 } } ----- stderr ----- diff --git a/crates/ruff/tests/snapshots/integration_test__rule_f401_output_text.snap b/crates/ruff/tests/snapshots/integration_test__rule_f401_output_text.snap index fa85f29d1e..e10cd64515 100644 --- a/crates/ruff/tests/snapshots/integration_test__rule_f401_output_text.snap +++ b/crates/ruff/tests/snapshots/integration_test__rule_f401_output_text.snap @@ -1,7 +1,8 @@ --- source: crates/ruff/tests/integration_test.rs +assertion_line: 1046 info: - program: ruff + program: buff args: - rule - F401 @@ -125,6 +126,14 @@ else: print("numpy is not installed") ``` +## basedpython +a `.by` or `.byi` file is not checked. an import there can be used without its +name ever being read: the `extension`s and conformances a module declares are +only applicable in the files that import it, and whether one of them supplies +something a file relies on is a question about types, which a linter checking one +file at a time cannot answer. `by check` reports unused imports in `.by` files +instead, with this rule's semantics, as `unused-import` + ## Options - `lint.ignore-init-module-imports` - `lint.pyflakes.allowed-unused-imports` diff --git a/crates/ruff_benchmark/benches/ty.rs b/crates/ruff_benchmark/benches/ty.rs index 937c172cdc..be5c1aee51 100644 --- a/crates/ruff_benchmark/benches/ty.rs +++ b/crates/ruff_benchmark/benches/ty.rs @@ -548,6 +548,38 @@ fn benchmark_large_enum_membership(criterion: &mut Criterion) { }); } +/// basedpython: regression benchmark for calling a method on a large enum's `name` +/// +/// `name` is a union of one string literal per member, so `kind.name.lower` is a union of as many +/// bound methods, one per name. none of them is redundant with another, and building that union +/// should not have to compare every pair of them to find that out +fn benchmark_enum_name_method(criterion: &mut Criterion) { + const NUM_ENUM_MEMBERS: usize = 512; + + setup_rayon(); + + let mut code = "from enum import IntEnum\n\nclass E(IntEnum):\n".to_string(); + for i in 0..NUM_ENUM_MEMBERS { + writeln!(&mut code, " m{i} = {i}").ok(); + } + code.push_str( + "\ndef lower(kind: E) -> str:\n return kind.name.lower()\n\n\ + def starts(kind: E) -> bool:\n return kind.name.startswith(\"m\")\n", + ); + + criterion.bench_function("ty_micro[enum_name_method]", |b| { + b.iter_batched_ref( + || setup_micro_case(&code), + |case| { + let Case { db } = case; + let result = db.check(); + assert_eq!(result.len(), 0); + }, + BatchSize::SmallInput, + ); + }); +} + fn benchmark_enum_comparison(criterion: &mut Criterion, name: &str, code: &str) { setup_rayon(); @@ -1909,6 +1941,7 @@ criterion_group!( benchmark_complex_constrained_attributes_3, benchmark_gradual_vararg_call, benchmark_large_enum_membership, + benchmark_enum_name_method, benchmark_many_enum_members, benchmark_narrowed_str_enum_comparison, benchmark_optional_str_enum_comparison, diff --git a/crates/ruff_db/src/panic.rs b/crates/ruff_db/src/panic.rs index 6d12e5bd0f..fb8de34328 100644 --- a/crates/ruff_db/src/panic.rs +++ b/crates/ruff_db/src/panic.rs @@ -35,12 +35,34 @@ impl std::fmt::Display for Payload { } } +/// What was being done to a file when it panicked, as a panic diagnostic says it +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PanicActivity { + Checking, + Transpiling, +} + +impl std::fmt::Display for PanicActivity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + PanicActivity::Checking => "checking", + PanicActivity::Transpiling => "transpiling", + }) + } +} + impl PanicError { pub fn resume_unwind(self) -> ! { std::panic::resume_unwind(self.payload.0) } - pub fn to_diagnostic_message(&self, path: Option) -> String { + /// The message a panic diagnostic carries, saying what was being done to `path` + /// when it panicked + pub fn to_diagnostic_message( + &self, + activity: PanicActivity, + path: Option, + ) -> String { use std::fmt::Write; let mut message = String::new(); @@ -51,7 +73,7 @@ impl PanicError { } if let Some(path) = path { - let _ = write!(&mut message, " when checking `{path}`"); + let _ = write!(&mut message, " when {activity} `{path}`"); } let _ = write!(&mut message, ": `{payload}`", payload = self.payload); diff --git a/crates/ruff_db/src/vendored.rs b/crates/ruff_db/src/vendored.rs index eafd696acd..bd817e130d 100644 --- a/crates/ruff_db/src/vendored.rs +++ b/crates/ruff_db/src/vendored.rs @@ -135,10 +135,13 @@ impl VendoredFileSystem { &self, dir: impl AsRef, ) -> impl Iterator + '_ { - let directory_prefix = NormalizedVendoredPath::from(dir.as_ref()) - .with_trailing_slash() - .0 - .into_owned(); + let normalized = NormalizedVendoredPath::from(dir.as_ref()); + // the archive's root has no name of its own: every entry is under it + let directory_prefix = if normalized.as_str().is_empty() { + String::new() + } else { + normalized.with_trailing_slash().0.into_owned() + }; self.inner.0.file_names().filter_map(move |name| { // Any entry that doesn't have the `path` (with a @@ -650,6 +653,19 @@ pub(crate) mod tests { "); } + #[test] + fn read_directory_root() { + let mock_typeshed = mock_typeshed(); + + for root in ["", ".", "./"] { + assert_eq!( + readdir_snapshot(&mock_typeshed, root), + "vendored://stdlib/", + "{root:?}" + ); + } + } + #[test] fn read_directory_asyncio() { let mock_typeshed = mock_typeshed(); diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F401_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F401_basedpython.by index 3ae6e5258e..3dc883cf95 100644 --- a/crates/ruff_linter/resources/test/fixtures/pyflakes/F401_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F401_basedpython.by @@ -1,32 +1,10 @@ -# `from x export y` is basedpython's explicit re-export spelling, so an -# exported name is deliberately public and never "imported but unused" — -# the same exemption `from x import y as y` gets -from a export exported_name -from b import y as y - -# a plain import of an unreferenced name is still reported +# F401 does not check a `.by` file. whether an import there is used can depend on the +# `extension`s and conformances its module declares, which only types tell, so +# `by check` reports an unused import instead, as `unused-import` +import json from c import unused_name - -# and an exported name may of course also be used locally -from d export used_name - -print(used_name) - -# an `extension` header references both the type it names and the interfaces it -# declares conformance to, so neither import is unused -from e import extended_type, implemented_type, interface_type +from e import extended_type extension extended_type: def extra(self) -> int: return 0 - -extension implemented_type(interface_type): - def m(self) -> int: - return 1 - -# a `raises` clause names exception types, so the import that supplies them is -# used — deleting it would leave a name the type checker cannot resolve -from f import RaisedError - -def may_fail() raises RaisedError: - raise RaisedError diff --git a/crates/ruff_linter/src/fix/edits.rs b/crates/ruff_linter/src/fix/edits.rs index d8ee983fce..90ac7c5a9c 100644 --- a/crates/ruff_linter/src/fix/edits.rs +++ b/crates/ruff_linter/src/fix/edits.rs @@ -5,15 +5,13 @@ use anyhow::{Context, Result}; use ruff_python_ast::AnyNodeRef; use ruff_python_ast::name::Name; use ruff_python_ast::token::{self, Tokens, parenthesized_range}; -use ruff_python_ast::{self as ast, Arguments, ExceptHandler, Expr, ExprList, Parameters, Stmt}; +use ruff_python_ast::{self as ast, Arguments, Expr, ExprList, Parameters, Stmt}; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; use ruff_python_semantic::SemanticModel; use ruff_python_trivia::textwrap::dedent_to; -use ruff_python_trivia::{ - PythonWhitespace, SimpleTokenKind, SimpleTokenizer, has_leading_content, is_python_whitespace, -}; -use ruff_source_file::{LineRanges, NewlineWithTrailingNewline, UniversalNewlines}; +use ruff_python_trivia::{PythonWhitespace, SimpleTokenKind, SimpleTokenizer}; +use ruff_source_file::{LineRanges, UniversalNewlines}; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use crate::Edit; @@ -23,44 +21,14 @@ use crate::fix::codemods; use crate::fix::codemods::CodegenStylist; use crate::line_width::{IndentWidth, LineLength, LineWidthBuilder}; -/// Return the [`Edit`] to use when deleting a [`Stmt`]. -/// -/// In some cases, this is as simple as deleting the [`TextRange`] of the [`Stmt`] -/// itself. However, there are a few exceptions: -/// - If the [`Stmt`] is _not_ the terminal statement in a multi-statement line, -/// we need to delete up to the start of the next statement (and avoid -/// deleting any content that precedes the statement). -/// - If the [`Stmt`] is the terminal statement in a multi-statement line, we need -/// to avoid deleting any content that precedes the statement. -/// - If the [`Stmt`] has no trailing and leading content, then it's convenient to -/// remove the entire start and end lines. -/// - If the [`Stmt`] is the last statement in its parent body, replace it with a -/// `pass` instead. +/// Return the [`Edit`] to use when deleting a [`Stmt`]. See [`ruff_python_edits::delete_stmt`]. pub(crate) fn delete_stmt( stmt: &Stmt, parent: Option<&Stmt>, locator: &Locator, indexer: &Indexer, ) -> Edit { - if parent.is_some_and(|parent| is_lone_child(stmt, parent)) { - // If removing this node would lead to an invalid syntax tree, replace - // it with a `pass`. - Edit::range_replacement("pass".to_string(), stmt.range()) - } else { - if let Some(semicolon) = trailing_semicolon(stmt.end(), locator) { - let next = next_stmt_break(semicolon, locator); - Edit::deletion(stmt.start(), next) - } else if has_leading_content(stmt.start(), locator.contents()) { - Edit::range_deletion(stmt.range()) - } else if let Some(start) = - indexer.preceded_by_continuations(stmt.start(), locator.contents()) - { - Edit::deletion(start, stmt.end()) - } else { - let range = locator.full_lines_range(stmt.range()); - Edit::range_deletion(range) - } - } + ruff_python_edits::delete_stmt(stmt, parent, locator.contents(), indexer) } /// Generate a [`Edit`] to delete a comment (for example: a `noqa` directive). @@ -475,113 +443,6 @@ pub(crate) fn adjust_indentation( Ok(module_text) } -/// Determine if a vector contains only one, specific element. -fn is_only(vec: &[T], value: &T) -> bool { - vec.len() == 1 && vec[0] == *value -} - -/// Determine if a child is the only statement in its body. -fn is_lone_child(child: &Stmt, parent: &Stmt) -> bool { - match parent { - Stmt::FunctionDef(ast::StmtFunctionDef { body, .. }) - | Stmt::ClassDef(ast::StmtClassDef { body, .. }) - | Stmt::With(ast::StmtWith { body, .. }) - if is_only(body, child) => - { - return true; - } - Stmt::For(ast::StmtFor { body, orelse, .. }) - | Stmt::While(ast::StmtWhile { body, orelse, .. }) - if (is_only(body, child) || is_only(orelse, child)) => - { - return true; - } - Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) if (is_only(body, child) - || elif_else_clauses - .iter() - .any(|ast::ElifElseClause { body, .. }| is_only(body, child))) => - { - return true; - } - Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) if (is_only(body, child) - || is_only(orelse, child) - || is_only(finalbody, child) - || handlers.iter().any(|handler| match handler { - ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) => { - is_only(body, child) - } - })) => - { - return true; - } - Stmt::Match(ast::StmtMatch { cases, .. }) - if cases.iter().any(|case| is_only(&case.body, child)) => - { - return true; - } - _ => {} - } - false -} - -/// Return the location of a trailing semicolon following a `Stmt`, if it's part -/// of a multi-statement line. -fn trailing_semicolon(offset: TextSize, locator: &Locator) -> Option { - let contents = locator.after(offset); - - for line in NewlineWithTrailingNewline::from(contents) { - let trimmed = line.trim_whitespace_start(); - - if trimmed.starts_with(';') { - let colon_offset = line.text_len() - trimmed.text_len(); - return Some(offset + line.start() + colon_offset); - } - - if !trimmed.starts_with('\\') { - break; - } - } - None -} - -/// Find the next valid break for a `Stmt` after a semicolon. -fn next_stmt_break(semicolon: TextSize, locator: &Locator) -> TextSize { - let start_location = semicolon + TextSize::from(1); - - for line in - NewlineWithTrailingNewline::with_offset(locator.after(start_location), start_location) - { - let trimmed = line.trim_whitespace(); - // Skip past any continuations. - if trimmed.starts_with('\\') { - continue; - } - - return if trimmed.is_empty() { - // If the line is empty, then despite the previous statement ending in a - // semicolon, we know that it's not a multi-statement line. - line.start() - } else { - // Otherwise, find the start of the next statement. (Or, anything that isn't - // whitespace.) - let relative_offset = line.find(|c: char| !is_python_whitespace(c)).unwrap(); - line.start() + TextSize::try_from(relative_offset).unwrap() - }; - } - - locator.line_end(start_location) -} - /// Add leading whitespace to a snippet, if it's immediately preceded an identifier or keyword. pub(crate) fn pad_start(mut content: String, start: TextSize, locator: &Locator) -> String { // Ex) When converting `except(ValueError,)` from a tuple to a single argument, we need to @@ -675,12 +536,10 @@ mod tests { use ruff_python_ast::Stmt; use ruff_python_codegen::Stylist; use ruff_python_parser::{parse_expression, parse_module}; - use ruff_text_size::{Ranged, TextRange, TextSize}; + use ruff_text_size::{TextRange, TextSize}; use crate::fix::apply_fixes; - use crate::fix::edits::{ - add_to_dunder_all, make_redundant_alias, next_stmt_break, trailing_semicolon, - }; + use crate::fix::edits::{add_to_dunder_all, make_redundant_alias}; use crate::{Edit, Fix, Locator, Violation}; /// Parse the given source using [`Mode::Module`] and return the first statement. @@ -689,72 +548,6 @@ mod tests { Ok(suite.into_iter().next().unwrap()) } - #[test] - fn find_semicolon() -> Result<()> { - let contents = "x = 1"; - let stmt = parse_first_stmt(contents)?; - let locator = Locator::new(contents); - assert_eq!(trailing_semicolon(stmt.end(), &locator), None); - - let contents = "x = 1; y = 1"; - let stmt = parse_first_stmt(contents)?; - let locator = Locator::new(contents); - assert_eq!( - trailing_semicolon(stmt.end(), &locator), - Some(TextSize::from(5)) - ); - - let contents = "x = 1 ; y = 1"; - let stmt = parse_first_stmt(contents)?; - let locator = Locator::new(contents); - assert_eq!( - trailing_semicolon(stmt.end(), &locator), - Some(TextSize::from(6)) - ); - - let contents = r" -x = 1 \ - ; y = 1 -" - .trim(); - let stmt = parse_first_stmt(contents)?; - let locator = Locator::new(contents); - assert_eq!( - trailing_semicolon(stmt.end(), &locator), - Some(TextSize::from(10)) - ); - - Ok(()) - } - - #[test] - fn find_next_stmt_break() { - let contents = "x = 1; y = 1"; - let locator = Locator::new(contents); - assert_eq!( - next_stmt_break(TextSize::from(4), &locator), - TextSize::from(5) - ); - - let contents = "x = 1 ; y = 1"; - let locator = Locator::new(contents); - assert_eq!( - next_stmt_break(TextSize::from(5), &locator), - TextSize::from(6) - ); - - let contents = r" -x = 1 \ - ; y = 1 -" - .trim(); - let locator = Locator::new(contents); - assert_eq!( - next_stmt_break(TextSize::from(10), &locator), - TextSize::from(12) - ); - } - #[test] fn redundant_alias() -> Result<()> { let contents = "import x, y as y, z as bees"; diff --git a/crates/ruff_linter/src/message/mod.rs b/crates/ruff_linter/src/message/mod.rs index 55cb3e6ad4..290c0a874d 100644 --- a/crates/ruff_linter/src/message/mod.rs +++ b/crates/ruff_linter/src/message/mod.rs @@ -3,7 +3,7 @@ use std::fmt::Display; use std::io::Write; use std::path::Path; -use ruff_db::panic::PanicError; +use ruff_db::panic::{PanicActivity, PanicError}; use rustc_hash::FxHashMap; use ruff_db::diagnostic::{ @@ -31,7 +31,10 @@ pub fn create_panic_diagnostic(error: &PanicError, path: Option<&Path>) -> Diagn let mut diagnostic = Diagnostic::new( DiagnosticId::Panic, Severity::Fatal, - error.to_diagnostic_message(path.as_ref().map(|path| path.display())), + error.to_diagnostic_message( + PanicActivity::Checking, + path.as_ref().map(|path| path.display()), + ), ); diagnostic.sub(SubDiagnostic::new( diff --git a/crates/ruff_linter/src/message/snapshots/ruff_linter__message__sarif__tests__results.snap b/crates/ruff_linter/src/message/snapshots/ruff_linter__message__sarif__tests__results.snap index c7365b7af2..34acd5e898 100644 --- a/crates/ruff_linter/src/message/snapshots/ruff_linter__message__sarif__tests__results.snap +++ b/crates/ruff_linter/src/message/snapshots/ruff_linter__message__sarif__tests__results.snap @@ -129,7 +129,7 @@ expression: value "rules": [ { "fullDescription": { - "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" + "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## basedpython\na `.by` or `.byi` file is not checked. an import there can be used without its\nname ever being read: the `extension`s and conformances a module declares are\nonly applicable in the files that import it, and whether one of them supplies\nsomething a file relies on is a question about types, which a linter checking one\nfile at a time cannot answer. `by check` reports unused imports in `.by` files\ninstead, with this rule's semantics, as `unused-import`\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" }, "help": { "text": "`{name}` imported but unused; consider using `importlib.util.find_spec` to test for availability" diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/unused_import.rs b/crates/ruff_linter/src/rules/pyflakes/rules/unused_import.rs index cd03b14cc2..8696ceac28 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/unused_import.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/unused_import.rs @@ -132,6 +132,14 @@ use crate::{Applicability, Fix, FixAvailability, Violation}; /// print("numpy is not installed") /// ``` /// +/// ## basedpython +/// a `.by` or `.byi` file is not checked. an import there can be used without its +/// name ever being read: the `extension`s and conformances a module declares are +/// only applicable in the files that import it, and whether one of them supplies +/// something a file relies on is a question about types, which a linter checking one +/// file at a time cannot answer. `by check` reports unused imports in `.by` files +/// instead, with this rule's semantics, as `unused-import` +/// /// ## Options /// - `lint.ignore-init-module-imports` /// - `lint.pyflakes.allowed-unused-imports` @@ -326,6 +334,13 @@ fn find_dunder_all_exprs<'a>(semantic: &'a SemanticModel) -> Vec<&'a ast::Expr> /// ¬__init__.py ∧ 3rdpty → safe, remove /// pub(crate) fn unused_import(checker: &Checker, scope: &Scope) { + // basedpython: the checker's `unused-import` owns this rule for `.by` files, because an + // import used only through the extensions its module declares reads no name this + // per-file, typeless analysis could see + if checker.source_type.is_basedpython() { + return; + } + // Collect all unused imports by statement. let mut unused: BTreeMap<(NodeId, Exceptions), Vec> = BTreeMap::default(); let mut ignored: BTreeMap<(NodeId, Exceptions), Vec> = BTreeMap::default(); diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-import_F401_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-import_F401_basedpython.by.snap index 57f888e5fc..d0b409f39e 100644 --- a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-import_F401_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-import_F401_basedpython.by.snap @@ -1,18 +1,4 @@ --- source: crates/ruff_linter/src/rules/pyflakes/mod.rs --- -F401 [*] `c.unused_name` imported but unused - --> F401_basedpython.by:8:15 - | - 7 | # a plain import of an unreferenced name is still reported - 8 | from c import unused_name - | ^^^^^^^^^^^ - 9 | -10 | # and an exported name may of course also be used locally - | -help: Remove unused import: `c.unused_name` - | -7 | # a plain import of an unreferenced name is still reported - - from c import unused_name -8 | - | + diff --git a/crates/ruff_python_edits/Cargo.toml b/crates/ruff_python_edits/Cargo.toml index 046045452f..da700dc768 100644 --- a/crates/ruff_python_edits/Cargo.toml +++ b/crates/ruff_python_edits/Cargo.toml @@ -11,11 +11,15 @@ repository = { workspace = true } license = { workspace = true } [dependencies] +ruff_diagnostics = { workspace = true } ruff_python_ast = { workspace = true } +ruff_python_index = { workspace = true } +ruff_python_trivia = { workspace = true } ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } ruff_python_parser = { workspace = true } [lints] diff --git a/crates/ruff_python_edits/src/delete.rs b/crates/ruff_python_edits/src/delete.rs new file mode 100644 index 0000000000..038bad57e6 --- /dev/null +++ b/crates/ruff_python_edits/src/delete.rs @@ -0,0 +1,225 @@ +//! deleting a statement without leaving the source around it invalid + +use ruff_diagnostics::Edit; +use ruff_python_ast::{self as ast, ExceptHandler, Stmt}; +use ruff_python_index::Indexer; +use ruff_python_trivia::{PythonWhitespace, has_leading_content, is_python_whitespace}; +use ruff_source_file::{LineRanges, NewlineWithTrailingNewline}; +use ruff_text_size::{Ranged, TextLen, TextSize}; + +/// Return the [`Edit`] to use when deleting a [`Stmt`]. +/// +/// In some cases, this is as simple as deleting the +/// [`TextRange`](ruff_text_size::TextRange) of the [`Stmt`] itself. However, there are a +/// few exceptions: +/// - If the [`Stmt`] is _not_ the terminal statement in a multi-statement line, +/// we need to delete up to the start of the next statement (and avoid +/// deleting any content that precedes the statement). +/// - If the [`Stmt`] is the terminal statement in a multi-statement line, we need +/// to avoid deleting any content that precedes the statement. +/// - If the [`Stmt`] has no trailing and leading content, then it's convenient to +/// remove the entire start and end lines. +/// - If the [`Stmt`] is the last statement in its parent body, replace it with a +/// `pass` instead. +pub fn delete_stmt(stmt: &Stmt, parent: Option<&Stmt>, source: &str, indexer: &Indexer) -> Edit { + if parent.is_some_and(|parent| is_lone_child(stmt, parent)) { + // If removing this node would lead to an invalid syntax tree, replace + // it with a `pass`. + Edit::range_replacement("pass".to_string(), stmt.range()) + } else { + if let Some(semicolon) = trailing_semicolon(stmt.end(), source) { + let next = next_stmt_break(semicolon, source); + Edit::deletion(stmt.start(), next) + } else if has_leading_content(stmt.start(), source) { + Edit::range_deletion(stmt.range()) + } else if let Some(start) = indexer.preceded_by_continuations(stmt.start(), source) { + Edit::deletion(start, stmt.end()) + } else { + let range = source.full_lines_range(stmt.range()); + Edit::range_deletion(range) + } + } +} + +/// Determine if a vector contains only one, specific element. +fn is_only(vec: &[T], value: &T) -> bool { + vec.len() == 1 && vec[0] == *value +} + +/// Determine if a child is the only statement in its body. +fn is_lone_child(child: &Stmt, parent: &Stmt) -> bool { + match parent { + Stmt::FunctionDef(ast::StmtFunctionDef { body, .. }) + | Stmt::ClassDef(ast::StmtClassDef { body, .. }) + | Stmt::With(ast::StmtWith { body, .. }) + if is_only(body, child) => + { + return true; + } + Stmt::For(ast::StmtFor { body, orelse, .. }) + | Stmt::While(ast::StmtWhile { body, orelse, .. }) + if (is_only(body, child) || is_only(orelse, child)) => + { + return true; + } + Stmt::If(ast::StmtIf { + body, + elif_else_clauses, + .. + }) if (is_only(body, child) + || elif_else_clauses + .iter() + .any(|ast::ElifElseClause { body, .. }| is_only(body, child))) => + { + return true; + } + Stmt::Try(ast::StmtTry { + body, + handlers, + orelse, + finalbody, + .. + }) if (is_only(body, child) + || is_only(orelse, child) + || is_only(finalbody, child) + || handlers.iter().any(|handler| match handler { + ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) => { + is_only(body, child) + } + })) => + { + return true; + } + Stmt::Match(ast::StmtMatch { cases, .. }) + if cases.iter().any(|case| is_only(&case.body, child)) => + { + return true; + } + _ => {} + } + false +} + +/// Return the location of a trailing semicolon following a `Stmt`, if it's part +/// of a multi-statement line. +fn trailing_semicolon(offset: TextSize, source: &str) -> Option { + let contents = &source[offset.to_usize()..]; + + for line in NewlineWithTrailingNewline::from(contents) { + let trimmed = line.trim_whitespace_start(); + + if trimmed.starts_with(';') { + let colon_offset = line.text_len() - trimmed.text_len(); + return Some(offset + line.start() + colon_offset); + } + + if !trimmed.starts_with('\\') { + break; + } + } + None +} + +/// Find the next valid break for a `Stmt` after a semicolon. +fn next_stmt_break(semicolon: TextSize, source: &str) -> TextSize { + let start_location = semicolon + TextSize::from(1); + + for line in NewlineWithTrailingNewline::with_offset( + &source[start_location.to_usize()..], + start_location, + ) { + let trimmed = line.trim_whitespace(); + // Skip past any continuations. + if trimmed.starts_with('\\') { + continue; + } + + return if trimmed.is_empty() { + // If the line is empty, then despite the previous statement ending in a + // semicolon, we know that it's not a multi-statement line. + line.start() + } else { + // Otherwise, find the start of the next statement. (Or, anything that isn't + // whitespace.) + let relative_offset = line.find(|c: char| !is_python_whitespace(c)).unwrap(); + line.start() + TextSize::try_from(relative_offset).unwrap() + }; + } + + source.line_end(start_location) +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use ruff_python_ast::Stmt; + use ruff_python_parser::parse_module; + use ruff_text_size::{Ranged, TextSize}; + + use super::{next_stmt_break, trailing_semicolon}; + + /// Parse the given source using [`Mode::Module`] and return the first statement. + fn parse_first_stmt(source: &str) -> Result { + let suite = parse_module(source)?.into_suite(); + Ok(suite.into_iter().next().unwrap()) + } + + #[test] + fn find_semicolon() -> Result<()> { + let contents = "x = 1"; + let stmt = parse_first_stmt(contents)?; + assert_eq!(trailing_semicolon(stmt.end(), contents), None); + + let contents = "x = 1; y = 1"; + let stmt = parse_first_stmt(contents)?; + assert_eq!( + trailing_semicolon(stmt.end(), contents), + Some(TextSize::from(5)) + ); + + let contents = "x = 1 ; y = 1"; + let stmt = parse_first_stmt(contents)?; + assert_eq!( + trailing_semicolon(stmt.end(), contents), + Some(TextSize::from(6)) + ); + + let contents = r" +x = 1 \ + ; y = 1 +" + .trim(); + let stmt = parse_first_stmt(contents)?; + assert_eq!( + trailing_semicolon(stmt.end(), contents), + Some(TextSize::from(10)) + ); + + Ok(()) + } + + #[test] + fn find_next_stmt_break() { + let contents = "x = 1; y = 1"; + assert_eq!( + next_stmt_break(TextSize::from(4), contents), + TextSize::from(5) + ); + + let contents = "x = 1 ; y = 1"; + assert_eq!( + next_stmt_break(TextSize::from(5), contents), + TextSize::from(6) + ); + + let contents = r" +x = 1 \ + ; y = 1 +" + .trim(); + assert_eq!( + next_stmt_break(TextSize::from(10), contents), + TextSize::from(12) + ); + } +} diff --git a/crates/ruff_python_edits/src/lib.rs b/crates/ruff_python_edits/src/lib.rs index 26b50a9e5a..19fc06358e 100644 --- a/crates/ruff_python_edits/src/lib.rs +++ b/crates/ruff_python_edits/src/lib.rs @@ -3,5 +3,7 @@ Utilities for constructing edits to Python source code. */ pub use self::call::unwrapped_call_argument; +pub use self::delete::delete_stmt; mod call; +mod delete; diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/basedpython_stub_surface.byi b/crates/ruff_python_formatter/resources/test/fixtures/ruff/basedpython_stub_surface.byi index f851b714b1..3528de38da 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/basedpython_stub_surface.byi +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/basedpython_stub_surface.byi @@ -29,6 +29,15 @@ class Socket: let match: str var case: bytes +# it is also the name a declaration keyword introduces +protocol match: + def go(self) -> int + +newtype lazy = int + +enum class case: + case A + # the vendored typeshed spells an optional with the postfix marker, in every # position a stub writes one class Cursor: diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/extension.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/extension.by index 22ff5877ad..31e52e7881 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/extension.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/extension.by @@ -13,6 +13,12 @@ extension list[Element: int]: return sum(self) +# a soft keyword names the extended type like any other name +extension type: + def short_name(self) -> str: + return self.__name__[:2] + + # computed property extension str: @property diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/init_method.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/init_method.by index 9b66296126..0d57c3b558 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/init_method.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/init_method.by @@ -20,6 +20,10 @@ class PlainParams: init(self, a: int, let b: int) +class SoftKeywordNames: + init(let cast: int, var match: int, private let type: int, local lazy: int) + + class DefinitionModifiers: private init(self, let a: int) diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by index debd81db43..e6691036b9 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by @@ -22,10 +22,19 @@ def with_context(local fn: () -> None, context c: int) -> None: ... def not_modifiers(local, once) -> None: ... +# a soft keyword after the modifiers is the parameter's name +def soft_keyword_names(local type: int, once match: () -> None, context cast: int) -> None: ... + + # keyword-only, variadic and defaulted parameters def positions(local a: int, *, once cb: () -> None = print) -> None: ... +# in a callable type the modifier borrows the parameter it precedes, which may be +# named by a soft keyword +def borrows(make: (local type[int], once str) -> int) -> int: ... + + # lambdas take the same modifiers handler = lambda local fn: fn diff --git a/crates/ruff_python_formatter/tests/snapshots/format@basedpython_stub_surface.byi.snap b/crates/ruff_python_formatter/tests/snapshots/format@basedpython_stub_surface.byi.snap index 12fb4e902e..1256174d06 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@basedpython_stub_surface.byi.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@basedpython_stub_surface.byi.snap @@ -35,6 +35,15 @@ class Socket: let match: str var case: bytes +# it is also the name a declaration keyword introduces +protocol match: + def go(self) -> int + +newtype lazy = int + +enum class case: + case A + # the vendored typeshed spells an optional with the postfix marker, in every # position a stub writes one class Cursor: @@ -77,6 +86,15 @@ class Socket: let match: str var case: bytes +# it is also the name a declaration keyword introduces +protocol match: + def go(self) -> int + +newtype lazy = int + +enum class case: + case A + # the vendored typeshed spells an optional with the postfix marker, in every # position a stub writes one class Cursor: diff --git a/crates/ruff_python_formatter/tests/snapshots/format@extension.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@extension.by.snap index 51b608fe02..8ddfbc65c2 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@extension.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@extension.by.snap @@ -19,6 +19,12 @@ extension list[Element: int]: return sum(self) +# a soft keyword names the extended type like any other name +extension type: + def short_name(self) -> str: + return self.__name__[:2] + + # computed property extension str: @property @@ -81,6 +87,12 @@ extension list[Element: int]: return sum(self) +# a soft keyword names the extended type like any other name +extension type: + def short_name(self) -> str: + return self.__name__[:2] + + # computed property extension str: @property diff --git a/crates/ruff_python_formatter/tests/snapshots/format@init_method.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@init_method.by.snap index 8ebac7ff2f..9551be94e4 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@init_method.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@init_method.by.snap @@ -26,6 +26,10 @@ class PlainParams: init(self, a: int, let b: int) +class SoftKeywordNames: + init(let cast: int, var match: int, private let type: int, local lazy: int) + + class DefinitionModifiers: private init(self, let a: int) @@ -98,6 +102,10 @@ class PlainParams: init(self, a: int, let b: int) +class SoftKeywordNames: + init(let cast: int, var match: int, private let type: int, local lazy: int) + + class DefinitionModifiers: private init(self, let a: int) diff --git a/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap index c8ed01bbd4..e25707fe2f 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap @@ -28,10 +28,19 @@ def with_context(local fn: () -> None, context c: int) -> None: ... def not_modifiers(local, once) -> None: ... +# a soft keyword after the modifiers is the parameter's name +def soft_keyword_names(local type: int, once match: () -> None, context cast: int) -> None: ... + + # keyword-only, variadic and defaulted parameters def positions(local a: int, *, once cb: () -> None = print) -> None: ... +# in a callable type the modifier borrows the parameter it precedes, which may be +# named by a soft keyword +def borrows(make: (local type[int], once str) -> int) -> int: ... + + # lambdas take the same modifiers handler = lambda local fn: fn @@ -68,10 +77,21 @@ def with_context(local fn: () -> None, context c: int) -> None: ... def not_modifiers(local, once) -> None: ... +# a soft keyword after the modifiers is the parameter's name +def soft_keyword_names( + local type: int, once match: () -> None, context cast: int +) -> None: ... + + # keyword-only, variadic and defaulted parameters def positions(local a: int, *, once cb: () -> None = print) -> None: ... +# in a callable type the modifier borrows the parameter it precedes, which may be +# named by a soft keyword +def borrows(make: (local type[int], once str) -> int) -> int: ... + + # lambdas take the same modifiers handler = lambda local fn: fn diff --git a/crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py b/crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py new file mode 100644 index 0000000000..280199b84e --- /dev/null +++ b/crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py @@ -0,0 +1,9 @@ +# basedpython identity-comparison operators. python has no such operator, so a +# `.py` file spelling one is a syntax error, while the comparison keeps its shape +x === y +x !== y + +# chained with each other. a chain mixing in the `is` keyword is rejected in +# basedpython mode, and has its own test in `parser::tests` +a === b !== c +a !== b diff --git a/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py b/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py deleted file mode 100644 index e460bb77b6..0000000000 --- a/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py +++ /dev/null @@ -1,10 +0,0 @@ -# basedpython identity-comparison operators. this fixture parses in python -# mode, where the lexer still produces the tokens — a `.py` file spelling them -# was never valid python, and the parser's job here is only to keep the shape -x === y -x !== y - -# chained with each other. a chain mixing in the `is` keyword is rejected in -# basedpython mode, and has its own test in `parser::tests` -a === b !== c -a !== b diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index f9df346a4a..363393511d 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -159,6 +159,17 @@ fn force_unwrap_before_debug_equal() -> ParseErrorType { ) } +/// basedpython: which infix cast a `cast` keyword followed by `after_cast` is, or `None` when +/// that `cast` is an ordinary name +fn infix_cast_kind(after_cast: TokenKind) -> Option { + match after_cast { + TokenKind::Exclamation => Some(CastKind::Checked), + TokenKind::Question => Some(CastKind::Try), + kind if EXPR_SET.contains(kind) || kind.is_soft_keyword() => Some(CastKind::Static), + _ => None, + } +} + impl<'src> Parser<'src> { /// Returns `true` if the parser is at a name or keyword (including soft keyword) token. pub(super) fn at_name_or_keyword(&self) -> bool { @@ -175,6 +186,24 @@ impl<'src> Parser<'src> { self.current_token_kind().is_soft_keyword() } + /// basedpython: whether the token `n` places past the next one is a name, which a keyword + /// in front of it introduces or modifies — the `P` of `protocol P:`, the `type` of + /// `extension type:` or `(local type[int]) -> None`. a soft keyword is a name there like + /// anywhere else, except a `cast` that is the infix cast operator: `protocol cast int` + /// casts a value named `protocol`, as `x cast int` casts `x` + /// + /// this is the test for a keyword that could itself be an ordinary name, and so the + /// operand of that `cast`. where the position is already pinned down — the tokens after + /// the name are checked as well, or no expression is being read — the statement parser's + /// `declares_a_name` answers for a single token kind + pub(super) fn peek_is_name(&mut self, n: usize) -> bool { + match self.peek_nth(n).0 { + TokenKind::Name => true, + TokenKind::Cast => infix_cast_kind(self.peek_nth(n + 1).0).is_none(), + kind => kind.is_soft_keyword(), + } + } + /// basedpython: whether the expression just parsed ended with a suite, which /// makes it the last thing in the statement it belongs to. /// @@ -204,7 +233,8 @@ impl<'src> Parser<'src> { /// `None` if no variance keyword was present. /// /// Disambiguates `out` from a bare reference to a variable named `out`: - /// the variance form requires the next token to be a *name* (`out T`). + /// the variance form requires the next token to be a *name* (`out T`), as + /// [`Self::peek_is_name`] decides it. /// Two adjacent names are never valid Python, so `out T` is unambiguous — /// whereas `out[...]`, `out(...)`, `out.attr`, `out + 1` etc. are an /// ordinary subscript / call / attribute / arithmetic on a variable named @@ -213,7 +243,7 @@ impl<'src> Parser<'src> { &mut self, ) -> Option { use ruff_python_ast::helpers::UseSiteVariance; - let next_starts_type_expr = matches!(self.peek(), TokenKind::Name); + let next_starts_type_expr = self.peek_is_name(0); let variance = if self.at(TokenKind::In) { // `in` is a hard keyword, so its presence in subscript-start // position is unambiguously variance. @@ -278,19 +308,20 @@ impl<'src> Parser<'src> { /// range of the keyword itself. Returns `None` if no modifier is present. /// /// Disambiguated exactly like [`Self::eat_basedpython_variance_prefix`]: the - /// modifier form requires the next token to be a *name*, and two adjacent - /// names are never valid Python. So `literal str` is unambiguously the - /// modifier, while `literal`, `final[x]`, `literal.attr`, `final(x)` etc. - /// stay ordinary references to a variable of that name. The consequence is - /// that a type which does not start with a name — a parenthesized callable - /// type, a string forward reference, a starred type — cannot carry a bare - /// modifier; see the docs for the recommended spelling. + /// modifier form requires the next token to be a *name* + /// ([`Self::peek_is_name`]), and two adjacent names are never valid + /// Python. So `literal str` is unambiguously the modifier, while `literal`, + /// `final[x]`, `literal.attr`, `final(x)` etc. stay ordinary references to + /// a variable of that name. The consequence is that a type which does not + /// start with a name — a parenthesized callable type, a string forward + /// reference, a starred type — cannot carry a bare modifier; see the docs + /// for the recommended spelling. fn eat_basedpython_type_modifier_prefix( &mut self, ) -> Option<(ruff_python_ast::helpers::TypeModifier, TextRange)> { use ruff_python_ast::helpers::TypeModifier; - if !self.at(TokenKind::Name) || !matches!(self.peek(), TokenKind::Name) { + if !self.at(TokenKind::Name) || !self.peek_is_name(0) { return None; } let range = self.current_token_range(); @@ -577,14 +608,7 @@ impl<'src> Parser<'src> { // expression level (where `left_precedence` is `None`). Lowered by // `transforms::checked_cast`. if current_token == TokenKind::Cast - && let Some(cast_kind) = match self.peek() { - TokenKind::Exclamation => Some(CastKind::Checked), - TokenKind::Question => Some(CastKind::Try), - peeked if EXPR_SET.contains(peeked) || peeked.is_soft_keyword() => { - Some(CastKind::Static) - } - _ => None, - } + && let Some(cast_kind) = infix_cast_kind(self.peek()) { if left_precedence > OperatorPrecedence::None { break; @@ -1187,7 +1211,7 @@ impl<'src> Parser<'src> { TokenKind::Name if self.options.is_basedpython && self.src_text(self.current_token_range()) == "some" - && self.peek() == TokenKind::Name => + && self.peek_is_name(0) => { self.add_error( ParseErrorType::OtherError( @@ -2291,6 +2315,9 @@ impl<'src> Parser<'src> { CmpOp::Is => { // accept either `is` or `===`; bump whichever the lexer emitted if self.at(TokenKind::EqEqEqual) { + self.error_if_not_basedpython( + "identity operator `===` is not valid in .py files".to_string(), + ); (TokenKind::EqEqEqual, None) } else { (TokenKind::Is, None) @@ -2299,6 +2326,9 @@ impl<'src> Parser<'src> { CmpOp::IsNot => { // accept either `is not` or `!==`; bump whichever the lexer emitted if self.at(TokenKind::BangEqEqual) { + self.error_if_not_basedpython( + "identity operator `!==` is not valid in .py files".to_string(), + ); (TokenKind::BangEqEqual, None) } else { (TokenKind::Is, Some(TokenKind::Not)) @@ -3592,7 +3622,7 @@ impl<'src> Parser<'src> { let mut borrow = ParameterBorrow::None; while self.at(TokenKind::Name) && matches!(self.src_text(self.current_token_range()), "local" | "once") - && self.peek() == TokenKind::Name + && self.peek_is_name(0) // `(local x=1)` is an anonymous named tuple *value*, which has no // parameters to borrow — leave the name alone so it keeps its // existing reading @@ -4806,13 +4836,20 @@ impl<'src> Parser<'src> { self.expect(TokenKind::Rpar); - // do not lift this tuple to is_anon_named_tuple even when it has - // plain `name: T` fields — markers (`/`, `*`) signal the user - // wants parameter-spec semantics (positional-only / keyword-only), - // which conflicts with the implicit NamedTuple synthesis. anon-NT - // lifting is reserved for tuples without markers (the existing - // `(name: T, ...)` and `(int, name: T)` mixed dispatch upstream) - let is_anon_named_tuple = false; + // a tuple with no marker (`/`, `*`) and no variadic is an anonymous named tuple + // whenever it has a `name: T` field, wherever the field comes — `(int, str, + // name: T)` as much as the `(int, name: T)` dispatched upstream. a marker or a + // variadic says the tuple is a parameter list, which no named tuple can spell + let is_anon_named_tuple = slash.is_none() + && star.is_none() + && elts + .iter() + .any(|elt| matches!(elt, Expr::Named(named) if named.label().is_some())) + && !elts.iter().any(|elt| match elt { + Expr::Starred(_) => true, + Expr::Named(named) => matches!(*named.target, Expr::Starred(_)), + _ => false, + }); ast::ExprTuple { elts, @@ -4822,7 +4859,7 @@ impl<'src> Parser<'src> { parenthesized: true, is_anon_named_tuple, is_anon_named_tuple_value: false, - is_parameter_shape: true, + is_parameter_shape: !is_anon_named_tuple, callable_shape: CallableParameterShape { slash, star, diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 6410073993..015d791e4b 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -137,6 +137,13 @@ fn mark_context(stmt: Stmt, has_context: bool) -> Stmt { /// `let type: int` declares a field called `type`, which is what both /// `socket.SocketType` and `asyncio.TransportSocket` have. Only the token kind /// differs from a plain name — [`Parser::parse_identifier`] already reads either. +/// +/// use it only where `kind` cannot be the infix `cast` operator: either the tokens +/// around it are checked too (`let NAME =`, `class let NAME`), or it follows a +/// parameter's modifiers or an enum body's `case`, where no expression is being read +/// (`case cast(radius: float)` declares a variant named `cast`). where the word in front +/// of `kind` could be a value that a following `cast` operates on (`protocol cast +/// int`), ask [`Parser::peek_is_name`] instead fn declares_a_name(kind: TokenKind) -> bool { kind == TokenKind::Name || kind.is_soft_keyword() } @@ -758,22 +765,20 @@ impl<'src> Parser<'src> { let kw = self.src_text(self.current_token_range()); // single-keyword introducers without modifier-chain support - if kw == "newtype" - && self.peek() == TokenKind::Name - && self.peek_nth(1).0 == TokenKind::Equal + if kw == "newtype" && declares_a_name(self.peek()) && self.peek_nth(1).0 == TokenKind::Equal { self.error_if_not_basedpython( "`newtype` declarations are not valid in .py files".to_string(), ); return Some(self.parse_newtype_decl(start)); } - if kw == "protocol" && self.peek() == TokenKind::Name { + if kw == "protocol" && self.peek_is_name(0) { self.error_if_not_basedpython( "`protocol` class syntax is not valid in .py files".to_string(), ); return Some(self.parse_protocol_def(start, DecoratorList::new())); } - if kw == "extension" && self.peek() == TokenKind::Name { + if kw == "extension" && self.peek_is_name(0) { self.error_if_not_basedpython( "`extension` declarations are not valid in .py files".to_string(), ); @@ -792,7 +797,7 @@ impl<'src> Parser<'src> { ); return Some(self.parse_build_def(start)); } - if kw == "implements" && self.peek() == TokenKind::Name { + if kw == "implements" && self.peek_is_name(0) { self.error_if_not_basedpython( "`implements` declarations are not valid in .py files".to_string(), ); @@ -802,8 +807,7 @@ impl<'src> Parser<'src> { // sum type when its body has payload variants, an idiomatic `Enum` when // its variants are all unit). the `class` keyword is part of the // declaration - if kw == "enum" && self.peek() == TokenKind::Class && self.peek_nth(1).0 == TokenKind::Name - { + if kw == "enum" && self.peek() == TokenKind::Class && declares_a_name(self.peek_nth(1).0) { self.error_if_not_basedpython( "`enum class` declarations are not valid in .py files".to_string(), ); @@ -811,7 +815,7 @@ impl<'src> Parser<'src> { } // a bare `enum E:` (no `class`) is not valid — based enums are written // `enum class E:`. report it but still parse the body for recovery - if kw == "enum" && self.peek() == TokenKind::Name { + if kw == "enum" && self.peek_is_name(0) { self.add_error( ParseErrorType::OtherError( "based enums must be written `enum class E:`, not `enum E:`".to_string(), @@ -1011,7 +1015,7 @@ impl<'src> Parser<'src> { // modifier chain — `private enum class E:`, `export protocol P:`. // dispatch through the modifier path so the chain is carried // as decorators on the introduced class - if (text == "protocol" && following == TokenKind::Name) + if (text == "protocol" && self.peek_is_name(idx)) || (text == "enum" && following == TokenKind::Class) { self.error_if_not_basedpython(format!( @@ -1383,7 +1387,7 @@ impl<'src> Parser<'src> { Stmt::FunctionDef(self.parse_function_definition(decorators, start)) } else if self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "protocol" - && self.peek() == TokenKind::Name + && self.peek_is_name(0) { // `private protocol P:` — the modifier decorators ride alongside the // synthetic `protocol_class` marker; both lower by disjoint edits @@ -2172,7 +2176,7 @@ impl<'src> Parser<'src> { let mut args: Vec = Vec::new(); loop { - if !self.at(TokenKind::Name) { + if !self.at_name_or_soft_keyword() { self.add_error( ParseErrorType::OtherError("`implements` takes an interface name".to_string()), self.current_token_range(), @@ -2515,9 +2519,7 @@ impl<'src> Parser<'src> { // `case` followed by a name declares variant(s) — `case A`, // `case A, B, C`, `case Circle(radius: float), Empty`. any other use of // `case` (`case = 1`, `case.x`, …) is an ordinary identifier statement - if self.at(TokenKind::Case) - && (self.peek() == TokenKind::Name || self.peek().is_soft_keyword()) - { + if self.at(TokenKind::Case) && declares_a_name(self.peek()) { self.parse_case_variants(statements); return; } @@ -7556,7 +7558,7 @@ impl<'src> Parser<'src> { // appended after them. if self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "protocol" - && self.peek() == TokenKind::Name + && self.peek_is_name(0) { self.error_if_not_basedpython( "`protocol` class syntax is not valid in .py files".to_string(), @@ -7575,9 +7577,8 @@ impl<'src> Parser<'src> { // basedpython: a modifier chain also prefixes a *binding* — `export let // a = 1` — which is not what `parse_with_modifier` reads. When the chain // does not end at a definition the statement is parsed as the decorated - // binding it is instead - if at_modifier && (!self.options.is_basedpython || self.modifier_chain_ends_at_definition()) - { + // binding it is instead, or in a `.py` file gets python's own recovery + if at_modifier && self.modifier_chain_ends_at_definition() { return self.parse_with_modifier(start, decorators); } @@ -7676,7 +7677,7 @@ impl<'src> Parser<'src> { TokenKind::Name => { let text = self.src_text(range); // `protocol P:` and `enum class E:` introduce a definition - if text == "protocol" && declares_a_name(self.peek_nth(idx).0) { + if text == "protocol" && self.peek_is_name(idx) { return true; } if text == "enum" && self.peek_nth(idx).0 == TokenKind::Class { @@ -7885,7 +7886,7 @@ impl<'src> Parser<'src> { // permissive here; the `init_method` lowering rejects combinations that // are invalid for this context while self.at(TokenKind::Name) - && self.peek() == TokenKind::Name + && declares_a_name(self.peek()) && is_param_modifier_kw(self.src_text(self.current_token_range())) { let kw = self.src_text(self.current_token_range()).to_string(); @@ -7901,7 +7902,7 @@ impl<'src> Parser<'src> { if matches!(allow_context, AllowContextModifier::Yes) && self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "context" - && self.peek() == TokenKind::Name + && declares_a_name(self.peek()) { self.error_if_not_basedpython( "`context` parameter modifier is not valid in .py files".to_string(), @@ -7916,11 +7917,11 @@ impl<'src> Parser<'src> { // order (`once local fn`). Like `let`, the keywords carry no AST field: // the `Parameter.range` covers them (it starts before this run) and the // strip transform + ty analysis detect them from the source span. The - // trailing `Name` guard keeps a parameter literally named `local`/`once` + // trailing name guard keeps a parameter literally named `local`/`once` // (`def f(local)`) from being read as a modifier. while self.at(TokenKind::Name) && matches!(self.src_text(self.current_token_range()), "local" | "once") - && self.peek() == TokenKind::Name + && declares_a_name(self.peek()) { self.error_if_not_basedpython( "`local` / `once` parameter modifiers are not valid in .py files".to_string(), @@ -7972,6 +7973,9 @@ impl<'src> Parser<'src> { let mut is_some = false; let annotation = match function_kind { FunctionKind::FunctionDef if self.eat(TokenKind::Colon) => { + // the bound is any type, including one that does not start with a name + // (`some () -> int`, `some (A | B)`), so this asks whether an expression + // follows rather than `peek_is_name` if self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "some" && (EXPR_SET.contains(self.peek()) || self.peek().is_soft_keyword()) diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index f9899eac1e..bb8d01a41e 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -3,8 +3,8 @@ use std::assert_matches; use ruff_python_ast::comparable::ComparableStmt; use ruff_python_ast::helpers::{UseSiteVariance, use_site_variance_marker}; use ruff_python_ast::{ - Expr, InterpolatedStringElement, IpyEscapeKind, ModModule, Number, Operator, Pattern, Stmt, - TypeParam, UnaryOp, is_destructure_binder, + CastKind, Expr, InterpolatedStringElement, IpyEscapeKind, ModModule, Number, Operator, Pattern, + Stmt, TypeParam, UnaryOp, is_destructure_binder, }; use ruff_text_size::Ranged; @@ -72,6 +72,28 @@ fn identity_operators_are_recorded_per_operator() { ); } +/// `===` and `!==` are basedpython's identity operators. python has no such +/// operator, so a `.py` file spelling one is a syntax error there, as every +/// other basedpython-only form is +#[test] +fn identity_operators_are_not_python() { + for source in ["a === b\n", "a !== b\n", "a == b === c\n"] { + let parsed = parse_module(source); + let Err(error) = parsed else { + panic!("expected a syntax error for {source:?}"); + }; + assert!( + matches!(&error.error, ParseErrorType::BasedPythonOnly(message) if message.contains("is not valid in .py files")), + "{source:?}: {error}" + ); + } + assert!( + parse_basedpython_module_with_errors("a === b !== c\n") + .errors() + .is_empty() + ); +} + #[test] fn a_type_test_may_not_be_chained() { // python chains `a is int is str` into `a is int and int is str`, whose @@ -373,6 +395,122 @@ fn basedpython_declares_a_soft_keyword_name() { assert!(alias.is_private); } +/// the name a declaration introduced by a keyword declares: a class or a newtype +fn introduced_name(stmt: &Stmt) -> &str { + match stmt { + Stmt::ClassDef(class) => class.name.as_str(), + Stmt::AnnAssign(assign) => match assign.target.as_ref() { + Expr::Name(name) => name.id.as_str(), + other => panic!("expected a name target, got {other:?}"), + }, + other => panic!("expected a declaration, got {other:?}"), + } +} + +/// the interface at `index` in the `implements` declaration that is the whole body of +/// the one class in `parsed` +fn implemented_interface(parsed: &Parsed, index: usize) -> &str { + let [Stmt::ClassDef(class)] = parsed.suite().as_slice() else { + panic!("expected one class, got {:?}", parsed.suite()); + }; + let [Stmt::Expr(statement)] = class.body.as_slice() else { + panic!( + "expected one `implements` declaration, got {:?}", + class.body + ); + }; + let Expr::Call(call) = statement.value.as_ref() else { + panic!( + "expected an `implements` marker call, got {:?}", + statement.value + ); + }; + match call.arguments.args.get(index) { + Some(Expr::Name(name)) => name.id.as_str(), + other => panic!("expected an interface name at {index}, got {other:?}"), + } +} + +#[test] +fn basedpython_an_introducer_declares_a_soft_keyword_name() { + // a soft keyword after `protocol`, `extension`, `newtype`, `implements` or + // `enum class` is the name the declaration introduces + for keyword in ["match", "case", "type", "lazy", "cast"] { + for source in [ + format!("protocol {keyword}:\n x: int\n"), + format!("private protocol {keyword}:\n x: int\n"), + format!("@runtime_checkable\nprotocol {keyword}:\n x: int\n"), + format!("@runtime_checkable\nprivate protocol {keyword}:\n x: int\n"), + format!("extension {keyword}:\n def f(self) -> int: ...\n"), + format!("newtype {keyword} = int\n"), + format!("enum class {keyword}:\n case A\n"), + format!("private enum class {keyword}:\n case A\n"), + ] { + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + let [stmt] = parsed.suite().as_slice() else { + panic!("expected one statement for `{source}`"); + }; + assert_eq!(introduced_name(stmt), keyword, "{source}"); + } + + for (source, index) in [ + (format!("class A:\n implements {keyword}, B\n"), 0), + (format!("class A:\n implements B, {keyword}\n"), 1), + ] { + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + assert_eq!(implemented_interface(&parsed, index), keyword, "{source}"); + } + } + + // a bare `enum E:` is reported as one error, whatever names the enum + for keyword in ["match", "type", "cast"] { + let source = format!("enum {keyword}:\n case A\n"); + let parsed = parse_basedpython_module_with_errors(&source); + let errors: Vec = parsed + .errors() + .iter() + .map(|error| error.error.to_string()) + .collect(); + assert_eq!( + errors, + ["based enums must be written `enum class E:`, not `enum E:`"], + "{source}" + ); + assert_eq!(introduced_name(&parsed.suite()[0]), keyword, "{source}"); + } + + // `cast` followed by a type is the infix cast operator, so the keyword in front of it + // is the value being cast rather than a declaration of something named `cast` + for keyword in ["protocol", "extension", "implements", "enum"] { + for operator in ["cast", "cast!", "cast?"] { + let source = format!("{keyword} {operator} int\n"); + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + let [Stmt::Expr(statement)] = parsed.suite().as_slice() else { + panic!("expected an expression statement for `{source}`"); + }; + let Expr::Call(call) = statement.value.as_ref() else { + panic!("expected a cast for `{source}`"); + }; + assert!(call.cast_kind.is_some(), "{source}"); + } + } + + // a decorated modifier chain decides that it ends at a protocol by the same test, so + // a cast there is reported as a decorator on something that is not a definition + let parsed = parse_basedpython_module_with_errors("@d\nprivate protocol cast int\n"); + assert!( + parsed.errors().iter().any(|error| error + .error + .to_string() + .contains("Expected a definition or a binding after decorator")), + "got {:?}", + parsed.errors() + ); +} + #[test] fn basedpython_context_composes_with_the_declaration_modifiers() { // `context` takes its place in the modifier chain rather than forming a statement of @@ -600,6 +738,69 @@ fn basedpython_local_once_as_bare_param_name() { } } +/// the parameter names of the first function in `parsed`, which may be the first +/// statement of a class body +fn parameter_names(parsed: &Parsed) -> Vec { + let function = match parsed.suite().first() { + Some(Stmt::FunctionDef(function)) => function, + Some(Stmt::ClassDef(class)) => match class.body.first() { + Some(Stmt::FunctionDef(function)) => function, + other => panic!("expected a method, got {other:?}"), + }, + other => panic!("expected a function or a class, got {other:?}"), + }; + function + .parameters + .iter() + .map(|parameter| parameter.name().to_string()) + .collect() +} + +#[test] +fn basedpython_param_modifier_before_a_soft_keyword_name() { + // a soft keyword lexes as its own token kind, but after a parameter modifier it + // is the parameter's name like any other identifier + for keyword in ["match", "case", "type", "lazy", "cast"] { + for modifier in [ + "let", + "var", + "private", + "private let", + "local", + "once", + "once local", + "context", + "context local", + ] { + let source = format!("def f({modifier} {keyword}: int, {modifier} b: int): ...\n"); + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + assert_eq!(parameter_names(&parsed), [keyword, "b"], "{source}"); + } + let source = format!("class A:\n init(let {keyword}: int, var b: int = 1)\n"); + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + assert_eq!(parameter_names(&parsed), ["self", keyword, "b"], "{source}"); + } + + // a modifier keyword followed by a separator is the parameter itself, whether or + // not a soft keyword names a parameter beside it + for (source, names) in [ + ("def f(let, match): ...\n", &["let", "match"][..]), + ("def f(local, type): ...\n", &["local", "type"]), + ("def f(match, let): ...\n", &["match", "let"]), + ("def f(let let): ...\n", &["let"]), + ( + "def f(context: int, cast: int): ...\n", + &["context", "cast"], + ), + ] { + let parsed = parse_basedpython_module_with_errors(source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + assert_eq!(parameter_names(&parsed), names, "{source}"); + } +} + #[test] fn basedpython_local_once_param_rejected_in_py() { // the modifiers are `.by`-only; a `.py` file using them collects a @@ -688,6 +889,10 @@ fn basedpython_local_once_in_callable_type() { // an implicit receiver is not one of `args`, so the modifiers still // line up with the written parameters ("f: int.(local str) -> None", &[Local]), + // a soft keyword after the modifier is the type it borrows + ("f: (local type[int]) -> None", &[Local]), + ("f: (int, once match) -> None", &[NoBorrow, Once]), + ("f: (local cast) -> None", &[Local]), ] { let parsed = parse_basedpython_module(source); let [Stmt::AnnAssign(assign)] = parsed.syntax().body.as_slice() else { @@ -710,6 +915,37 @@ fn basedpython_local_once_in_callable_type() { } } +/// `cast` followed by a type is the infix cast operator, so the `local` written before it +/// is the value it casts rather than a modifier on a parameter named `cast` +#[test] +fn basedpython_local_before_an_infix_cast_is_its_operand() { + for (source, kind) in [ + ("y = (local cast int)\n", CastKind::Static), + ("y = (local cast! int)\n", CastKind::Checked), + ("y = (local cast? int)\n", CastKind::Try), + ("y = (local cast int, once)\n", CastKind::Static), + ] { + let parsed = parse_basedpython_module_with_errors(source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + let [Stmt::Assign(assign)] = parsed.suite().as_slice() else { + panic!("expected an assignment for `{source}`"); + }; + let call = match assign.value.as_ref() { + Expr::Call(call) => call, + Expr::Tuple(tuple) => match tuple.elts.first() { + Some(Expr::Call(call)) => call, + other => panic!("expected a cast first in `{source}`, got {other:?}"), + }, + other => panic!("expected a cast in `{source}`, got {other:?}"), + }; + assert_eq!(call.cast_kind, Some(kind), "{source}"); + let [_, Expr::Name(value)] = &*call.arguments.args else { + panic!("expected the cast's value to be a name in `{source}`"); + }; + assert_eq!(value.id.as_str(), "local", "{source}"); + } +} + #[test] fn basedpython_local_once_in_callable_type_rejected_in_py() { // the modifiers are `.by`-only here too @@ -808,6 +1044,30 @@ fn basedpython_decorated_protocol_keyword() { assert_eq!(names, ["runtime_checkable", "protocol_class"]); } +/// in a `.py` file a decorator over a modifier word that starts no definition gets python's +/// own recovery, as a decorator over any other statement does +#[test] +fn a_decorator_over_a_modifier_word_in_a_py_file_recovers() { + for source in [ + "@d\nfinal x\n", + "@d\nfinal = 1\n", + "@d\nprivate x = 1\n", + "@d\nprivate protocol cast int\n", + ] { + let parsed = crate::parse_unchecked(source, ParseOptions::from(Mode::Module)); + assert!( + parsed + .errors() + .iter() + .any(|error| error.error.to_string().starts_with( + "Expected class, function definition or async function definition" + )), + "{source}: {:?}", + parsed.errors() + ); + } +} + #[test] fn basedpython_final_in_modifier_chain_keeps_final_marker() { // `final` combined with another modifier must still carry the `__final__` @@ -1729,6 +1989,48 @@ fn test_anon_named_tuple_mixed_type() { insta::assert_debug_snapshot!(parsed.syntax()); } +/// a named field makes the tuple an anonymous named tuple wherever it comes, not only right after +/// the first positional one, while a marker or a variadic keeps it a parameter list +#[test] +fn test_anon_named_tuple_named_field_after_positionals() { + let annotation_tuple = |source: &str| { + let parsed = parse_basedpython_module(source); + let Some(Stmt::AnnAssign(assign)) = parsed.suite().first().cloned() else { + panic!("expected an annotated assignment in {source:?}"); + }; + let Expr::Tuple(tuple) = *assign.annotation else { + panic!("expected a tuple annotation in {source:?}"); + }; + tuple + }; + for source in [ + "a: (int, str, name: bytes)\n", + "a: (int, local b: str)\n", + "a: (int, str, name: bytes, int)\n", + "a: (int, (b := str), name: bytes)\n", + ] { + let tuple = annotation_tuple(source); + assert!(tuple.is_anon_named_tuple, "{source:?}"); + assert!(!tuple.is_parameter_shape, "{source:?}"); + } + // a parenthesized walrus is a positional element, not a field: it binds its target + let tuple = annotation_tuple("a: (int, (b := str), name: bytes)\n"); + let Expr::Named(walrus) = &tuple.elts[1] else { + panic!("expected the walrus as the second element"); + }; + assert!(walrus.label().is_none()); + for source in [ + "a: (int, /, name: str)\n", + "a: (int, *, name: str)\n", + "a: (int, *args: str)\n", + "a: (int, **kw: str)\n", + ] { + let tuple = annotation_tuple(source); + assert!(!tuple.is_anon_named_tuple, "{source:?}"); + assert!(tuple.is_parameter_shape, "{source:?}"); + } +} + #[test] fn test_inline_protocol_type() { let source = "a: protocol(a: int; b: str; def f(self) -> int)\n"; @@ -1881,6 +2183,14 @@ fn basedpython_use_site_variance_in_any_slice_element() { None, ], ), + // a soft keyword is a name after `out` like after any other keyword, but + // `out cast int` casts a variable named `out` + ("a: list[out type[int]]\n", vec![Some(UseSiteVariance::Out)]), + ( + "a: dict[int, out match]\n", + vec![None, Some(UseSiteVariance::Out)], + ), + ("a: X[out cast int]\n", vec![None]), ] { let parsed = parse_basedpython_module(source); assert_eq!( @@ -2877,6 +3187,9 @@ fn basedpython_type_modifier_prefix() { ("a: final int = 1\n", TypeModifier::Final), ("a: literal list[*]\n", TypeModifier::Literal), ("a: final dict[str, int]\n", TypeModifier::Final), + // a soft keyword is a name after the modifier like after any other keyword + ("a: final type[int]\n", TypeModifier::Final), + ("a: literal match\n", TypeModifier::Literal), ] { let parsed = parse_basedpython_module(source); let [Stmt::AnnAssign(ann)] = parsed.syntax().body.as_slice() else { @@ -2957,6 +3270,9 @@ fn basedpython_type_modifier_keywords_stay_identifiers() { "a: final[int]\n", "a: literal.Alias\n", "a: final()\n", + // `cast` followed by a type is the infix cast operator, so this casts a + // variable called `literal` + "a: literal cast int\n", ] { let parsed = parse_basedpython_module(source); let [Stmt::AnnAssign(ann)] = parsed.syntax().body.as_slice() else { @@ -3454,6 +3770,10 @@ fn a_misplaced_some_is_one_error() { "def g(xs: list[int]): ...", ), ("x: some int = 1\n", "x: int = 1"), + ( + "def h(xs: list[some type[int]]): ...\n", + "def h(xs: list[type[int]]): ...", + ), ("y = cast(some int?, 1)\n", "y = cast(int?, 1)"), ] { let parsed = parse_basedpython_module_with_errors(source); @@ -3481,6 +3801,26 @@ fn a_misplaced_some_is_one_error() { } } +/// at the top of a parameter's annotation `some` takes any type as its bound, not only one +/// that starts with a name +#[test] +fn some_at_the_top_of_an_annotation_takes_any_type() { + for source in [ + "def f(x: some int): ...\n", + "def f(x: some type[int]): ...\n", + "def f(x: some () -> int): ...\n", + "def f(x: some (int | str)): ...\n", + "def f(x: some \"Forward\"): ...\n", + ] { + let parsed = parse_basedpython_module_with_errors(source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + let Some(Stmt::FunctionDef(function)) = parsed.suite().first() else { + panic!("expected a function for `{source}`"); + }; + assert!(function.parameters.args[0].parameter.is_some, "{source}"); + } +} + /// a variable named `some` followed by anything that continues an expression is that /// variable, as it is in python #[test] diff --git a/crates/ruff_python_parser/src/semantic_errors.rs b/crates/ruff_python_parser/src/semantic_errors.rs index 49fd6f1de5..b5f9fbe009 100644 --- a/crates/ruff_python_parser/src/semantic_errors.rs +++ b/crates/ruff_python_parser/src/semantic_errors.rs @@ -13,7 +13,6 @@ use ruff_python_ast::{ }; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use rustc_hash::{FxBuildHasher, FxHashSet}; -use std::cell::RefCell; use std::fmt::Display; #[derive(Debug, Default)] @@ -862,30 +861,6 @@ impl SemanticSyntaxChecker { } } - /// basedpython: the two `match` checks that a `.by` source holds back, run - /// over python. - /// - /// A bare `case A:` name may be an enum member of the subject rather than a - /// capture, and no syntactic rule can tell which — so for a `.by` source the - /// type checker reports both of these instead. The transpiler runs them - /// again over the python it emits, where every name is spelled out and the - /// answer is plain again, which is what keeps a `match` the checker rejected - /// from being emitted as python that cannot be parsed. - pub fn python_match_statement_errors( - stmt: &ast::StmtMatch, - python_version: PythonVersion, - ) -> Vec { - let context = CollectMatchErrors { - python_version, - errors: RefCell::default(), - }; - Self::irrefutable_match_case(stmt, &context); - for case in &stmt.cases { - Self::check_pattern_bindings(&case.pattern, &context); - } - context.errors.into_inner() - } - /// Reports a pattern that binds the same name more than once. fn check_pattern_bindings(pattern: &Pattern, ctx: &Ctx) { let mut visitor = MatchPatternVisitor { @@ -2985,79 +2960,6 @@ pub trait SemanticSyntaxContext { } } -/// The context [`SemanticSyntaxChecker::python_match_statement_errors`] runs its -/// two checks under. -/// -/// Neither reads anything about the surrounding scopes, so every other method is -/// answered with the value that makes its check a no-op — this context must -/// never be handed to the checker at large. -struct CollectMatchErrors { - python_version: PythonVersion, - errors: RefCell>, -} - -impl SemanticSyntaxContext for CollectMatchErrors { - fn report_semantic_error(&self, error: SemanticSyntaxError) { - self.errors.borrow_mut().push(error); - } - - fn python_version(&self) -> PythonVersion { - self.python_version - } - - fn future_annotations_or_stub(&self) -> bool { - false - } - fn lazy_import_context(&self) -> Option { - None - } - fn source(&self) -> &'static str { - "" - } - fn global(&self, _name: &str) -> Option { - None - } - fn has_nonlocal_binding(&self, _name: &str) -> bool { - false - } - fn in_async_context(&self) -> bool { - false - } - fn in_await_allowed_context(&self) -> bool { - false - } - fn in_yield_allowed_context(&self) -> bool { - false - } - fn in_sync_comprehension(&self) -> bool { - false - } - fn in_class_body_comprehension(&self) -> bool { - false - } - fn in_module_scope(&self) -> bool { - false - } - fn in_function_scope(&self) -> bool { - false - } - fn def_is_method(&self) -> bool { - true - } - fn in_generator_context(&self) -> bool { - false - } - fn in_notebook(&self) -> bool { - false - } - fn in_loop_context(&self) -> bool { - false - } - fn is_bound_parameter(&self, _name: &str) -> bool { - false - } -} - /// Modified version of [`std::str::EscapeDefault`] that does not escape single or double quotes. struct EscapeDefault<'a>(&'a str); diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__identity_compare.py.snap similarity index 74% rename from crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap rename to crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__identity_compare.py.snap index cae10a971b..e64f560302 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__identity_compare.py.snap @@ -1,6 +1,6 @@ --- source: crates/ruff_python_parser/tests/fixtures.rs -input_file: crates/ruff_python_parser/resources/valid/expressions/identity_compare.py +input_file: crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py --- ## AST @@ -8,20 +8,20 @@ input_file: crates/ruff_python_parser/resources/valid/expressions/identity_compa Module( ModModule { node_index: NodeIndex(None), - range: 0..410, + range: 0..337, body: [ Expr( StmtExpr { node_index: NodeIndex(None), - range: 234..241, + range: 161..168, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 234..241, + range: 161..168, left: Name( ExprName { node_index: NodeIndex(None), - range: 234..235, + range: 161..162, id: Name("x"), ctx: Load, }, @@ -33,7 +33,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 240..241, + range: 167..168, id: Name("y"), ctx: Load, }, @@ -53,15 +53,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 242..249, + range: 169..176, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 242..249, + range: 169..176, left: Name( ExprName { node_index: NodeIndex(None), - range: 242..243, + range: 169..170, id: Name("x"), ctx: Load, }, @@ -73,7 +73,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 248..249, + range: 175..176, id: Name("y"), ctx: Load, }, @@ -93,15 +93,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 388..401, + range: 315..328, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 388..401, + range: 315..328, left: Name( ExprName { node_index: NodeIndex(None), - range: 388..389, + range: 315..316, id: Name("a"), ctx: Load, }, @@ -114,7 +114,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 394..395, + range: 321..322, id: Name("b"), ctx: Load, }, @@ -122,7 +122,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 400..401, + range: 327..328, id: Name("c"), ctx: Load, }, @@ -143,15 +143,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 402..409, + range: 329..336, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 402..409, + range: 329..336, left: Name( ExprName { node_index: NodeIndex(None), - range: 402..403, + range: 329..330, id: Name("a"), ctx: Load, }, @@ -163,7 +163,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 408..409, + range: 335..336, id: Name("b"), ctx: Load, }, @@ -184,3 +184,47 @@ Module( }, ) ``` +## Errors + + | +1 | # basedpython identity-comparison operators. python has no such operator, so a +2 | # `.py` file spelling one is a syntax error, while the comparison keeps its shape +3 | x === y + | ^^^ Syntax Error: identity operator `===` is not valid in .py files +4 | x !== y + | + + + | +2 | # `.py` file spelling one is a syntax error, while the comparison keeps its shape +3 | x === y +4 | x !== y + | ^^^ Syntax Error: identity operator `!==` is not valid in .py files +5 | +6 | # chained with each other. a chain mixing in the `is` keyword is rejected in + | + + + | +6 | # chained with each other. a chain mixing in the `is` keyword is rejected in +7 | # basedpython mode, and has its own test in `parser::tests` +8 | a === b !== c + | ^^^ Syntax Error: identity operator `===` is not valid in .py files +9 | a !== b + | + + + | +6 | # chained with each other. a chain mixing in the `is` keyword is rejected in +7 | # basedpython mode, and has its own test in `parser::tests` +8 | a === b !== c + | ^^^ Syntax Error: identity operator `!==` is not valid in .py files +9 | a !== b + | + + + | +7 | # basedpython mode, and has its own test in `parser::tests` +8 | a === b !== c +9 | a !== b + | ^^^ Syntax Error: identity operator `!==` is not valid in .py files diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index 384ee2a04f..f0b9da6c16 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -8,7 +8,7 @@ Default level: error · Added in 0.0.64 · Related issues · -View source +View source @@ -44,7 +44,7 @@ class Base(ABC): Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -90,7 +90,7 @@ class Derived(Base): # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -123,7 +123,7 @@ f(1, b=s1) # ok — explicit Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -131,13 +131,18 @@ Added in 0 Checks for conversion sites where more than one conversion applies — two dunders, a dunder and an in-scope conformance, or two applicable -`implementation`s of the same interface and type. +`implementation`s of the same interface and type. A plain tuple literal +that more than one expected anonymous named tuple accepts is one too: +`(1,)` where `(a: int) | (b: int)` is expected. **Why is this bad?** `__from__` and `__into__` are hand-written bodies that can disagree, so which one runs must not depend on arbitrary ordering. Remove one of them, -or write the conversion you want explicitly. +or write the conversion you want explicitly. A plain tuple literal is +constructed as the anonymous named tuple it stands for, which decides the +field names the value has, and nothing in the literal says which one was +meant: name the fields, as in `(a=1)`. **Example** @@ -159,7 +164,7 @@ report(Celsius()) # error: two conversions apply Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -194,7 +199,7 @@ extension list: Default level: warn · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -258,7 +263,7 @@ class SubProto(BaseProto, Protocol): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -340,7 +345,7 @@ value = unknown # ty: ignore[unresolved-reference] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -383,7 +388,7 @@ a4 = True + 1 # ok — a boolean used as a boolean Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -435,7 +440,7 @@ Foo.method() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -463,7 +468,7 @@ Calling a non-callable object will raise a `TypeError` at runtime. Default level: error · Added in 0.0.7 · Related issues · -View source +View source @@ -498,7 +503,7 @@ def f(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -546,7 +551,7 @@ def main(): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -593,7 +598,7 @@ def Fixed(show: bool): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -626,7 +631,7 @@ a = 1 # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -660,7 +665,7 @@ class C(A, B): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -706,7 +711,7 @@ def App(done: bool) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -740,7 +745,7 @@ class B(A): ... # error Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -785,7 +790,7 @@ type Tree = int | list[Tree] # valid recursive alias Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -821,7 +826,7 @@ class Example: Default level: warn · Added in 0.0.1-alpha.16 · Related issues · -View source +View source @@ -860,7 +865,7 @@ old_func() # error: [deprecated] Default level: ignore · Added in 0.0.78 · Related issues · -View source +View source @@ -1032,7 +1037,7 @@ soundness checks from their type checker, and it may have false positives in som Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1064,7 +1069,7 @@ This rule is currently disabled by default because of the number of false positi Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1095,7 +1100,7 @@ class B(A, A): ... # error Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -1135,7 +1140,7 @@ class A: # error Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -1246,7 +1251,7 @@ Python code. Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -1295,7 +1300,7 @@ def bar() -> str: # error: [empty-body] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -1356,7 +1361,7 @@ def h(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -1458,7 +1463,7 @@ def foo() -> "intt\b": ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1497,7 +1502,7 @@ def f(local fn: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1538,7 +1543,7 @@ for x in [1, 2, 3]: Default level: warn · Added in 0.0.50 · Related issues · -View source +View source @@ -1578,7 +1583,7 @@ def g(value: ~A) -> None: ... # error: [experimental-syntax] Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -1612,7 +1617,7 @@ def my_function() -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.40 · Related issues · -View source +View source @@ -1647,7 +1652,7 @@ let a = 1 Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -1764,7 +1769,7 @@ def test() -> "Literal[5]": Default level: ignore · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -1817,7 +1822,7 @@ unpacking — is not a declaration, and is never reported. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -1892,7 +1897,7 @@ print(Labelled) # warning: prints `` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -1931,7 +1936,7 @@ def audit(account: Account) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1967,7 +1972,7 @@ class C(A, B): ... # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1996,7 +2001,7 @@ t[3] # error Default level: warn · Added in 0.0.1-alpha.33 · Related issues · -View source +View source @@ -2033,7 +2038,7 @@ class MyClass: ... Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2069,7 +2074,7 @@ class Point: Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -2164,7 +2169,7 @@ will produce instances with an atypical memory layout. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2196,7 +2201,7 @@ func("foo") # error: [invalid-argument-type] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2226,7 +2231,7 @@ a: int = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2307,7 +2312,7 @@ box.value = 1 # okay Default level: error · Added in 0.0.33 · Related issues · -View source +View source @@ -2352,7 +2357,7 @@ class Sub(Base): Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -2394,7 +2399,7 @@ asyncio.run(main()) Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2421,7 +2426,7 @@ class A(42): ... # error: [invalid-base] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2459,7 +2464,7 @@ build: # error: `build` is an experimental feature, and is off Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.5 · Related issues · -View source +View source @@ -2496,7 +2501,7 @@ extension str(A): # error: `str` does not answer every member of `A` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2525,7 +2530,7 @@ with 1: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -2558,7 +2563,7 @@ class Fahrenheit: Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -2611,7 +2616,7 @@ See: Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -2647,7 +2652,7 @@ class A: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2679,7 +2684,7 @@ a: str # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.82 · Related issues · -View source +View source @@ -2750,7 +2755,7 @@ decorator def traced(fn: (...) -> object, label: str = "") -> int: Default level: warn · Added in 0.0.20 · Related issues · -View source +View source @@ -2806,7 +2811,7 @@ class Pet(Enum): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2871,7 +2876,7 @@ This rule corresponds to Ruff's Default level: error · Added in 0.0.1-alpha.28 · Related issues · -View source +View source @@ -2925,7 +2930,7 @@ class D(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -2958,7 +2963,7 @@ extension list[T: int]: # error: `list` declares no type parameter `T` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -2987,7 +2992,7 @@ Author.objects.filter(name__startswith=1) # error: lookup wants `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -3023,7 +3028,7 @@ def test_user(user: int) -> None: # error: fixture provides `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -3071,7 +3076,7 @@ f"{'name':>10}" # ok Default level: error · Added in 0.0.1-alpha.35 · Related issues · -View source +View source @@ -3121,7 +3126,7 @@ class NonFrozenChild(FrozenBase): # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3176,7 +3181,7 @@ class E(Generic[V]): Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -3271,7 +3276,7 @@ a = 20 / 0 # type: ignore Default level: error · Added in 0.0.1-alpha.17 · Related issues · -View source +View source @@ -3318,7 +3323,7 @@ carol = Person(name="Carol", aeg=25) # typo! Default level: warn · Added in 0.0.15 · Related issues · -View source +View source @@ -3379,7 +3384,7 @@ def f(x, y, /): # Python 3.8+ syntax Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3419,7 +3424,7 @@ def f(t: TypeVar("U")): ... # ty: ignore[invalid-type-form] Default level: error · Added in 0.0.18 · Related issues · -View source +View source @@ -3470,7 +3475,7 @@ match object(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3504,7 +3509,7 @@ class B(metaclass=42): ... # error Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -3621,7 +3626,7 @@ Correct use of `@override` is enforced by ty's [`invalid-explicit-override`](#in Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -3664,7 +3669,7 @@ implements Backend # error: `Backend` is not a protocol Default level: error · Added in 0.0.72 · Related issues · -View source +View source @@ -3702,7 +3707,7 @@ from module import missing # error Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -3767,7 +3772,7 @@ TypeError: typing.ClassVar[int] is not valid as type argument Default level: warn · Added in 0.0.31 · Related issues · -View source +View source @@ -3813,7 +3818,7 @@ admin[0] # "Alice" Default level: error · Added in 0.0.1-alpha.27 · Related issues · -View source +View source @@ -3851,7 +3856,7 @@ Baz = NewType("Baz", int | str) # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3928,7 +3933,7 @@ def foo(x: object = "a", y: int = 0) -> object: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -3966,7 +3971,7 @@ B().f() # 1, not 2 Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4021,7 +4026,7 @@ def h(a: S = 1) -> S: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4054,7 +4059,7 @@ def test_add(a: int, b: int) -> None: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4090,7 +4095,7 @@ P2 = ParamSpec() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4142,7 +4147,7 @@ Declare the type variable with `TypeVar("T", covariant=True)` instead. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4213,7 +4218,7 @@ def g(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -4248,7 +4253,7 @@ def g[T](value: T) raises T: # error: `T@g` is not always an exception Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4281,7 +4286,7 @@ if m := re.match("(a)(b)", "ab"): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -4313,7 +4318,7 @@ type Alias[reified T] = list[T] # error: an alias's parameters are erased Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.83 · Related issues · -View source +View source @@ -4384,7 +4389,7 @@ class Override(Base): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4530,7 +4535,7 @@ def detail(request, pk: int): ... # ok Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -4569,7 +4574,7 @@ import "data/missing.json" as missing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4677,7 +4682,7 @@ class C: ... Default level: error · Added in 0.0.10 · Related issues · -View source +View source @@ -4728,7 +4733,7 @@ class MyClass: Default level: error · Added in 0.0.1-alpha.6 · Related issues · -View source +View source @@ -4774,7 +4779,7 @@ InvalidAlias = TypeAliasType("InvalidAlias", list[T], type_params=(list[T],)) # Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -4840,7 +4845,7 @@ Bar[int] # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4873,7 +4878,7 @@ TYPE_CHECKING = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4908,7 +4913,7 @@ b: Annotated[int] # error Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -4965,7 +4970,7 @@ class C: Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -5031,7 +5036,7 @@ class Owner[U]: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5088,7 +5093,7 @@ V = TypeVar("V", list[int], int) # valid constrained Type Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -5130,7 +5135,7 @@ U = TypeVar("U", int, str, default=bytes) # error: [invalid-type-variable-defau Default level: error · Added in 0.0.28 · Related issues · -View source +View source @@ -5166,7 +5171,7 @@ class Child(Base): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5207,7 +5212,7 @@ def f(options: dict[str, object]): Default level: error · Added in 0.0.9 · Related issues · -View source +View source @@ -5241,7 +5246,7 @@ class Foo(TypedDict): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5282,7 +5287,7 @@ type Alias[out T] = list[T] # error: `list` is invariant Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -5322,7 +5327,7 @@ class Point: Default level: error · Added in 0.0.25 · Related issues · -View source +View source @@ -5356,7 +5361,7 @@ def gen() -> Iterator[int]: Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5422,7 +5427,7 @@ def h(arg2: type): Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -5471,7 +5476,7 @@ def g(arg: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -5502,7 +5507,7 @@ def f(s: str): Default level: warn · Added in 0.0.30 · Related issues · -View source +View source @@ -5544,7 +5549,7 @@ Movie = TypedDict("Film", {"title": str}) # error: [mismatched-type-name] Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -5612,7 +5617,7 @@ and nothing is reported. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5643,7 +5648,7 @@ func() # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5682,7 +5687,7 @@ def g(): ... Default level: ignore · Preview (since 0.0.76) · Related issues · -View source +View source @@ -5771,7 +5776,7 @@ Add `urllib3` to `project.dependencies` if your code imports it directly. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -5799,7 +5804,7 @@ from django.db import models # warning: install `django-stubs` for precise type Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.81 · Related issues · -View source +View source @@ -5851,7 +5856,7 @@ def ignore(event: str): ... Default level: error · Level under ty-compatible: ignore · Added in 0.0.41 · Related issues · -View source +View source @@ -5912,7 +5917,7 @@ class ExplicitChild(Parent): Default level: error · Added in 0.0.75 · Related issues · -View source +View source @@ -5999,7 +6004,7 @@ class Item: Default level: error · Level under ty-compatible: ignore · Added in 0.0.45 · Related issues · -View source +View source @@ -6037,7 +6042,7 @@ def handle(m: re.Match[str]) -> str: Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -6076,7 +6081,7 @@ alice["age"] # KeyError Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -6123,7 +6128,7 @@ def App(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6156,7 +6161,7 @@ def f(a: int | None): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6194,7 +6199,7 @@ func("string") # error: [no-matching-overload] Default level: error · Added in 0.0.30 · Related issues · -View source +View source @@ -6231,7 +6236,7 @@ class Sub(Super): ... # error: [non-callable-init-subclass] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -6262,7 +6267,7 @@ def f(x: int | str) -> int: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6291,7 +6296,7 @@ def f(a: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6335,7 +6340,7 @@ def g(o: object, shape: Shape): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6364,7 +6369,7 @@ for i in 34: # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6392,7 +6397,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6421,7 +6426,7 @@ def f(once done: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6449,7 +6454,7 @@ def f(once done: () -> None): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6491,7 +6496,7 @@ def f(x: int?): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6546,7 +6551,7 @@ def g(name: str | None): Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -6583,7 +6588,7 @@ class B(A): Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -6620,7 +6625,7 @@ class B(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.38 · Related issues · -View source +View source @@ -6663,7 +6668,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6694,7 +6699,7 @@ f(1, x=2) # error Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6725,7 +6730,7 @@ f(x=1) # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6763,7 +6768,7 @@ A.c # error Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6801,7 +6806,7 @@ A()[0] # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6845,7 +6850,7 @@ from module import a # error Default level: warn · Added in 0.0.23 · Related issues · -View source +View source @@ -6877,7 +6882,7 @@ html.parser # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6913,7 +6918,7 @@ print(x) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -6952,7 +6957,7 @@ Id("x") # error: `Id`'s constructor is private Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -6983,7 +6988,7 @@ __all__ = ["helper"] # error: `helper` is private Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7015,7 +7020,7 @@ from helpers import Key # error: `Key` is private to `helpers` Default level: warn · Added in 0.0.60 · Related issues · -View source +View source @@ -7090,7 +7095,7 @@ def test() -> "int": Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7132,7 +7137,7 @@ def g(a: bool | None): Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7167,7 +7172,7 @@ cast(int, f()) # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7221,7 +7226,7 @@ if sys.version_info >= (3, 12): # ok — artificially constant Default level: warn · Added in 0.0.18 · Related issues · -View source +View source @@ -7259,7 +7264,7 @@ class C: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7314,7 +7319,7 @@ class Sub(Base): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7363,7 +7368,7 @@ def f(value: int | str) -> int: Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -7427,7 +7432,7 @@ def g(values: tuple[int, ...]) -> None: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -7460,7 +7465,7 @@ class C: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -7496,7 +7501,7 @@ class C[T]: Default level: warn · Added in 0.0.71 · Related issues · -View source +View source @@ -7539,7 +7544,7 @@ def build(t: Tag) -> None: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -7583,7 +7588,7 @@ class Outer[T]: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7632,7 +7637,7 @@ def Observed(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7676,7 +7681,7 @@ def Counter(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7710,7 +7715,7 @@ static_assert(int(2.0 * 3.0) == 6) # error Default level: warn · Added in 0.0.39 · Related issues · -View source +View source @@ -7762,7 +7767,7 @@ limitation. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7796,7 +7801,7 @@ class B(A): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7828,7 +7833,7 @@ class Circle(Shape): ... # error: `Shape` is sealed in another workspace Default level: error · Added in 0.0.1-alpha.30 · Related issues · -View source +View source @@ -7937,7 +7942,7 @@ class Book: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7967,7 +7972,7 @@ f("foo") # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8006,7 +8011,7 @@ def find(items: list[int]) -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8053,7 +8058,7 @@ g: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8088,7 +8093,7 @@ f: # error: the block returns `None`, not `str` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8127,7 +8132,7 @@ def _(x: int): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -8158,7 +8163,7 @@ class User(BaseModel): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8217,7 +8222,7 @@ class A: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -8290,7 +8295,7 @@ the project registers with `@register.simple_block_tag`. Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -8356,7 +8361,7 @@ what the projects depending on it read. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8385,7 +8390,7 @@ def f() raises TypeError: Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8414,7 +8419,7 @@ reveal_type(1) # revealed: Literal[1] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8452,7 +8457,7 @@ def check(x: int | None) -> asserts x: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8482,7 +8487,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8513,7 +8518,7 @@ f(x=1, y=2) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -8724,7 +8729,7 @@ page does not render at all. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -8758,7 +8763,7 @@ implements Backend # error: this module does not answer `Backend` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8817,7 +8822,7 @@ def Held(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8850,7 +8855,7 @@ A().foo # error Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -8925,7 +8930,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8953,7 +8958,7 @@ import foo # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8982,7 +8987,7 @@ def check(value: int | None) -> asserts values: # error: `values` is nothing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9100,7 +9105,7 @@ is one whose template set cannot be established. Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -9228,7 +9233,7 @@ Python code. Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -9269,7 +9274,7 @@ def f(a: object, b: int, c: Any): Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9414,7 +9419,7 @@ Python code. Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9561,7 +9566,7 @@ generator boundaries. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -9611,7 +9616,7 @@ A() # error: nothing says which specialization this is Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -9658,7 +9663,7 @@ def Observed(items: StateList[int]): ... # ok: an observable handle Default level: warn · Added in 0.0.1-alpha.7 · Related issues · -View source +View source @@ -9704,7 +9709,7 @@ class D(C): ... # error: [unsupported-base] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9753,7 +9758,7 @@ b1 < b2 < b1 # error Default level: warn · Level under ty-compatible: ignore · Added in 0.0.12 · Related issues · -View source +View source @@ -9798,7 +9803,7 @@ def factory(base: type[Base]) -> type: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9830,7 +9835,7 @@ A() + A() # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -9873,7 +9878,7 @@ reveal_type(project.root) # revealed: "." Default level: warn · Added in 0.0.21 · Related issues · -View source +View source @@ -9946,13 +9951,93 @@ Set [`analysis.respect-type-ignore-comments`](https://docs.astral.sh/ty/reference/configuration/#respect-type-ignore-comments) to `false` to prevent this rule from reporting unused `type: ignore` comments. +## `unused-import` + + +Default level: warn · basedpython only, so absent under ty-compatible · +Added in 0.0.1-beta.2 · +Related issues · +View source + + + +**What it does** + +Checks a basedpython file for an import that nothing uses: no name it binds +is read, and no `extension` or conformance it brings into scope supplies +anything the file relies on. + +**Why is this bad?** + +An unused import costs time when the module is imported, can create an +import cycle, and tells a reader the module depends on something it does +not. + +The extension members a module declares are only applicable in the files +that import it, so an import can be used without its name ever being read. +This rule counts that as a use, which a linter working one file at a time +cannot — so in a `.by` file it takes the place of buff's `F401`, which +leaves `.by` files alone. + +Otherwise the rule is `F401`'s: + +- a read of a binding counts for every later binding of that name in the + scope. An import nothing reads before another import rebinds the name + is reported too, as `F811` reports the redefinition, so one round of + fixes removes every one. An import inside a class body is never + reported +- `import a as a`, `from a import b as b` and `from a export b` are + re-exports, a name listed in `__all__` is exported, and + `from __future__ import …` is a directive — none of them is reported +- in a package's `__init__.by`, and inside a `try` that handles + `ImportError`, an unused import is reported without a fix: it may be a + re-export, or a test for whether the module is installed + +**Examples** + +```by +# colours.by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +```by +# main.by +import json # error: `json` imported but unused +import colours # ok — the extension it declares is used below +from enum import Enum + +class Colour(Enum): + RED = 1 + +def describe(colour: Colour) -> str: + return colour.lower_name +``` + +Use instead: + +```by +import colours +from enum import Enum +``` + +**Fix safety** + +The fix removes the import, and is safe: nothing in the file reads it. +Removing an import also stops the imported module from being run, which +matters only if the module's import has a side effect the program relies +on. + ## `unused-return-value` Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.71 · Related issues · -View source +View source @@ -10038,7 +10123,7 @@ to `false`. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.81 · Related issues · -View source +View source @@ -10083,7 +10168,7 @@ parse("1") Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -10162,7 +10247,7 @@ def foo(x: int | str) -> int | str: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index eabf23b338..43d732428a 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -238,7 +238,7 @@ pub(crate) fn cmd_run( let mut traceback_entries: Vec = Vec::new(); let mut staging = Staging::new(tmp.path()); let mut layout = RuntimeLayout::default(); - let ok = render_check_and_transpile( + let status = render_check_and_transpile( &db, &handles, &mut Emit { @@ -256,8 +256,8 @@ pub(crate) fn cmd_run( Ok(()) }, )?; - if !ok { - return Ok(ExitStatus::Failure); + if !matches!(status, ExitStatus::Success) { + return Ok(status); } // a program is its data as much as its modules: a `.py` module it imports, a // json file it opens, a template it renders. running out of a directory @@ -960,7 +960,7 @@ pub(crate) fn cmd_build( let mut entries: Vec = Vec::new(); let mut requirements = by_transforms::RuntimeRequirements::default(); let mut layout = RuntimeLayout::default(); - let ok = render_check_and_transpile( + let status = render_check_and_transpile( &db, &handles, &mut Emit { @@ -1003,8 +1003,9 @@ pub(crate) fn cmd_build( } staging.finish()?; - if !ok? { - return Ok(ExitStatus::Failure); + let status = status?; + if !matches!(status, ExitStatus::Success) { + return Ok(status); } eprintln!("\nbuild complete ({file_count} files)"); Ok(ExitStatus::Success) @@ -1337,17 +1338,17 @@ pub(crate) fn cmd_compile( Err(error) => break 'compiling Err(error), }; - let program_file = ty_python_semantic::Db::program_file(&db, *file); + let program_file = ty_python_semantic::Db::program_file(&*db, *file); let parsed = - ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); - let model = ty_python_semantic::SemanticModel::new(&db, program_file); + ruff_db::parsed::parsed_module(&*db, program_file.python_file(&*db)).load(&*db); + let model = ty_python_semantic::LoweringModel::new(&*db, program_file); // a `.py` source needs no transpiling to be its own interpreted fallback let mut options = options.clone(); if path.extension().is_some_and(|x| x == "py") { options.language = by_irbuild::Language::Python; } let mut lowered = by_irbuild::build_module( - &db, + &*db, &model.program_environment(), &model, parsed.suite(), @@ -1458,8 +1459,9 @@ pub(crate) fn cmd_compile( // the transpile's verdict is answered after the tree is finished, the way // `build` answers it: a diagnostic must not cost the tree, because a `build/` // a debugger cannot read is worse than one built from a partial check - if !transpiled? { - return Ok(ExitStatus::Failure); + let transpiled = transpiled?; + if !matches!(transpiled, ExitStatus::Success) { + return Ok(transpiled); } eprintln!("\ncompiled {compiled} module(s)"); @@ -1719,7 +1721,7 @@ fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { } let roots = module_roots(&db, &root); - let ok = render_check_and_transpile( + render_check_and_transpile( &db, &handles, &mut Emit { @@ -1739,12 +1741,7 @@ fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { fs::write(&py, emitted.python).with_context(|| format!("{}", py.display()))?; Ok(()) }, - )?; - Ok(if ok { - ExitStatus::Success - } else { - ExitStatus::Failure - }) + ) } /// Every first-party `.py`/`.pyi` file under `root`, skipping non-source @@ -2275,9 +2272,9 @@ fn compilable_files(root: &Path) -> Vec { } /// Check every file, render diagnostics, then for each non-blocked file call -/// `consume` with the transpiled Python. Returns `Ok(false)` if the check -/// outcome blocks per `gate`, or a transpiler bug occurred (caller should -/// propagate failure). +/// `consume` with the transpiled Python. Answers the exit status the command +/// ends with: a failure when something was reported, and an internal error when +/// that was `by` itself panicking, the way `by check` tells the two apart /// /// The deciding is [`by_stage::emit::check_and_transpile`], and only the /// **rendering** is here. That is the whole difference between the two callers @@ -2291,12 +2288,18 @@ fn render_check_and_transpile( handles: &[(PathBuf, ruff_db::files::File)], emit: &mut Emit<'_>, consume: impl FnMut(&Transpiled<'_>) -> anyhow::Result<()>, -) -> anyhow::Result { +) -> anyhow::Result { let emitted = by_stage::emit::check_and_transpile(db, handles, emit, consume)?; if !emitted.diagnostics.is_empty() { render_diagnostics(db, &emitted.diagnostics)?; } - Ok(emitted.ok) + Ok(if emitted.ok { + ExitStatus::Success + } else if emitted.diagnostics.iter().any(|d| d.severity().is_fatal()) { + ExitStatus::InternalError + } else { + ExitStatus::Failure + }) } /// Render diagnostics to stderr in the same format as `by check`. The diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index ac19499a8a..8a58a3b049 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -69,12 +69,13 @@ where // type inference recurses with the shape of the program it is checking, so // how deep a file it can survive is decided by the stack it runs on. the // rayon pool asks for `STACK_SIZE`, but the thread a process starts on gets a - // platform default — 1 MiB on windows — and the commands that check on the - // calling thread rather than through the pool (`run`, `build`, `transpile`, - // `compile`) were overflowing it there. so the whole command runs on a thread - // this codebase has sized for the job, wherever it is dispatched to. a command - // does more per term than a check does, since it lowers the expression as well - // as inferring it, and `STACK_SIZE` is sized to leave room for that + // platform default — 1 MiB on windows — and a command that infers on the + // calling thread rather than through the pool (a single-file `transpile`, + // `compile`'s check and lowering) overflowed it there. so the whole command + // runs on a thread this codebase has sized for the job, wherever it is + // dispatched to. a command does more per term than a check does, since it + // lowers the expression as well as inferring it, and `STACK_SIZE` is sized to + // leave room for that std::thread::scope(|scope| { let command = std::thread::Builder::new() .stack_size(STACK_SIZE) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 833d244324..7a24d1e7c4 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1,3 +1,4 @@ +use std::fmt::Write as _; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; @@ -3434,6 +3435,62 @@ fn build_emits_every_file_it_can_past_a_broken_one() { assert!(stderr.contains("broken.by"), "got:\n{stderr}"); } +/// the modules are transpiled in parallel and handed over in source order. a +/// build reports every module it could not emit, in that order, while a run +/// stops at the first one: the program cannot run without it, so nothing after +/// it is reported or run +#[test] +fn transpile_failures_are_reported_in_source_order() { + let dir = tempfile::tempdir().expect("tempdir"); + // each module takes over the name the runtime's `Some(...)` wrapper is + // provided under, which the transpile refuses. `by check` has nothing to say + // about it, so a run gets as far as transpiling + let refused = "from typing import Optional\n\nx = Some(3)\nprint(x!)\n"; + // the first module is the larger, so it is the last to finish: an order + // the transpiles finished in would put it after `b.by` + let mut larger = String::new(); + for i in 0..400 { + let _ = write!(larger, "\ndef f{i}(x: int) -> int:\n return x + {i}\n"); + } + fs::write(dir.path().join("a.by"), format!("{refused}{larger}")).unwrap(); + fs::write(dir.path().join("b.by"), refused).unwrap(); + fs::write(dir.path().join("main.by"), "print('main ran')\n").unwrap(); + let by = |args: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(args) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by") + }; + let refusals = |stderr: &str| -> Vec { + let lines: Vec<&str> = stderr.lines().collect(); + lines + .windows(2) + .filter(|pair| pair[0].starts_with("error[transpile-refused]")) + .map(|pair| pair[1].trim().to_owned()) + .collect() + }; + + let build = by(&["build"]); + let stderr = String::from_utf8_lossy(&build.stderr); + assert!(!build.status.success(), "{stderr}"); + assert_eq!( + refusals(&stderr), + ["--> a.by:1:20", "--> b.by:1:20"], + "got:\n{stderr}" + ); + + let run = by(&["run", "main"]); + let stderr = String::from_utf8_lossy(&run.stderr); + assert!(!run.status.success(), "{stderr}"); + assert_eq!(refusals(&stderr), ["--> a.by:1:20"], "got:\n{stderr}"); + assert!( + !String::from_utf8_lossy(&run.stdout).contains("main ran"), + "the program ran past a module it could not transpile" + ); +} + /// `by build` walks the project's own file set, so `src.exclude` applies to it /// exactly as it does to `by check` #[test] @@ -4887,9 +4944,10 @@ def main(): reason = "a skipped test prints why it was skipped" )] fn a_lazy_from_import_resolves_a_submodule_and_refuses_a_missing_name_as_python_does() { - // `_LazyAttr` defers the attribute read, and reading an attribute is not how a - // submodule gets bound: `urllib/__init__.py` never imports `parse`, and cpython - // binds it only because `__import__` is handed a fromlist. so a transpiled + // a deferred `from` import reads an attribute off the module on first use, and + // reading an attribute is not how a submodule gets bound: `urllib/__init__.py` + // never imports `parse`, and cpython binds it only because `__import__` is + // handed a fromlist. so a transpiled // `from urllib import parse` used to raise `AttributeError` where the same source // run by cpython is fine — a wrong answer in shipped output rather than a decline // @@ -4940,7 +4998,7 @@ fn a_lazy_from_import_resolves_a_submodule_and_refuses_a_missing_name_as_python_ let program = String::from_utf8_lossy(&transpiled.stdout).into_owned(); // the laziness is the thing under test, so its absence must not pass silently assert!( - program.contains("_lazy_attr(\"urllib\", \"parse\")"), + program.contains("_lazy_from(_by_lazy, \"urllib\", {\"parse\": \"parse\"})"), "{program}" ); fs::write(dir.path().join("prog.py"), &program).unwrap(); @@ -6672,9 +6730,9 @@ fn a_built_package_runs_off_one_copy_of_the_runtime() { !one.contains("def _checked_cast("), "and does not define it as well:\n{one}" ); - // a helper reached through the lazy-import proxy is a proxy call on every use + // a deferred helper would be an attribute read on every use assert!( - !one.contains("_lazy_attr(\"app._by_runtime\""), + !one.contains("\"app._by_runtime\""), "the runtime import is eager:\n{one}" ); // the lowering asks for helpers in two places — `cast!` during the AST rewrite, the diff --git a/crates/ty/tests/cli/fixes.rs b/crates/ty/tests/cli/fixes.rs index 21424a58e0..5b50d94dfd 100644 --- a/crates/ty/tests/cli/fixes.rs +++ b/crates/ty/tests/cli/fixes.rs @@ -202,6 +202,100 @@ fn fix() -> anyhow::Result<()> { Ok(()) } +/// `--fix` removes the imports nothing in a `.by` file uses, and keeps the one used only +/// through the extension its module declares +#[test] +fn fix_unused_imports() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ( + "colours.by", + r#" + from enum import Enum + + extension Enum: + let lower_name: str + get() = self.name.lower() + "#, + ), + ( + "main.by", + r#" + import json + from enum import Enum + import colours + from collections import OrderedDict, deque + + class Colour(Enum): + RED = 1 + + def describe(colour: Colour, seen: OrderedDict[str, int]) -> str: + return colour.lower_name + "#, + ), + ])?; + + assert_cmd_snapshot!(case.command().arg("--fix"), @" + success: true + exit_code: 0 + ----- stdout ----- + Found 2 diagnostics (2 fixed, 0 remaining). + + ----- stderr ----- + "); + + assert_snapshot!(fs::read_to_string(case.root().join("main.by"))?, @" + + from enum import Enum + import colours + from collections import OrderedDict + + class Colour(Enum): + RED = 1 + + def describe(colour: Colour, seen: OrderedDict[str, int]) -> str: + return colour.lower_name + "); + + // the fixed file checks clean, the extension still resolving + assert_cmd_snapshot!(case.command(), @" + success: true + exit_code: 0 + ----- stdout ----- + All checks passed! + + ----- stderr ----- + "); + + Ok(()) +} + +/// a name imported many times over is fixed in one round, however many times: every unread +/// import of it is reported at once, so the fixes do not converge one import per round. a +/// block left with nothing in it is left with `pass` and nothing else +#[test] +fn fix_unused_imports_in_one_round() -> anyhow::Result<()> { + let repeated = "import os\n".repeat(12); + let source = format!("{repeated}\nif True:\n import json; import sys\n"); + let case = CliTest::with_file("main.by", &source)?; + + assert_cmd_snapshot!(case.command().arg("--fix"), @" + success: true + exit_code: 0 + ----- stdout ----- + Found 14 diagnostics (14 fixed, 0 remaining). + + ----- stderr ----- + "); + + assert_snapshot!(fs::read_to_string(case.root().join("main.by"))?, @" + + if True: + pass + "); + + Ok(()) +} + #[test] fn fix_unfixable() -> anyhow::Result<()> { let case = CliTest::with_files([ diff --git a/crates/ty/tests/mdtest_divergence.rs b/crates/ty/tests/mdtest_divergence.rs index c824375074..3135088749 100644 --- a/crates/ty/tests/mdtest_divergence.rs +++ b/crates/ty/tests/mdtest_divergence.rs @@ -9,6 +9,15 @@ //! //! blocks carrying expected diagnostics are skipped — their runtime behaviour //! is intentionally unspecified. +//! +//! a second leg holds `unused-import` to the same contract: each such block has its unused +//! imports removed by `by check --fix`, and must then check the same and do the same at +//! runtime — every soundness check on, so a check that reads an import is run. a removed +//! import whose binding the transpiled program still reads, because the lowering spelled a +//! name the checker never recorded a read of, shows up as a program that behaves +//! differently, or no longer checks. it sees only what this corpus exercises: an import read +//! only by a lowering, in a checker-clean block of a single file. a multi-file section is not +//! run, so a read across modules is out of its reach, as is any lowering no such block makes use std::fs; use std::io::Write; @@ -147,13 +156,17 @@ fn transpile(source: &str, min_version: &str) -> Result { // full of stub bodies (`def f[T]() -> T: ...`) that legitimately violate // their annotations when executed — the checks would report those as // divergences rather than transpiler bugs, so this harness opts out + transpile_with(source, min_version, "none") +} + +fn transpile_with(source: &str, min_version: &str, soundness: &str) -> Result { let mut child = Command::new(env!("CARGO_BIN_EXE_by")) .args([ "transpile", "--min-version", min_version, "--soundness", - "none", + soundness, ]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -174,6 +187,112 @@ fn transpile(source: &str, min_version: &str) -> Result { } } +/// the error-level diagnostics `by check` reports for the module `dir/main.by`, without their +/// positions, which removing an import shifts +fn check_errors(dir: &Path) -> Vec { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args([ + "check", + "--output-format", + "concise", + "--exit-zero", + "--no-progress", + ]) + .arg("main.by") + .current_dir(dir) + .output() + .expect("failed to spawn by"); + let mut errors: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .filter(|line| line.contains(": error[")) + .map(|line| line.splitn(4, ':').nth(3).unwrap_or(line).trim().to_owned()) + .collect(); + errors.sort(); + errors +} + +/// what the module `source` does when it is transpiled with every soundness check on and +/// run as `dir/main.py`: whether it transpiles, its exit status and its output, with `dir` +/// left out so two directories compare +fn behaviour(source: &str, min_version: &str, python: &str, dir: &Path) -> String { + let transpiled = match transpile_with(source, min_version, "all") { + Ok(transpiled) => transpiled, + Err(error) => return format!("transpile failed:\n{error}"), + }; + let py = dir.join("main.py"); + fs::write(&py, with_reveal_stub(&transpiled)).unwrap(); + let run = Command::new(python) + .arg(&py) + .current_dir(dir) + .output() + .expect("failed to run python"); + let text = format!( + "status {:?}\n{}\n{}", + run.status.code(), + String::from_utf8_lossy(&run.stdout), + String::from_utf8_lossy(&run.stderr), + ); + text.replace(&dir.to_string_lossy().into_owned(), "") +} + +/// the second leg: remove the block's unused imports with `by check --fix`, and require the +/// result to check and behave as the block did. `None` when it does, or when there was +/// nothing to remove +fn unused_import_leg( + block: &str, + min_version: &str, + python: &str, + root: &Path, + key: &str, +) -> Option { + let project = format!( + "[project]\nname = \"leg\"\nversion = \"0\"\nrequires-python = \">={min_version}\"\n" + ); + let original = root.join(format!("{key}_written")); + let fixed = root.join(format!("{key}_fixed")); + for dir in [&original, &fixed] { + fs::create_dir_all(dir).unwrap(); + fs::write(dir.join("pyproject.toml"), &project).unwrap(); + fs::write(dir.join("main.by"), block).unwrap(); + } + Command::new(env!("CARGO_BIN_EXE_by")) + .args([ + "check", + "--ignore", + "all", + "--warn", + "unused-import", + "--fix", + "--exit-zero", + "--no-progress", + ]) + .arg("main.by") + .current_dir(&fixed) + .output() + .expect("failed to spawn by"); + let without = fs::read_to_string(fixed.join("main.by")).unwrap(); + if without == block { + return None; + } + let checked = (check_errors(&original), check_errors(&fixed)); + if checked.0 != checked.1 { + return Some(format!( + "removing its unused imports changed what the checker reports:\nbefore: {:?}\nafter: {:?}\n--- after ---\n{without}", + checked.0, checked.1 + )); + } + let behaved = ( + behaviour(block, min_version, python, &original), + behaviour(&without, min_version, python, &fixed), + ); + (behaved.0 != behaved.1).then(|| { + format!( + "removing its unused imports changed what the program does:\n--- before ---\n{}\n--- after ---\n{}\n--- without the imports ---\n{without}", + behaved.0, behaved.1 + ) + }) +} + /// Stub `reveal_type` (an mdtest debugging device with no runtime binding) /// after the `__future__` import, which must stay first. fn with_reveal_stub(transpiled: &str) -> String { @@ -406,6 +525,15 @@ fn clean_mdtest_blocks_run() { "{name} block {i}: checker-clean block crashed at runtime:\n{}\n--- block ---\n{block}", String::from_utf8_lossy(&run.stderr) )); + continue; + } + let key = format!("{}_{i}", name.trim_end_matches(".md").replace('-', "_")); + if let Some(divergence) = + unused_import_leg(block, &version, &python, tmp.path(), &key) + { + failures.lock().unwrap().push(format!( + "{name} block {i}: {divergence}\n--- block ---\n{block}" + )); } } }); diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index adeeedf665..80968a995e 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -6558,6 +6558,103 @@ class A: "#); } + #[test] + fn semantic_tokens_soft_keyword_parameter_after_a_modifier() { + // a soft keyword after a parameter's modifiers is the parameter's name + let test = SemanticTokenTest::new_by( + " +class A: + init(let cast: int, var match: int) + +def f(local type: int, once lazy: int): ... +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "A" @ 7..8: Class [definition] + "init" @ 14..18: Keyword + "cast" @ 23..27: Parameter [definition] + "int" @ 29..32: Class + "match" @ 38..43: Parameter [definition] + "int" @ 45..48: Class + "f" @ 55..56: Function [definition] + "local" @ 57..62: Keyword + "type" @ 63..67: Parameter [definition] + "int" @ 69..72: Class + "once" @ 74..78: Keyword + "lazy" @ 79..83: Parameter [definition] + "int" @ 85..88: Class + "#); + } + + #[test] + fn semantic_tokens_soft_keyword_after_a_callable_type_modifier() { + // a soft keyword after a callable-type parameter's modifier is the type it + // borrows, while `local cast int` stays a cast of `local` + let test = SemanticTokenTest::new_by( + " +def f(make: (local type[int], once str) -> None): ... + +local = 1 +x = (local cast int) +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "f" @ 5..6: Function [definition] + "make" @ 7..11: Parameter [definition] + "local" @ 14..19: Keyword + "type" @ 20..24: Class + "int" @ 25..28: Class + "once" @ 31..35: Keyword + "str" @ 36..39: Class + "None" @ 44..48: BuiltinConstant + "local" @ 56..61: Variable [definition] + "1" @ 64..65: Number + "x" @ 66..67: Variable [definition] + "local" @ 71..76: Variable + "cast" @ 77..81: Keyword + "int" @ 82..85: Class + "#); + } + + #[test] + fn semantic_tokens_soft_keyword_after_a_declaration_keyword() { + // a soft keyword after `protocol`, `newtype` or `enum class` is the name the + // declaration introduces + let test = SemanticTokenTest::new_by( + " +protocol match: + x: int + +newtype lazy = int + +enum class case: + case A +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "protocol" @ 1..9: Keyword + "match" @ 10..15: Class [definition] + "x" @ 21..22: Variable [definition] + "int" @ 24..27: Class + "newtype" @ 29..36: Keyword + "lazy" @ 37..41: Variable [definition] + "int" @ 44..47: Class + "enum class" @ 49..59: Keyword + "case" @ 60..64: Class [definition] + "case" @ 70..74: Keyword + "A" @ 75..76: Class [definition] + "#); + } + #[test] fn semantic_tokens_super_keyword() { let test = SemanticTokenTest::new_by( diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index 8f3cd91631..74d2288f3a 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -19,6 +19,7 @@ use ruff_db::diagnostic::{ Annotation, Diagnostic, DiagnosticId, Severity, Span, SubDiagnostic, SubDiagnosticSeverity, }; use ruff_db::files::{File, system_path_to_file}; +use ruff_db::panic::PanicActivity; use ruff_db::parsed::parsed_module; use ruff_db::system::{SystemPath, SystemPathBuf, deduplicate_nested_paths}; use rustc_hash::FxHashSet; @@ -1092,7 +1093,7 @@ pub(crate) fn check_file_impl( let source_file = file.file(db); { let db = AssertUnwindSafe(db); - match catch(&**db, source_file, || { + match catch_panic(&**db, source_file, PanicActivity::Checking, || { let script = Script::for_file(*db, source_file); if let Some(script) = script && !script.has_valid_settings(*db) @@ -1196,7 +1197,17 @@ impl Iterator for ProjectFilesIter<'_> { impl FusedIterator for ProjectFilesIter<'_> {} -fn catch(db: &dyn Db, file: File, f: F) -> Result +/// Run `f`, the work of `activity` on `file`, and report a panic in it as a `panic` diagnostic +/// against the file rather than unwinding. +/// +/// A cancellation is not a panic and is resumed, except for a panic propagated from a query +/// another thread was running, which is reported against this file too. +pub fn catch_panic( + db: &dyn Db, + file: File, + activity: PanicActivity, + f: F, +) -> Result where F: FnOnce() -> R + UnwindSafe, { @@ -1227,7 +1238,7 @@ where } } - let message = error.to_diagnostic_message(Some(file.path(db))); + let message = error.to_diagnostic_message(activity, Some(file.path(db))); let mut diagnostic = Diagnostic::new(DiagnosticId::Panic, Severity::Fatal, message); diagnostic.add_bug_sub_diagnostics("%5Bpanic%5D"); @@ -1269,17 +1280,59 @@ where #[cfg(test)] mod tests { - use crate::check_file_impl; use crate::db::Db as _; use crate::db::testing::TestDb; use crate::{IncludeResult, ProjectMetadata}; + use crate::{catch_panic, check_file_impl}; + use ruff_db::diagnostic::DiagnosticId; use ruff_db::files::system_path_to_file; + use ruff_db::panic::PanicActivity; use ruff_db::source::source_text; use ruff_db::system::{DbWithTestSystem, DbWithWritableSystem as _, SystemPath, SystemPathBuf}; use ruff_db::testing::assert_function_query_was_not_run; use ty_python_semantic::Db as _; use ty_python_semantic::types::check_types; + /// a query this file's work waited on panicked on another thread: that is a panic to + /// report against this file too, not a cancellation to hand back to the caller + #[test] + fn a_panic_propagated_from_another_thread_is_reported() -> ruff_db::system::Result<()> { + let project = ProjectMetadata::new("test", SystemPathBuf::from("/")); + let mut db = TestDb::new(project); + db.write_file("test.py", "")?; + let file = system_path_to_file(&db, "test.py").unwrap(); + + let caught = catch_panic(&db, file, PanicActivity::Transpiling, || { + std::panic::resume_unwind(Box::new(salsa::Cancelled::PropagatedPanic)) + }); + + let diagnostic = caught.expect_err("the propagated panic is reported"); + assert_eq!(diagnostic.id(), DiagnosticId::Panic); + let message = diagnostic.headline_message(); + assert!(message.contains("when transpiling"), "{message}"); + Ok(()) + } + + /// a pending write cancels the work, and the caller is the one that knows what to do about + /// that, so it goes back unchanged + #[test] + fn a_cancellation_is_handed_back() -> ruff_db::system::Result<()> { + let project = ProjectMetadata::new("test", SystemPathBuf::from("/")); + let mut db = TestDb::new(project); + db.write_file("test.py", "")?; + let file = system_path_to_file(&db, "test.py").unwrap(); + + let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + catch_panic(&db, file, PanicActivity::Transpiling, || { + std::panic::resume_unwind(Box::new(salsa::Cancelled::PendingWrite)) + }) + })); + + let payload = unwound.expect_err("the cancellation unwound past the catch"); + assert!(payload.downcast_ref::().is_some()); + Ok(()) + } + #[test] fn check_file_skips_type_checking_when_file_cant_be_read() -> ruff_db::system::Result<()> { let project = ProjectMetadata::new("test", SystemPathBuf::from("/")); diff --git a/crates/ty_python_core/src/definition.rs b/crates/ty_python_core/src/definition.rs index 60904b433e..3a8776141c 100644 --- a/crates/ty_python_core/src/definition.rs +++ b/crates/ty_python_core/src/definition.rs @@ -1463,6 +1463,11 @@ pub struct ImportDefinitionKind { } impl ImportDefinitionKind { + /// which of the statement's aliases this definition is the binding of + pub fn alias_index(&self) -> usize { + self.alias_index as usize + } + pub fn import<'ast>(&self, module: &'ast ParsedModuleRef) -> &'ast ast::StmtImport { self.node.node(module) } @@ -1484,6 +1489,11 @@ pub struct ImportFromDefinitionKind { } impl ImportFromDefinitionKind { + /// which of the statement's aliases this definition is the binding of + pub fn alias_index(&self) -> usize { + self.alias_index as usize + } + pub fn import<'ast>(&self, module: &'ast ParsedModuleRef) -> &'ast ast::StmtImportFrom { self.node.node(module) } diff --git a/crates/ty_python_core/src/use_def.rs b/crates/ty_python_core/src/use_def.rs index 77efdb903a..6523883f1a 100644 --- a/crates/ty_python_core/src/use_def.rs +++ b/crates/ty_python_core/src/use_def.rs @@ -723,7 +723,8 @@ enum DefinitionEntry<'db> { Unused(Definition<'db>), Used(Definition<'db>), Undefined, - Deleted, + /// the place a `del`, or the end of an `except … as` clause, unbound + Deleted(ScopedPlaceId), } impl<'db> DefinitionEntry<'db> { @@ -733,7 +734,7 @@ impl<'db> DefinitionEntry<'db> { | Self::Unused(definition) | Self::Used(definition) => DefinitionState::Defined(definition), Self::Undefined => DefinitionState::Undefined, - Self::Deleted => DefinitionState::Deleted, + Self::Deleted(_) => DefinitionState::Deleted, } } } @@ -991,7 +992,22 @@ impl<'db> UseDefMap<'db> { DefinitionEntry::Used(definition) => Some((id, definition, true)), DefinitionEntry::DeclarationPart(_) | DefinitionEntry::Undefined - | DefinitionEntry::Deleted => None, + | DefinitionEntry::Deleted(_) => None, + }) + } + + /// every place this scope unbinds, with the point in the scope's history it does: a + /// `del`, the end of an `except … as` clause, and the end of a block that declared a + /// name. ordered with [`Self::definitions_with_usage`] by the ids both hand out + pub fn deletions(&self) -> impl Iterator + '_ { + self.all_definitions + .iter_enumerated() + .filter_map(|(id, entry)| match entry { + DefinitionEntry::Deleted(place) => Some((id, place)), + DefinitionEntry::DeclarationPart(_) + | DefinitionEntry::Unused(_) + | DefinitionEntry::Used(_) + | DefinitionEntry::Undefined => None, }) } @@ -2647,7 +2663,7 @@ impl<'db> UseDefMapBuilder<'db> { } pub(super) fn delete_binding(&mut self, place: ScopedPlaceId) { - let def_id = self.push_definition(DefinitionEntry::Deleted); + let def_id = self.push_definition(DefinitionEntry::Deleted(place)); let pending = self.pending_reachability.current; let place_state = pending_place_state_mut(place, &mut self.symbol_states, &mut self.member_states); diff --git a/crates/ty_python_semantic/Cargo.toml b/crates/ty_python_semantic/Cargo.toml index 756b33fea8..fc0504d550 100644 --- a/crates/ty_python_semantic/Cargo.toml +++ b/crates/ty_python_semantic/Cargo.toml @@ -19,6 +19,7 @@ ruff_macros = { workspace = true } ruff_memory_usage = { workspace = true } ruff_python_ast = { workspace = true, features = ["salsa"] } ruff_python_edits = { workspace = true } +ruff_python_index = { workspace = true } ruff_python_literal = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } diff --git a/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md b/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md index 2db20e4684..3250523d90 100644 --- a/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md +++ b/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md @@ -236,3 +236,22 @@ a: (name: str, age: int) = ("a", 0) b: (name: str, age: int) = ("b", 1) b = a ``` + +## A named tuple the program makes is never anonymous + +Only an anonymous named tuple form makes an anonymous named tuple. A `NamedTuple` class the program +makes itself is an ordinary named tuple whatever it is called, even when its name is the one +basedpython gives the class it synthesizes, so a plain tuple is not read as an instance of it. + +```by +from typing import NamedTuple + +def instances[T](cls: type[T]) -> list[T]: + return [] + +xs = instances(NamedTuple("_AnonNamedTuple_deadbeef", [("a", int)])) +reveal_type(xs) # revealed: list[_AnonNamedTuple_deadbeef] + +# error: [invalid-argument-type] +xs.append((1,)) +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_anonymous_named_tuple_literals.md b/crates/ty_python_semantic/resources/mdtest/basedpython_anonymous_named_tuple_literals.md new file mode 100644 index 0000000000..604e9ef371 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_anonymous_named_tuple_literals.md @@ -0,0 +1,303 @@ +# basedpython: a plain tuple literal built as an anonymous named tuple + +a plain tuple literal standing where an [anonymous named tuple](anonymous_named_tuple.md) is +expected is an instance of it, and the transpiler constructs that class around it, so the fields the +checker accepts reading are there at runtime. every block here runs + +## Every position that gives the literal an expected type + +the expectation can come from anywhere bidirectional inference reaches: an argument, a keyword +argument, an element of `*args`, a default, a `yield`, an element or value of a collection literal +at any depth, a method call on a collection, or a write to an attribute. each value below has its +fields at runtime + +```by +from typing import Iterator + +def take(p: (a: int, b: int)) -> int: + return p.a + +def keyword(*, p: (a: int, b: int)) -> int: + return p.b + +def star(*ps: (a: int, b: int)) -> int: + return sum(p.a for p in ps) + +def default(p: (a: int, b: int) = reveal_type((5, 6))) -> int: # revealed: (a: int, b: int) + return p.a + +def pairs() -> Iterator[(a: int, b: int)]: + yield reveal_type((1, 2)) # revealed: (a: int, b: int) + +class Box: + var pair: (a: int, b: int) = (0, 0) + +assert take(reveal_type((7, 8))) == 7 # revealed: (a: int, b: int) +assert keyword(p=reveal_type((7, 8))) == 8 # revealed: (a: int, b: int) +assert star(reveal_type((1, 2)), (3, 4)) == 4 # revealed: (a: int, b: int) +assert default() == 5 +assert next(pairs()).b == 2 + +items: list[(a: int, b: int)] = [] +items.append(reveal_type((1, 2))) # revealed: (a: int, b: int) +assert items[0].a == 1 + +by_name: dict[str, (a: int, b: int)] = {"k": reveal_type((1, 2))} # revealed: (a: int, b: int) +assert by_name["k"].b == 2 + +nested: list[list[(a: int, b: int)]] = [[reveal_type((1, 2))]] # revealed: (a: int, b: int) +assert nested[0][0].a == 1 + +box = Box() +box.pair = reveal_type((3, 4)) # revealed: (a: int, b: int) +assert box.pair.b == 4 +``` + +where a plain tuple type is expected, a literal stays a plain tuple: + +```by +t: tuple[int, int] = reveal_type((3, 4)) # revealed: (3, 4) +``` + +## A field that is itself an anonymous named tuple + +the literal for such a field is expected as that field's type, so it is built as the inner class: + +```by +def outer(p: (inner: (x: int, y: int), label: str)) -> int: + return p.inner.y + +assert outer(((1, 2), "a")) == 2 +``` + +## One arm of a union + +an optional anonymous named tuple builds the literal as its one anonymous named tuple arm: + +```by +maybe: (a: int) | None = reveal_type((1,)) # revealed: (a: int) +assert maybe.a == 1 +``` + +where the expected type allows several anonymous named tuples with as many fields as the literal +has, the one whose field types accept its elements is built: + +```by +either: (a: int) | (b: str) = reveal_type(("x",)) # revealed: (b: str) +assert either.b == "x" + +other: (a: int) | (b: str) = reveal_type((1,)) # revealed: (a: int) +assert other.a == 1 +``` + +an anonymous named tuple arm that does not accept the literal leaves it a plain tuple, inferred +against the rest of the expected type: + +```by +loose: (a: list[int]) | tuple[list[object]] = ([1, "s"],) +assert loose == ([1, "s"],) +``` + +an anonymous named tuple is a subtype of the plain tuple of its field types, so a union with that +tuple is the tuple alone, and the literal stays a plain tuple: + +```by +plain: (a: int) | tuple[int] = reveal_type((1,)) # revealed: (1,) +assert type(plain).__name__ == "tuple" +``` + +## A literal more than one arm accepts is an error + +the class decides which field names the value has at runtime, and nothing in a plain tuple says +which one was meant, so a literal more than one anonymous named tuple accepts is reported rather +than built as whichever the union lists first: + +```by +# error: [ambiguous-conversion] "More than one anonymous named tuple accepts this tuple" +either: (a: int) | (b: int) = (1,) +``` + +## A write through a union receiver whose members agree + +one value reaches whichever class the receiver turns out to be. where each declares the same +anonymous named tuple, the literal is built as it: + +```by +class Left: + var pair: (a: int, b: int) = (0, 0) + +class Right: + var pair: (a: int, b: int) = (0, 0) + +def assign(target: Left | Right) -> None: + target.pair = reveal_type((1, 2)) # revealed: (a: int, b: int) + assert target.pair.a == 1 + +assign(Left()) +assign(Right()) +``` + +## A write through a union receiver whose members build the same class + +the members need not declare the same type, only build the same class: one that may also hold `None` +builds the literal as the same anonymous named tuple + +```by +type P = (a: int, b: int) + +class First: + var pair: P = (0, 0) + +class Second: + var pair: P = (0, 0) + +class Optional: + var pair: P | None = None + +def assign(target: First | Second | Optional) -> None: + target.pair = (1, 2) + assert target.pair.b == 2 + +assign(First()) +assign(Optional()) +``` + +## A write through a union receiver one of whose members builds nothing + +a member that builds nothing from the literal accepts the class another builds, which is a subtype +of the plain tuple: + +```by +type P = (a: int, b: int) + +class Built: + var pair: P = (0, 0) + +class Loose: + var pair: object = 0 + +def loosen(target: Built | Loose) -> None: + target.pair = (1, 2) + assert target.pair == (1, 2) + +loosen(Built()) +loosen(Loose()) +``` + +## A write through a union receiver whose members disagree + +where the members declare different anonymous named tuples, no single value is both, so the write is +rejected: + +```by +class Left: + var pair: (a: int, b: int) = (0, 0) + +class Other: + var pair: (c: int, d: int) = (0, 0) + +def assign(target: Left | Other) -> None: + # error: [invalid-assignment] "The members of `Left | Other` would build different anonymous named tuples from this value for attribute `pair`" + target.pair = (1, 2) +``` + +## A write through a property setter + +a property's setter takes the value, so a literal written to the property is built as what the +setter's parameter declares + +```by +type Q = (c: int, d: int) + +class Store: + var held: Q = (0, 0) + + @property + def pair(self) -> Q: + return self.held + + @pair.setter + def pair(self, value: Q): + self.held = value + +store = Store() +store.pair = (1, 2) +assert store.pair.d == 2 +``` + +and a receiver that may be either a class declaring one anonymous named tuple or one whose setter +takes another would build two different classes: + +```by +type P = (a: int, b: int) + +class Declared: + var pair: P = (0, 0) + +class Through: + var held: Q = (0, 0) + + @property + def pair(self) -> Q: + return self.held + + @pair.setter + def pair(self, value: Q): + self.held = value + +def assign(target: Declared | Through) -> None: + # error: [invalid-assignment] "The members of `Declared | Through` would build different anonymous named tuples from this value for attribute `pair`" + target.pair = (1, 2) +``` + +## A write through a union receiver of other differing declarations + +a value with no plain tuple literal for the members to build differently is checked against each +member's own declaration, as python is, so a literal each of them accepts in its own way is fine: + +```by +from typing import TypedDict, NotRequired + +class Narrow(TypedDict): + a: int + +class Wide(TypedDict): + a: int + b: NotRequired[str] + +class Ints: + items: list[int] = [] + cfg: Narrow = {"a": 0} + +class Mixed: + items: list[int | str] = [] + cfg: Wide = {"a": 0} + +def assign(target: Ints | Mixed) -> None: + target.items = [1] + target.cfg = {"a": 1} + assert target.items == [1] + +assign(Ints()) +assign(Mixed()) +``` + +## A `.py` file keeps a plain tuple a tuple + +a `.py` file is python, where a plain tuple is never a named tuple, so passing one where a `.by` +module expects an anonymous named tuple is an error there: + +`shapes.by`: + +```by +def take(p: (a: int, b: int)) -> int: + return p.a +``` + +`main.py`: + +```py +from shapes import take + +take((1, 2)) # error: [invalid-argument-type] +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_class_self_ref.md b/crates/ty_python_semantic/resources/mdtest/basedpython_class_self_ref.md index b37deb172d..a015f03f79 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_class_self_ref.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_class_self_ref.md @@ -39,11 +39,12 @@ reveal_type(a["k"]) # revealed: list[A] ## direct base self-reference is still an error -a bare self-reference as a direct base is a genuine runtime error; the transpiler doesn't auto-quote -it, and ty reports the cyclic inheritance +a bare base is resolved where it is written, since the class is built on the object itself, so the +`A` not yet bound there is an unresolved reference, as python's `NameError` for it shows. a type +argument inside a base (`class A(list[A])`) is a forward reference the transpiler quotes ```by -class A(A): # error: [cyclic-class-definition] +class A(A): # error: [unresolved-reference] pass ``` @@ -53,3 +54,37 @@ class A(A): # error: [cyclic-class-definition] class A(list[B]): # error: [unresolved-reference] pass ``` + +## a cycle across modules is a cyclic definition + +each base is bound where it is written, by an import of the other module, so the cycle is between +two classes rather than a name read before it is bound + +`cycle_a.by`: + +```by +from cycle_b import B + +class A(B): # error: [cyclic-class-definition] + pass +``` + +`cycle_b.by`: + +```by +from cycle_a import A + +class B(A): # error: [cyclic-class-definition] + pass +``` + +## a cycle in a stub is a cyclic definition + +a stub is never run, so its bases are read as deferred and may name a class defined further down + +`cycle_stub.byi`: + +```byi +class C(D): ... # error: [cyclic-class-definition] +class D(C): ... # error: [cyclic-class-definition] +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_conformance.md b/crates/ty_python_semantic/resources/mdtest/basedpython_conformance.md index f15ee50ebd..16af69ca0b 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_conformance.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_conformance.md @@ -408,6 +408,25 @@ extension Widget(Show): class Widget: ... ``` +## a conformance declared above the protocol it names is reported + +the protocol is named where the conformance registers itself, so it has to be declared first too + +```by +# error: [invalid-conformance] "`Show` is declared after this conformance" +extension str(Show): + override def show(self) -> str: + return self + +protocol Show: + def show(self) -> str + +def render(value: Show) -> str: + return value.show() + +render("hi") +``` + ## a conformance's own member overrides the interface's default ```by diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md b/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md index dd3f3efbf3..d8aa0ba443 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md @@ -447,6 +447,36 @@ a = A(s) a.m() # ok — bound methods resolve ``` +## a stored function still has its `context` parameters filled + +a type inferred from a stored value is widened, but a `.by` module keeps a function's own type +rather than widening it to a callable type, which has no `context` parameters. so a function read +back out of a list, a dict, a tuple or an attribute is still a plain function, and its `context` +parameters are filled where it is called + +```by +def f(a: int, context b: str) -> str: + return b + +class Holder: + def __init__(self): + self.f = f + +context s = "filled" +listed = [f] +keyed = {"a": f} +paired = (f, 1) +held = Holder() +appended = [] +appended.append(f) + +reveal_type(listed[0](1)) # revealed: str +reveal_type(keyed["a"](1)) # revealed: str +reveal_type(paired[0](1)) # revealed: str +reveal_type(held.f(1)) # revealed: str +reveal_type(appended[0](1)) # revealed: str +``` + ## an overloaded callee fills a parameter every overload agrees on Which overload a call selects is decided by the arguments it is given, so an argument may be written diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md b/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md index 2f56c6172d..65532df1bc 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md @@ -326,6 +326,36 @@ f: sink(it) ``` +## a stored function's parameters are still checked + +a type inferred from a stored value is widened, but a `.by` module keeps a function's own type +rather than widening it to a callable type. so a function read back out of a list, a dict, a tuple +or an attribute still says which of its parameters are `local`, and handing a `local` to one that is +not still escapes + +```by +def keep(cb: () -> None): + cb() + +class Holder: + def __init__(self): + self.k = keep + +listed = [keep] +keyed = {"a": keep} +paired = (keep, 1) +held = Holder() +appended = [] +appended.append(keep) + +def use(local cb: () -> None): + listed[0](cb) # error: [escaping-local] + keyed["a"](cb) # error: [escaping-local] + paired[0](cb) # error: [escaping-local] + held.k(cb) # error: [escaping-local] + appended[0](cb) # error: [escaping-local] +``` + ## an opaque callee leaves the block unconstrained When the callee's callback shape cannot be inspected, there is no declaration to read, so the block diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md b/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md index f6fafc92d8..865e4b9c31 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md @@ -161,6 +161,49 @@ def main(): pass ``` +## a stored function still raises what it declares + +a type inferred from a stored value is widened, but a `.by` module keeps a function's own type +rather than widening it to a callable type, which has no `raises` clause. so a call through a +function read back out of a list, a dict, a tuple or an attribute still raises what the function +declares + +```by +def p() -> int raises ValueError: + raise ValueError + +class Holder: + def __init__(self): + self.p = p + +listed = [p] +keyed = {"a": p} +paired = (p, 1) +held = Holder() +appended = [] +appended.append(p) + +def from_list() -> int raises KeyError: + # error: [undeclared-raise] "`from_list` can raise `ValueError`, which its `raises` clause does not include" + return listed[0]() + +def from_dict() -> int raises KeyError: + # error: [undeclared-raise] "`from_dict` can raise `ValueError`, which its `raises` clause does not include" + return keyed["a"]() + +def from_tuple() -> int raises KeyError: + # error: [undeclared-raise] "`from_tuple` can raise `ValueError`, which its `raises` clause does not include" + return paired[0]() + +def from_attribute() -> int raises KeyError: + # error: [undeclared-raise] "`from_attribute` can raise `ValueError`, which its `raises` clause does not include" + return held.p() + +def from_append() -> int raises KeyError: + # error: [undeclared-raise] "`from_append` can raise `ValueError`, which its `raises` clause does not include" + return appended[0]() +``` + ## `raises Never` cannot raise ```by diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md b/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md index bde4de3668..3cf04862b9 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md @@ -26,9 +26,7 @@ class A: self.b = str(a) x = A(1) -# `str(a)` constructs, so the attribute's only assignment is exactly a `str` — see -# basedpython/features/exact-construction.md -reveal_type(x.b) # revealed: final str +reveal_type(x.b) # revealed: str ``` ## `let` parameter inside body-bearing `init` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_just_float.md b/crates/ty_python_semantic/resources/mdtest/basedpython_just_float.md index 51493fdea9..2dc7b26d78 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_just_float.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_just_float.md @@ -308,6 +308,89 @@ class Counter: reveal_type(self.label) # revealed: str ``` +## the rest of an inferred type widens as it does without the strict model + +of the numbers, the strict model holds back only the widening of `float` and `complex`. the rest of +the precision an inferred type is widened from goes as it does in a module without it: an attribute +assigned a constructor call is declared with the class rather than the exact `final` type of that +one instance, and one assigned a narrowed value is declared without what the narrowing ruled out, so +either can later hold an instance of a subclass; an attribute assigned a `range` is declared a +`range` whether or not that one was empty; and a covariant type argument keeps the literal it was +inferred from, since nothing reads a different type back out of it + +a function is the one thing a `.by` module widens differently, and not because of its numbers: it +keeps the function's own type rather than widening it to a callable type, because a `.by` function +can carry what no callable type expresses — reified type parameters, `context` parameters, `local` +parameters, a `raises` clause + +```by +class A: ... + +class B(A): ... + +class Out[out T]: + def __init__(self, t: T): ... + +class Holder: + def __init__(self, v: A): + self.a = A() + self.f = len + self.r = range(3) + self.out = Out(1) + if v is not B: + self.narrowed = v + +def replace(h: Holder): + reveal_type(h.a) # revealed: A + reveal_type(h.f) # revealed: def len(obj: Sized, /) -> int + reveal_type(h.out) # revealed: Out[1] + reveal_type(h.narrowed) # revealed: A + h.a = B() + h.narrowed = B() + h.r = range(0) +``` + +## `strict-float` gives a `.py` module the same widening + +a `.py` module that opts into the strict model widens its numbers exactly as the `.by` module above +does, holding back only `float` and `complex`. a function widens to its callable type as it does in +any `.py` module: keeping it is part of what `.by` is, not of its numeric model + +```toml +[analysis] +strict-float = true + +[environment] +python-version = "3.12" +``` + +```py +class A: ... +class B(A): ... + +class Out[T]: + def __init__(self, t: T) -> None: ... + def get(self) -> T: + raise NotImplementedError + +class Holder: + def __init__(self, v: A, x: float) -> None: + self.x = x + self.f = len + self.r = range(3) + self.out = Out(1) + if not isinstance(v, B): + self.narrowed = v + +def replace(h: Holder) -> None: + reveal_type(h.x) # revealed: float + reveal_type(h.f) # revealed: (obj: Sized, /) -> int + reveal_type(h.out) # revealed: Out[Literal[1]] + reveal_type(h.narrowed) # revealed: A + h.narrowed = B() + h.r = range(0) +``` + ## a container's element type, which is what the compiler lays out Reading an element out of a `list[float]` and iterating one both go through the same annotation, so diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md b/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md index 14245b340b..f514b526a7 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md @@ -30,6 +30,25 @@ x: Literal["a", "b"] = "a" reveal_type(x) # revealed: "a" ``` +## a string in a generic's type argument is a Literal + +a string is a literal type wherever a type expression holds it, so `list["a"]` is a list of the +string `"a"` rather than a forward reference to a name `a` + +```by +from typing import Callable + +type Named = dict[str, "y"] +Alias = list["x"] + +def f(a: list["a"], b: dict[str, "k"], c: Callable[["p"], "r"], n: Named, al: Alias): + reveal_type(a[0]) # revealed: "a" + reveal_type(b["z"]) # revealed: "k" + reveal_type(c("p")) # revealed: "r" + reveal_type(n["z"]) # revealed: "y" + reveal_type(al[0]) # revealed: "x" +``` + ## a union of string literals is a union of literal types `"foo" | "bar"` is two literal types joined, not a `str.__or__` that would fail at runtime: the diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_parameters.md b/crates/ty_python_semantic/resources/mdtest/basedpython_parameters.md index ac208dc604..e35018e000 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_parameters.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_parameters.md @@ -38,6 +38,20 @@ callable usage and is dropped when the tuple is used as a tuple type. c: (int, /, name: str) ``` +## a keyword variadic is not a tuple type + +A keyword variadic `**: T` collects keyword arguments, which a tuple has no place for, so it is only +valid in a callable's parameter list. Written in a tuple type it is rejected, whether or not the +tuple also has a marker or a positional variadic. + +```by +a: (int, **kw: str) # error: [invalid-type-form] +b: (**: str) # error: [invalid-type-form] +c: (int, /, **kw: str) # error: [invalid-type-form] +d: (int, *, name: str, **kw: bytes) # error: [invalid-type-form] +e: (int, *args: str, **kw: bytes) # error: [invalid-type-form] +``` + ## variadic tuple assignment A tuple value with arbitrarily many elements is assignable to a `(*: T)` annotation since the diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md b/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md index 8e5149309a..097face132 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md @@ -325,6 +325,37 @@ def g[T](x: T) -> T: c: (int) -> int = g # ok — erased generic specializes to a plain callable ``` +## a reified generic keeps its type when stored + +a type inferred from a stored value is widened, but a `.by` module keeps a function's own type +rather than widening it to a callable type, which has no slot for the specialization step either. so +a reified generic can still be specialized once it is read back out of a list, a dict, a tuple or an +attribute + +```by +def name[reified T]() -> str: + return T.__name__ + +class Holder: + def __init__(self): + self.f = name + +listed = [name] +keyed = {"a": name} +paired = (name, 1) +held = Holder() +appended = [] +appended.append(name) + +reveal_type(listed) # revealed: list[def name[T]() -> str] +reveal_type(held.f) # revealed: def name[T]() -> str +reveal_type(listed[0][int]()) # revealed: str +reveal_type(keyed["a"][str]()) # revealed: str +reveal_type(paired[0][bytes]()) # revealed: str +reveal_type(held.f[float]()) # revealed: str +reveal_type(appended[0][int]()) # revealed: str +``` + ## overrides must keep the reified interface Specializations flow through the base type — `a.f[int]()` on `a: A` dispatches to the override at diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md b/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md new file mode 100644 index 0000000000..885a516123 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md @@ -0,0 +1,406 @@ +# basedpython: a name that is not bound where the program reads it + +a basedpython lowering that writes a type into code that runs — a specialization the checker solved, +a soundness check, a type test — reads each name in it where that code runs. a binding made under +`if TYPE_CHECKING:` exists for the checker and never runs, which is how an import cycle is broken; a +binding made only on some paths, or only further down the module, is not there yet either. a +lowering leaves such a name out, and where the program needs the type at runtime to do what the +source asks, that is an error here rather than a `NameError` when the program runs + +## A specialization the program only refines is left out + +an empty `set()` declared as a `set[Decimal]` is a set whatever it is specialized as, so the +specialization the checker solved is left out rather than written with a name the program does not +bind + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +totals: set[Decimal] = set() +assert Ledger().entries == totals +``` + +## `TYPE_CHECKING` under another name + +the constant is the same constant whatever name it is imported under, and a block it guards is a +type-checking block all the same + +```by +from typing import TYPE_CHECKING as CHECKING + +if CHECKING: + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +## A block on any condition that reads `TYPE_CHECKING` + +the checker reads the constant as `True` and the program reads it as `False`, so a block whose +condition reads it may be taken by the checker and not by the program, whatever the condition does +with it. a binding made past such a condition is not one the program can be relied on to make, and +this does not ask which way the condition goes: `if not TYPE_CHECKING:` and +`if TYPE_CHECKING and ready:` count all the same + +### A `match` on the constant + +```by +from typing import TYPE_CHECKING + +match TYPE_CHECKING: + case True: + from decimal import Decimal + case _: + pass + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### A comparison with the constant + +```by +from typing import TYPE_CHECKING + +# error: [redundant-boolean-comparison] +if TYPE_CHECKING == True: + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### A call on the constant + +```by +from typing import TYPE_CHECKING + +if bool(TYPE_CHECKING): + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### The constant bound by a walrus + +```by +from typing import TYPE_CHECKING + +if checking := TYPE_CHECKING: + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### A `while` on the constant + +```by +from typing import TYPE_CHECKING + +while TYPE_CHECKING: + from decimal import Decimal + break + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### A value chosen by the constant + +a conditional expression on the constant binds one value for the checker and another for the +program. here the checker reads `Decimal` as the class, so a value declared as one gets a soundness +check, but the program binds `None`, which the check could not test against, so it is left out + +```by +from typing import TYPE_CHECKING +from decimal import Decimal as RuntimeDecimal + +Decimal = RuntimeDecimal if TYPE_CHECKING else None + +def parse(text: str) -> dynamic: + return RuntimeDecimal(text) + +price: Decimal = parse("1.5") +assert price == RuntimeDecimal("1.5") +``` + +## An import made only on some paths + +a name imported under a condition the program decides as it runs may not be bound when it is read + +```by +import os + +if os.environ.get("BASEDPYTHON_NEVER_SET"): + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +## A class declared further down the module + +the module's own statements run in order, so a statement above a class's declaration runs before the +class exists. a method runs later, when the module has declared everything + +```by +early: set[Later] = set() + +class Later: ... + +class Ledger: + entries: set[Later] + + def __init__(self): + self.entries = set() + +assert early == Ledger().entries +``` + +## A name the module unbinds + +a `del` unbinds a name, so by the time a later statement runs the name may be gone + +```by +from decimal import Decimal + +class Ledger: + entries: set[Decimal] + +ledger = Ledger() +del Decimal +ledger.entries = set() +assert ledger.entries == set() +``` + +## A name bound in one branch of a conditional expression + +a binding inside a conditional expression is made only when that branch is taken + +```by +import os +from decimal import Decimal as RuntimeDecimal + +_ = (Decimal := RuntimeDecimal) if os.environ.get("BASEDPYTHON_NEVER_SET") else None +entries: set[Decimal] = set() +assert entries == set() +``` + +## An instance made before a class its declaration names + +a method runs when it is called, which can be before the module reaches a class it names: here +`Ledger()` runs above `class Later` + +```by +class Ledger: + later: set[Later] + + def __init__(self): + self.later = set() + +ledger = Ledger() + +class Later: ... + +assert ledger.later == set() +``` + +## A reified type argument the program does not bind + +a reified type parameter is read at runtime, so a construction or a call that would pass it a class +the program does not bind is reported + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +class Box[reified T]: + def kind(self) -> str: + return T.__name__ + +def kind_of[reified T](value: T) -> str: + return T.__name__ + +class Holder: + box: Box[Decimal] + + def __init__(self): + # error: [unspecialized-reified-generic] + self.box = Box() + +def describe(value: Decimal) -> str: + # error: [unspecialized-reified-generic] + return kind_of(value) +``` + +## A type test or checked cast against a class the program does not bind + +`is` and `cast!` test the value against the class at runtime, so they cannot test against one the +program does not bind + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +def test(value: object) -> bool: + # error: [erased-type-check] "`is Decimal` cannot be checked at runtime: `Decimal` is not bound when it runs" + return value is Decimal + +def check(value: object) -> Decimal: + # error: [erased-type-check] "`Decimal` cannot be checked at runtime: `Decimal` is not bound when it runs" + return value cast! Decimal +``` + +## A failed test against an unbound class narrows + +a type test against a class the program does not bind is reported, and the checker reads the rest of +the function as it would if the test could run: the branch where the test fails rules the class out + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +def whole(value: int | Decimal) -> int: + # error: [erased-type-check] "`is Decimal` cannot be checked at runtime: `Decimal` is not bound when it runs" + if value is Decimal: + return 0 + reveal_type(value) # revealed: int + return value +``` + +## A reified call that has a reason of its own + +where a call's specialization cannot be written for a reason of its own, as for a lambda's type, +that reason is the one reported, even when another argument names a class the program does not bind + + + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +def pair[reified A, reified B](a: A, b: B) -> str: + return A.__name__ + B.__name__ + +def describe(value: Decimal) -> str: + # error: [unspecialized-reified-generic] + return pair(value, lambda: 1) +``` + +## A type test at the module's own level + +a test written at the module's own level runs as the module reaches it. a class the module unbinds +further down still holds where the test runs, so the test reads the class by the name the source +wrote, and the statements after it, the `assert`s among them, decide nothing about it + +```by +class Shown: ... + +def make() -> object: + return Shown() + +value = make() + +assert value is Shown +assert not (value is int) +assert isinstance(value, Shown) + +del Shown +``` + +## A class the program binds differently from the checker + +a module may import a class for the checker and bind the name some other way as it runs, in the +`else:` of the same block. the checker reads the block as always taken, so the class it resolves is +the `TYPE_CHECKING` one, and the name the program binds need not be that class. a test against it +would answer for whatever the program bound, so it is refused + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal +else: + Decimal = float + +def test(value: object) -> bool: + # error: [erased-type-check] "`is Decimal` cannot be checked at runtime: the `Decimal` bound when it runs need not be the class the checker resolved" + return value is Decimal + +def check(value: object) -> Decimal: + # error: [erased-type-check] "`Decimal` cannot be checked at runtime: the `Decimal` bound when it runs need not be the class the checker resolved" + return value cast! Decimal +``` + +## An unchecked cast to a class the program does not bind + +a plain `cast` tests nothing, so it is fine: its type is written as a string, which python never +evaluates + +```by +from typing import TYPE_CHECKING +from decimal import Decimal as RuntimeDecimal + +if TYPE_CHECKING: + from decimal import Decimal + +def widen(value: Decimal) -> Decimal | int: + return value cast Decimal | int + +assert widen(RuntimeDecimal(1)) == RuntimeDecimal(1) +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md new file mode 100644 index 0000000000..fbab84eee6 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md @@ -0,0 +1,1310 @@ +# unused imports + +`unused-import` reports an import in a `.by` file that nothing uses. an import is used when a name +it binds is read, and also when the module it names declares an `extension` or a conformance that +supplied something the file relies on — the extension's members are only applicable in files that +import its module, so the import is what makes them reachable even when its name is never read. + +the mdtest corpus turns this rule off, so the tests below turn it back on. + +```toml +[environment] +python-version = "3.12" + +[rules] +unused-import = "warn" +``` + +## an import nothing reads is reported + +```by +import json # error: [unused-import] "`json` imported but unused" +from os import path # error: [unused-import] "`os.path` imported but unused" +from collections import OrderedDict, deque # error: [unused-import] "`collections.deque` imported but unused" + +OrderedDict() +``` + +## an import whose name is read is not + +a read anywhere in the file counts, including from a nested function and from an annotation + +```by +import json +from collections import OrderedDict + +def dump(value: OrderedDict[str, int]) -> str: + return json.dumps(value) +``` + +## an import used only through an extension + +`colours` declares an extension of `Enum`, and `main` reads its member without ever reading the name +`colours` + +`colours.by`: + +```by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +`main.by`: + +```by +from enum import Enum +import colours + +class Colour(Enum): + RED = 1 + +def describe(colour: Colour) -> str: + return colour.lower_name + +describe(Colour.RED) +``` + +`colours` itself reads `Enum` only in the `extension Enum:` header, which names the type it extends + +## an extension module whose extension nothing uses is reported + +`colours.by`: + +```by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +`main.by`: + +```by +import colours # error: [unused-import] +``` + +## an import used only through a conformance + +`render` asks for a `Show`, and `"hi"` is one only because `conforming` declares that `str` +conforms. nothing in `main` names `conforming` + +`shows.by`: + +```by +protocol Show: + def show(self) -> str + +def render(value: Show) -> str: + return value.show() +``` + +`conforming.by`: + +```by +from shows import Show + +extension str(Show): + override def show(self) -> str: + return self +``` + +`main.by`: + +```by +from shows import render +import conforming + +render("hi") +``` + +## an import used only through a protocol extension reached by conformance + +`loud` extends the protocol itself, and a conforming `str` reaches the member through the +conformance — so both imports are used, and neither name is read + +`shows.by`: + +```by +protocol Show: + def show(self) -> str +``` + +`loud.by`: + +```by +from shows import Show + +extension Show: + def shout(self) -> str: + return self.show().upper() +``` + +`conforming.by`: + +```by +from shows import Show + +extension str(Show): + override def show(self) -> str: + return self +``` + +`main.by`: + +```by +import loud +import conforming + +def greet() -> str: + return "hi".shout() +``` + +## an import used only for a default a conformance takes from a protocol extension + +`str` answers `Show`'s requirement with the default `defaults` declares on the protocol, so the +conformance's witness table takes that default, and the import of `defaults` is what makes it +applicable + +`shows.by`: + +```by +protocol Show: + def show(self) -> str +``` + +`defaults.by`: + +```by +from shows import Show + +extension Show: + def show(self) -> str: + return "?" +``` + +`conforming.by`: + +```by +from shows import Show +import defaults + +extension str(Show): ... +``` + +## an import that brings a second conformance of the same pair into view + +the conflict is reported where the second conformance is declared, and the import that brings the +first into view is what causes it — so it is used until one of the two goes + +`shows.by`: + +```by +protocol Show: + def show(self) -> str +``` + +`first.by`: + +```by +from shows import Show + +extension str(Show): + override def show(self) -> str: + return self +``` + +`second.by`: + +```by +from shows import Show +import first + +# error: [invalid-conformance] +extension str(Show): + override def show(self) -> str: + return self +``` + +## an import used only by a type test answered from the conformance registry + +`value is Show` asks the registry, because a conformance to `Show` is in scope. without the import +the test is a plain `isinstance` against a protocol that does not allow one + +`shows.by`: + +```by +protocol Show: + def show(self) -> str +``` + +`conforming.by`: + +```by +from shows import Show + +extension str(Show): + override def show(self) -> str: + return self +``` + +`main.by`: + +```by +from shows import Show +import conforming + +def check(value: object) -> bool: + return value is Show +``` + +## an import used only through an operator an extension supplies + +`money.by`: + +```by +class Money: + def __init__(self, cents: int) -> None: + self.cents = cents +``` + +`arithmetic.by`: + +```by +from money import Money + +extension Money: + def __add__(self, other: Money) -> Money: + return Money(self.cents + other.cents) +``` + +`main.by`: + +```by +from money import Money +import arithmetic + +def total(a: Money, b: Money) -> Money: + return a + b +``` + +## an import used only through a unary operator or a comparison an extension supplies + +`money.by`: + +```by +class Money: + def __init__(self, cents: int) -> None: + self.cents = cents +``` + +`negation.by`: + +```by +from money import Money + +extension Money: + def __neg__(self) -> Money: + return Money(-self.cents) +``` + +`ordering.by`: + +```by +from money import Money + +extension Money: + def __lt__(self, other: Money) -> bool: + return self.cents < other.cents +``` + +`main.by`: + +```by +from money import Money +import negation +import ordering + +def refund(a: Money) -> Money: + return -a + +def cheaper(a: Money, b: Money) -> bool: + return a < b +``` + +## an import used only through a conversion an extension supplies + +`ids.by`: + +```by +class UserId: + pass + +def register(user: UserId) -> None: ... +``` + +`parsing.by`: + +```by +from ids import UserId + +extension UserId: + class def __from__(cls, value: int) -> UserId: + return UserId() +``` + +`main.by`: + +```by +from ids import register +import parsing + +register(3) +``` + +## every element of a collection literal that converts uses its extension + +each element of the list converts through a different module's extension, so both imports are used — +the second element's as much as the first's + +`ids.by`: + +```by +class UserId: + pass + +class Tag: + pass +``` + +`numbers.by`: + +```by +from ids import UserId + +extension UserId: + class def __from__(cls, value: int) -> UserId: + return UserId() +``` + +`labels.by`: + +```by +from ids import Tag + +extension Tag: + class def __from__(cls, value: str) -> Tag: + return Tag() +``` + +`main.by`: + +```by +from ids import UserId, Tag +import numbers +import labels + +xs: list[UserId | Tag] = [1, "s"] +``` + +## every element of a collection literal assigned to an attribute uses its extension + +`ids.by`: + +```by +class UserId: + pass + +class Tag: + pass +``` + +`numbers.by`: + +```by +from ids import UserId + +extension UserId: + class def __from__(cls, value: int) -> UserId: + return UserId() +``` + +`labels.by`: + +```by +from ids import Tag + +extension Tag: + class def __from__(cls, value: str) -> Tag: + return Tag() +``` + +`main.by`: + +```by +from ids import UserId, Tag +import numbers +import labels + +class Holder: + def __init__(self) -> None: + self.xs: list[UserId | Tag] = [] + +holder = Holder() +holder.xs = [1, "s"] +``` + +## any import of the module grants its extensions + +a `from` import of a name the file never reads still makes the module's extensions applicable, so it +is used when one of them is + +`colours.by`: + +```by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() + +def unrelated() -> None: ... +``` + +`main.by`: + +```by +from enum import Enum +from colours import unrelated + +class Colour(Enum): + RED = 1 + +def describe(colour: Colour) -> str: + return colour.lower_name +``` + +## an import inside a function grants the whole file + +an import makes a module's extensions applicable wherever in the file it is written, so one inside a +function that is never called is still used by an extension member read anywhere else + +`colours.by`: + +```by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +`main.by`: + +```by +from enum import Enum + +class Colour(Enum): + RED = 1 + +def setup() -> None: + import colours + +def describe(colour: Colour) -> str: + return colour.lower_name +``` + +## an extension member read in a receiver block + +a bare name inside a trailing lambda block whose callback declares a receiver reads the receiver's +members, and an extension's members with them + +`tags.by`: + +```by +class Tag: + def __init__(self) -> None: + self.children: list[str] = [] +``` + +`paragraphs.by`: + +```by +from tags import Tag + +extension Tag: + def p(self, text: str) -> None: + self.children.append(text) +``` + +`main.by`: + +```by +from tags import Tag +import paragraphs + +def page(fn: Tag.() -> None) -> Tag: + tag = Tag() + fn(tag) + return tag + +page: + p("hello") +``` + +## a receiver callable read as a method + +`x.fn()` reads `fn`, a receiver callable in scope, when `x` has no member of that name + +`render.by`: + +```by +def describe(value: int) -> str: + return str(value) + +renderer: int.() -> str = describe +``` + +`main.by`: + +```by +from render import renderer + +def show(x: int) -> str: + return x.renderer() +``` + +## an import read only as a context-sensitive name's qualifier + +`Red` where a `Color` is expected is lowered to `Color.Red`, so the import of `Color` is read +although the source never spells the name + +`palette.by`: + +```by +enum class Color: + case Red, Green + +def paint(c: Color) -> str: + return str(c) +``` + +`main.by`: + +```by +from palette import Color, paint + +paint(Red) +``` + +## an import read only as a case name's qualifier + +`case Red:` is lowered to `case Color.Red:`, and `case Circle(r):` to `case Shape.Circle(r):` + +`palette.by`: + +```by +enum class Color: + case Red, Green + +enum class Shape: + case Circle(radius: float) + case Dot + +def current() -> Color: + return Color.Green + +def outline() -> Shape: + return Shape.Dot +``` + +`main.by`: + +```by +from palette import Color, Shape, current, outline + +def describe() -> str: + match current(): + case Red: + return "red" + case _: + return "other" + +def radius() -> float: + match outline(): + case Circle(r): + return r + case _: + return 0.0 +``` + +## an import read only by a type test's runtime check + +`x is AB` is lowered to a test of each arm of the alias, `isinstance(x, A) or isinstance(x, B)`, so +it reads `A` and `B` although the source spells neither + +`kinds.by`: + +```by +class A: + pass + +class B: + pass + +type AB = A | B +``` + +`main.by`: + +```by +from kinds import AB, A, B + +def check(x: object) -> bool: + return x is AB +``` + +## an import read only by a type test against a module attribute + +`kinds.by`: + +```by +class A: + pass + +class B: + pass + +type AB = A | B +``` + +`main.by`: + +```by +import kinds +from kinds import A, B + +def check(x: object) -> bool: + return x is kinds.AB +``` + +## an import read only by a checked cast's runtime check + +`kinds.by`: + +```by +class A: + pass + +class B: + pass + +type AB = A | B +``` + +`main.by`: + +```by +from kinds import AB, A, B + +def narrow(x: object) -> AB: + return x cast! AB +``` + +## an import read only by a `raises` guard + +the guard a `raises` function is lowered behind names the exceptions it may raise + +`errors.by`: + +```by +class Oops(Exception): + pass + +class Bad(Exception): + pass + +type Errs = Oops | Bad +``` + +`main.by`: + +```by +from errors import Errs, Oops, Bad + +def run() -> None raises Errs: + raise Oops() +``` + +## an import read only by a soundness check + +the result of a call whose type comes from a type variable is checked at runtime against the type +the checker gave it, `isinstance(…, (A, B))` here, which reads `A` and `B`. a build may turn that +check off, but one that leaves it on reads them, so they are used either way + +`kinds.by`: + +```by +class A: + pass + +class B: + pass + +type AB = A | B +``` + +`main.by`: + +```by +from kinds import AB, A, B + +def pick[T](items: list[T]) -> T: + return items[0] + +def first(xs: list[AB]) -> AB: + y: AB = pick(xs) + return y +``` + +## an import read only by a reified call's specialization + +a bare call of a reified generic function is given the specialization the checker solved, +`describe[Widget]` here + +`widgets.by`: + +```by +class Widget: + pass + +def make() -> Widget: + return Widget() +``` + +`main.by`: + +```by +from widgets import Widget, make + +def describe[reified T](value: T) -> str: + return T.__name__ + +describe(make()) +``` + +## an import read only by a reified construction's specialization + +`widgets.by`: + +```by +class Widget: + pass + +def make() -> Widget: + return Widget() +``` + +`main.by`: + +```by +from widgets import Widget, make + +class Tagged[reified T]: + def __init__(self, value: T) -> None: + self.value = value + +Tagged(make()) +``` + +## an import read only by a union parameter's reified type parameter + +a parameter annotated with a union of specializations of one collection is given a reified type +parameter whose bound spells each arm's class, `(A, B)` here. the annotation spells `A` only through +another module's name for it + +`kinds.by`: + +```by +class A: + pass + +class B: + pass +``` + +`aliases.by`: + +```by +from kinds import A + +Alias = A +``` + +`main.by`: + +```by +from aliases import Alias +from kinds import A, B + +def size(xs: list[Alias] | list[B]) -> int: + return len(xs) +``` + +## an import read only by a type test's spelling of a class + +`Number` names `Fraction` as an attribute of `fractions`, but the test checks each arm against the +class as the module binds it, `isinstance(value, Fraction)`, which reads the `from` import + +```by +import fractions +from fractions import Fraction + +type Number = fractions.Fraction | int + +def numeric(value: object) -> bool: + return value is Number + +print(numeric(fractions.Fraction(1, 2)), numeric("x")) +``` + +## an import read only by a soundness check's spelling of a class + +the check on `pick`'s result reads `Fraction` the same way. a build that runs it rejects the `1` +smuggled into the list as `Any`; without the import there is no check to reject it + +```by +import fractions +from fractions import Fraction +from typing import Any + +def pick[T](items: list[T]) -> T: + return items[0] + +smuggled: Any = 1 +numbers: list[fractions.Fraction] = [smuggled] +first = pick(numbers) +print(type(first).__name__) +``` + +## explicit re-exports are not reported + +`import a as a`, `from a import b as b` and basedpython's `from a export b` say the name is part of +this module's interface + +```by +import json as json +from os import path as path +from collections export OrderedDict +``` + +## a name listed in `__all__` is exported + +```by +import json +from os import path + +__all__ = ["json", "path"] +``` + +## a name listed in an `__all__` the checker cannot read in full + +`__all__` built by concatenating something other than a list of names is too dynamic for the checker +to know every name in it, but a name spelled out in it is still exported + +```by +import json + +extra: list[str] = [] +__all__ = ["json"] + extra +``` + +## a file that does not parse is not checked + +a read the parser could not recover is invisible, so an import in a file with a syntax error is +never reported + +```by +import json + +x = 1 + # error: [invalid-syntax] +``` + +## a name another module's `__all__` adds to this one's + +the checker follows `__all__ += helpers.__all__` to the names `helpers` lists, so a name imported +from it is exported + +`helpers.by`: + +```by +def helper() -> None: ... + +__all__ = ["helper"] +``` + +`main.by`: + +```by +import helpers +from helpers import helper + +__all__ = ["helpers"] +__all__ += helpers.__all__ +``` + +## a `__future__` import is a directive + +```by +from __future__ import annotations +``` + +## an import rebound by something other than an import is not reported + +an assignment that rebinds the name makes the import a redefinition rather than an unused import, +which is `F811`'s concern + +```by +import json +import json as codec + +json = codec +``` + +## a read of an earlier binding counts for a later one + +```by +import os + +os.getcwd() + +import os +``` + +## every one of several unread imports of a name is reported + +an import rebound by another import before anything reads it is reported too, so one round of fixes +removes them all + +```by +import json # error: [unused-import] +import json # error: [unused-import] +``` + +## a `del` starts the name afresh + +a read before the `del` belongs to the binding it deletes, so it does not use an import made after + +```by +import os + +os.getcwd() +del os + +import os # error: [unused-import] +``` + +## a deleted import is not reported + +`del` reads the binding it deletes, and leaves the name unbound + +```by +import os + +del os +``` + +## a name declared `global` binds in the module + +an import in a function that declares the name `global` binds the module's name, and is not reported +there. the module's own import is reported when nothing reads the name + +```by +import json # error: [unused-import] + +def reload() -> None: + global json + import json +``` + +## a read through a `global` declaration uses the module's import + +```by +import json + +def dump() -> str: + global json + return json.dumps(1) +``` + +## an import in a class body is not reported + +```by +class Config: + import json +``` + +## a submodule import binds the top-level package + +`import os.path` binds `os`, so a read of `os` uses it + +```by +import os.path + +def cwd() -> str: + return os.getcwd() +``` + +## an unread submodule import is reported by its full name + +```by +import os.path # error: [unused-import] "`os.path` imported but unused" +``` + +## two submodule imports bind one name + +`import os.path` and `import email.mime` bind `os` and `email`; a second submodule of the same +package rebinds the same name, so a read through either uses both + +```by +import email.mime +import email.parser + +def parser() -> object: + return email.parser +``` + +## an import of the package does not redefine a submodule import + +`import os` after `import os.path` binds `os` again, but `os.path` stays reachable through it only +because of the first import, so the first is not an unused redefinition + +```by +import os.path +import os + +os.getcwd() +``` + +## a read earlier in a loop body uses an import later in it + +the second time round the loop, the read sees the import the first pass made. the checker cannot +tell that the first pass always takes the other branch, so it reports the read too + +```by +def poll() -> None: + for attempt in range(2): + if attempt: + os.getcwd() # error: [possibly-unresolved-reference] + else: + import os +``` + +## a name in a `raises` clause is read + +```by +from json import JSONDecodeError + +def parse(text: str) -> int raises JSONDecodeError: + raise JSONDecodeError("bad", text, 0) +``` + +## an import read in an annotation in a `TYPE_CHECKING` block + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections import OrderedDict + +def first(value: OrderedDict[str, int]) -> int: + return 0 +``` + +## an unread import in a `TYPE_CHECKING` block is reported + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections import OrderedDict # error: [unused-import] +``` + +## an annotation may read an import written after it + +annotations are evaluated lazily, so a name in one resolves to every binding the scope makes + +```by +def first(value: OrderedDict[str, int]) -> int: + return 0 + +from collections import OrderedDict +``` + +## a string in a type position reads no name + +in a `.by` file a string in a type position is a literal type, never a forward reference, so it uses +nothing + +```by +from collections import OrderedDict # error: [unused-import] + +def name(value: "OrderedDict") -> None: ... +``` + +## an import that tests for a module's availability + +an import in the body of a `try` that handles `ImportError` is usually there to find out whether the +module is installed. it is still reported, and says how to ask without importing + +```by +try: + import tomllib # snapshot +except ImportError: + pass +``` + +```snapshot +warning[unused-import]: `tomllib` imported but unused + --> src/mdtest_snippet.by:2:12 + | +2 | import tomllib # snapshot + | ^^^^^^^ +help: To test whether a module is available, use `importlib.util.find_spec` instead of importing it +``` + +## an import in a function defined inside such a `try` is not a test + +the function's body runs when it is called, outside the `try`, so its import gets the ordinary fix + +```by +try: + import tomllib # error: [unused-import] + + def load() -> None: + import json # snapshot +except ImportError: + pass +``` + +```snapshot +warning[unused-import]: `json` imported but unused + --> src/mdtest_snippet.by:5:16 + | +5 | import json # snapshot + | ^^^^ +help: Remove the unused import `json` + | +4 | def load() -> None: + - import json # snapshot +5 + pass # snapshot +6 | except ImportError: + | +``` + +## an unused import in a package's `__init__` may be a re-export + +a package's `__init__.by` is what importing the package runs, so an import there may be how the +package exports the name. it is still reported, and one of the project's own modules says how to +make the re-export explicit + +`pkg/models.by`: + +```by +class Widget: + pass +``` + +`pkg/__init__.by`: + +```by +import json # error: [unused-import] +from .models import Widget # snapshot +``` + +```snapshot +warning[unused-import]: `.models.Widget` imported but unused + --> src/pkg/__init__.by:2:21 + | +2 | from .models import Widget # snapshot + | ^^^^^^ +help: If this is a re-export, write it as `from … export …`, as a redundant alias, or list it in `__all__` +``` + +## an import in a function is reported in its scope + +```by +def load() -> None: + import json # error: [unused-import] +``` + +## the fix removes an unused alias and keeps the rest of the statement + +```by +from collections import OrderedDict, deque # snapshot + +OrderedDict() +``` + +```snapshot +warning[unused-import]: `collections.deque` imported but unused + --> src/mdtest_snippet.by:1:38 + | +1 | from collections import OrderedDict, deque # snapshot + | ^^^^^ +help: Remove the unused import `collections.deque` + | + - from collections import OrderedDict, deque # snapshot +1 + from collections import OrderedDict # snapshot +2 | + | +``` + +## the fix removes an alias with its own line inside parentheses + +```by +from collections import ( + OrderedDict, # read below + deque, # snapshot +) + +OrderedDict() +``` + +```snapshot +warning[unused-import]: `collections.deque` imported but unused + --> src/mdtest_snippet.by:3:5 + | +3 | deque, # snapshot + | ^^^^^ +help: Remove the unused import `collections.deque` + | +2 | OrderedDict, # read below + - deque, # snapshot +3 | ) + | +``` + +## the fix writes `pass` where removing the import would leave a block empty + +```by +def load() -> None: + import json # snapshot +``` + +```snapshot +warning[unused-import]: `json` imported but unused + --> src/mdtest_snippet.by:2:12 + | +2 | import json # snapshot + | ^^^^ +help: Remove the unused import `json` + | +1 | def load() -> None: + - import json # snapshot +2 + pass # snapshot + | +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md b/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md new file mode 100644 index 0000000000..98a527f32a --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md @@ -0,0 +1,285 @@ +# basedpython: forward references outside an annotation + +an annotation is read as deferred, so it may name a class the module defines further down. the same +holds for the type arguments of a specialization written as a value, `list[F]()` or +`Alias = dict[str, F]`, and for the types a typing construct such as `TypeVar` takes as call +arguments: python evaluates these where they stand, so the transpiler writes such a name as the +quoted forward reference python reads lazily (`list["F"]()`) + +```toml +[environment] +python-version = "3.12" +``` + +## a specialization's type arguments may name a later class + +```by +from typing import Callable, Union + +items = list[F]() +Alias = dict[str, F] +Maybe = Union[F, None] +Handler = Callable[[F], F] +nested = dict[str, list[F | None]]() + +def use(alias: Alias, maybe: Maybe, handler: Handler): + reveal_type(items) # revealed: final list[F] + reveal_type(alias) # revealed: dict[str, F] + reveal_type(maybe) # revealed: F | None + reveal_type(handler) # revealed: (F, /) -> F + reveal_type(nested) # revealed: final dict[str, list[F | None]] + +class F: ... +``` + +## a base's type arguments may name a later class + +a base is the object the class is built on, so it is read where it is written, but its type +arguments are a specialization like any other + +```by +class Base[T]: + def get(self) -> T: + raise NotImplementedError + +class Bar(Base[F]): ... + +def use(bar: Bar): + reveal_type(bar.get()) # revealed: F + +class F: ... +``` + +## a tuple type argument may name a later class + +a type argument holding the reference is quoted whole, so the tuple type it lowers to is read lazily +with it + +```by +from typing import TypeVar + +class Foo[T]: + def get(self) -> T: + raise NotImplementedError + +Alias = list[(F, int)] +pairs = dict[str, (F, int)]() +T = TypeVar("T", bound=(F, int)) + +class Bar(Foo[(F, int)]): ... + +def use(alias: Alias, bar: Bar): + reveal_type(alias) # revealed: list[(F, int)] + reveal_type(pairs) # revealed: final dict[str, (F, int)] + reveal_type(bar.get()) # revealed: (F, int) + +class F: ... +``` + +## basedpython type syntax around a later class + +an optional, an intersection or a callable type holding the reference is quoted whole, with what it +lowers to inside the quote + +```by +class Foo[T]: + def get(self) -> T: + raise NotImplementedError + +class G: ... + +Maybe = list[F?] +Both = list[F & G] +Handler = list[(F) -> None] +Method = list[int.() -> F] +maybes = list[F?]() + +class Bar(Foo[F?]): ... + +def use(maybe: Maybe, both: Both, handler: Handler, method: Method, bar: Bar): + reveal_type(maybe) # revealed: list[F | None] + reveal_type(both) # revealed: list[F & G] + reveal_type(handler) # revealed: list[(F, /) -> None] + reveal_type(method) # revealed: list[(int, /) -> F] + reveal_type(maybes) # revealed: final list[F | None] + reveal_type(bar.get()) # revealed: F | None + +class F: ... +``` + +## a `Callable` parameter list written as a tuple may name a later class + +each parameter of the list python's `Callable` takes is a type of its own + +```by +from typing import Callable + +Both = Callable[(F, int), F] +Returns = Callable[(int, str), F] + +def use(both: Both, returns: Returns): + reveal_type(both) # revealed: (F, int, /) -> F + reveal_type(returns) # revealed: (int, str, /) -> F + +class F: ... +``` + +## an anonymous named tuple may name a later class + +its class is written ahead of everything the module binds, so a field naming a later class is a +forward reference there + +```by +Fields = list[(first: F, count: int)] +fields: list[(first: F, count: int)] = [] + +def use(entries: Fields): + reveal_type(entries[0].first) # revealed: F + +class F: ... +``` + +## a typing construct's type arguments may name a later class + +```by +from typing import NamedTuple, NewType, TypedDict, TypeVar + +T = TypeVar("T", bound=F) +Id = NewType("Id", F) +Pair = NamedTuple("Pair", [("first", F)]) +Movie = TypedDict("Movie", {"lead": F}) + +def use(pair: Pair, movie: Movie, t: T) -> T: + reveal_type(pair.first) # revealed: F + reveal_type(movie["lead"]) # revealed: F + return t + +class F: ... +``` + +## a name nothing binds is still unresolved + +```by +items = list[Missing]() # error: [unresolved-reference] +``` + +## a base itself is read where it is written + +the class is built on the object, so a quote could not stand in for it, whether the class is generic +or not + +```by +class Bar(F): # error: [unresolved-reference] + pass + +class GenericBar[T](F): # error: [unresolved-reference] + pass + +class Shaped((*: F)): # error: [unresolved-reference] + pass + +class F: ... +``` + +## a keyword a class is built with is read where it is written + +a metaclass, or a keyword `__init_subclass__` takes, is an object the class is built with + +```by +class Tagged: + def __init_subclass__(cls, tag: object = None): + pass + +class WithMeta(metaclass=Meta): # error: [unresolved-reference] + pass + +class WithTag(Tagged, tag=F): # error: [unresolved-reference] + pass + +class GenericWithMeta[T](metaclass=Meta): # error: [unresolved-reference] + pass + +class Meta(type): ... +class F: ... +``` + +## a `TypedDict`'s `extra_items` may name a later class + +`extra_items=` is the type of the keys a `TypedDict` does not list, so it is a type like a +specialization's arguments + +```by +from typing_extensions import TypedDict + +class Movie(TypedDict, extra_items=F): ... +class Pair[T](TypedDict, extra_items=list[F]): ... + +class F: ... +``` + +## the bases of a class built by a call are read where they are written + +`type()` builds the class on the base objects, as a `class` statement does + +```by +import types + +Built = type("Built", (F,), dict()) # error: [unresolved-reference] +NewBuilt = types.new_class("NewBuilt", (F,)) # error: [unresolved-reference] + +class F: ... +``` + +## a reified class's type arguments are read where they are written + +a reified class keeps its type arguments as runtime values, which a forward reference cannot stand +in for + +```by +class Box[reified T]: + pass + +box = Box[F]() # error: [unresolved-reference] +boxes = list[Box[F]]() # error: [unresolved-reference] + +class F: ... +``` + +## the type a test or a cast checks for is read where it is written + +`is`, `cast!` and `cast?` check the value against the type when they run, and `cast` reads its type +as a type expression rather than a specialization + +```by +from typing import cast + +def make() -> object: + return [] + +value = make() +is_list = value is list[F] # error: [unresolved-reference] +checked = value cast? list[F] # error: [unresolved-reference] +cast_list = cast(list[F], value) # error: [unresolved-reference] + +class F: ... +``` + +## a value inside a specialization is read where it is written + +a `Literal`'s parameters and the metadata of an `Annotated` are values python evaluates while it +builds the specialization, even where the type around them is read as deferred + +```by +from enum import Enum +from typing import Annotated, Literal + +# error: [invalid-type-form] +# error: [unresolved-reference] +literals = list[Literal[E.A]]() +annotated = list[Annotated[int, F()]]() # error: [unresolved-reference] + +class E(Enum): + A = 1 + +class F: ... +``` diff --git a/crates/ty_python_semantic/resources/mdtest/known_constants.md b/crates/ty_python_semantic/resources/mdtest/known_constants.md index fdd396940d..e2752433ce 100644 --- a/crates/ty_python_semantic/resources/mdtest/known_constants.md +++ b/crates/ty_python_semantic/resources/mdtest/known_constants.md @@ -26,6 +26,57 @@ from typing import TYPE_CHECKING as TC reveal_type(TC) # revealed: Literal[True] ``` +### A name assigned the constant, then something else + +A condition reads whatever the name holds where it is tested, so once the alias is reassigned the +branch is decided by the new value: + +```py +from typing import TYPE_CHECKING + +TC = TYPE_CHECKING +TC = False + +if TC: + y = 1 +else: + y = "a" + +reveal_type(y) # revealed: Literal["a"] +``` + +### An alias imported in a class body + +A class body's names are not visible from its methods, so a method testing the same name reads +something else, which nothing here binds: + +```py +class K: + from typing import TYPE_CHECKING as KC + + def m(self): + if KC: # error: [unresolved-reference] + c = 1 + else: + c = "a" + reveal_type(c) # revealed: Literal[1, "a"] +``` + +### An alias imported in an enclosing function + +```py +def outer(): + from typing import TYPE_CHECKING as TC + + def inner(): + if TC: + b = 1 + else: + b = "a" + reveal_type(b) # revealed: Literal[1] + return inner +``` + ### Qualified conditions Qualified references to `TYPE_CHECKING` are evaluated eagerly, allowing narrowing from the diff --git a/crates/ty_python_semantic/resources/mdtest/redundant_condition.md b/crates/ty_python_semantic/resources/mdtest/redundant_condition.md index c762d663ba..f057eb1a12 100644 --- a/crates/ty_python_semantic/resources/mdtest/redundant_condition.md +++ b/crates/ty_python_semantic/resources/mdtest/redundant_condition.md @@ -965,7 +965,7 @@ if BAR >= (3, 14): # no diagnostic reveal_type(TYPE_CHECKINGGGGG) # revealed: Literal[True] -if TYPE_CHECKINGGGGG: # error: [redundant-condition] "This condition is always true" +if TYPE_CHECKINGGGGG: # no diagnostic pass reveal_type(sys_platform) # revealed: Literal["linux"] @@ -1143,6 +1143,27 @@ class PlatformAttributeCycle: pass ``` +## A `TYPE_CHECKING` constant that is not `typing`'s + +ty reads any constant named `TYPE_CHECKING` as true, but only `typing.TYPE_CHECKING` (and its +`typing_extensions` re-export) is a fact about the environment the program is checked in. A module's +own constant of that name, imported under another name, is an ordinary condition: + +`settings.py`: + +```py +TYPE_CHECKING = False +``` + +`main.py`: + +```py +from settings import TYPE_CHECKING as DEBUGGING + +if DEBUGGING: # error: [redundant-condition] "This condition is always true" + pass +``` + ## Environment-dependent assignment guards An assignment can depend on nested conditions or aliases of environment guards. The assigned value diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_che\342\200\246_-_basedpython___a_name_\342\200\246_-_A_reified_call_that_\342\200\246_(18932f76614d8074).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_che\342\200\246_-_basedpython___a_name_\342\200\246_-_A_reified_call_that_\342\200\246_(18932f76614d8074).snap" new file mode 100644 index 0000000000..cce848cabd --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_che\342\200\246_-_basedpython___a_name_\342\200\246_-_A_reified_call_that_\342\200\246_(18932f76614d8074).snap" @@ -0,0 +1,39 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_type_checking_imports.md - basedpython: a name that is not bound where the program reads it - A reified call that has a reason of its own +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md +--- + +# Source files + +## mdtest_snippet.by + +``` + 1 | from typing import TYPE_CHECKING + 2 | + 3 | if TYPE_CHECKING: + 4 | from decimal import Decimal + 5 | + 6 | def pair[reified A, reified B](a: A, b: B) -> str: + 7 | return A.__name__ + B.__name__ + 8 | + 9 | def describe(value: Decimal) -> str: +10 | # error: [unspecialized-reified-generic] +11 | return pair(value, lambda: 1) +``` + +# Diagnostics + +``` +error[unspecialized-reified-generic]: Cannot call reified generic function `pair` without explicit specialization + --> src/mdtest_snippet.by:11:12 + | +11 | return pair(value, lambda: 1) + | ^^^^^^^^^^^^^^^^^^^^^^ +info: inferred type `() -> int` for type parameter `B` has no runtime spelling — specialize with `pair[...]` + +``` diff --git a/crates/ty_python_semantic/resources/mdtest/union_types.md b/crates/ty_python_semantic/resources/mdtest/union_types.md index 0ea3eddaf2..4f673fb78a 100644 --- a/crates/ty_python_semantic/resources/mdtest/union_types.md +++ b/crates/ty_python_semantic/resources/mdtest/union_types.md @@ -150,6 +150,128 @@ def _( reveal_type(u2) # revealed: Literal[Color.RED] ``` +## Methods bound to literals + +```toml +[environment] +python-version = "3.12" +``` + +looking up a method on a union of literals gives a union of bound methods, one for each literal. a +method bound to `"a"` is not redundant with the same method bound to `"b"`, so each one is kept +along with its receiver + +```py +from enum import Enum +from typing import Literal + +class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + +def _(s: Literal["a", "b"], n: Literal[1, 2], color: Literal[Color.RED, Color.GREEN]): + # revealed: (Overload[def lower() -> LiteralString, def lower() -> str]) | (Overload[def lower() -> LiteralString, def lower() -> str]) + reveal_type(s.lower) + reveal_type(s.lower.__self__) # revealed: Literal["a", "b"] + reveal_type(s.lower()) # revealed: LiteralString + # revealed: () | () + reveal_type(s.startswith) + reveal_type(s.startswith("a")) # revealed: bool + reveal_type(n.bit_length.__self__) # revealed: Literal[1, 2] + reveal_type(color.__eq__.__self__) # revealed: Literal[Color.RED, Color.GREEN] +``` + +an enum's `name` is the union of its members' names, so a method called on it is bound to each name + +```py +def _(color: Color): + reveal_type(color.name.lower.__self__) # revealed: Literal["RED", "GREEN", "BLUE"] + reveal_type(color.name.lower()) # revealed: LiteralString +``` + +the same method bound to the same literal appears once. when one receiver is the promotable form of +the literal (the `"a"` written below) and the other is not (the declared `a`), the union keeps the +method bound to the unpromotable one, as a union of the two literals keeps the unpromotable literal. +promotion shows in the inferred list type + +```py +def _(flag: bool, a: Literal["a"]): + reveal_type(a.lower if flag else a.lower) # revealed: Overload[() -> LiteralString, () -> str] + reveal_type(a.lower if flag else "a".lower) # revealed: Overload[() -> LiteralString, () -> str] + reveal_type("a".lower if flag else a.lower) # revealed: Overload[() -> LiteralString, () -> str] + reveal_type(["a"]) # revealed: list[str] + reveal_type([(a.lower if flag else "a".lower).__self__]) # revealed: list[Literal["a"]] + reveal_type([("a".lower if flag else a.lower).__self__]) # revealed: list[Literal["a"]] +``` + +a type that holds every one of the bound methods subsumes them, whichever side of the union it is +on. a method bound to a `LiteralString` is such a type for the same method bound to a string literal + +```py +from typing import Callable +from typing_extensions import LiteralString + +def _(flag: bool, s: Literal["a", "b"], f: Callable[[], str], x: LiteralString): + reveal_type(s.lower if flag else f) # revealed: () -> str + reveal_type(f if flag else s.lower) # revealed: () -> str + reveal_type(s.lower if flag else x.lower) # revealed: Overload[() -> LiteralString, () -> str] + reveal_type(x.lower if flag else s.lower) # revealed: Overload[() -> LiteralString, () -> str] +``` + +literals of different kinds can share a method: `True`, `1` and a member of an `IntEnum` or +`IntFlag` all have `int.bit_length`, and a `StrEnum` member has the methods of `str`. a literal of +one kind is never a literal of another, even where the two compare equal like `True` and `1`, so +neither method stands for the other and both are kept + +```py +from enum import IntEnum, IntFlag, StrEnum + +class Code(IntEnum): + ONE = 1 + TWO = 2 + +class Bits(IntFlag): + ONE = 1 + TWO = 2 + +class Name(StrEnum): + A = "a" + B = "b" + +def _(flag: bool, b: Literal[True], i: Literal[1], a: Literal["a"]): + # revealed: (bound method Literal[True].bit_length() -> int) | (bound method Literal[1].bit_length() -> int) + reveal_type(b.bit_length if flag else i.bit_length) + reveal_type((b.bit_length if flag else i.bit_length).__self__) # revealed: Literal[True, 1] + reveal_type((i.bit_length if flag else b.bit_length).__self__) # revealed: Literal[1, True] + # revealed: (bound method Literal[Code.ONE].bit_length() -> int) | (bound method Literal[1].bit_length() -> int) + reveal_type(Code.ONE.bit_length if flag else i.bit_length) + reveal_type((Code.ONE.bit_length if flag else i.bit_length).__self__) # revealed: Literal[Code.ONE, 1] + reveal_type((i.bit_length if flag else Code.ONE.bit_length).__self__) # revealed: Literal[1, Code.ONE] + reveal_type((Bits.ONE.bit_length if flag else i.bit_length).__self__) # revealed: Literal[Bits.ONE, 1] + reveal_type((i.bit_length if flag else Bits.ONE.bit_length).__self__) # revealed: Literal[1, Bits.ONE] + # revealed: (bound method Literal[Name.A].lower() -> str) | (Overload[() -> LiteralString, () -> str]) + reveal_type(Name.A.lower if flag else a.lower) + reveal_type((Name.A.lower if flag else a.lower).__self__) # revealed: Literal[Name.A, "a"] + reveal_type((a.lower if flag else Name.A.lower).__self__) # revealed: Literal["a", Name.A] +``` + +## Methods bound to template literals + +a template literal holds many strings, like `LiteralString`. a method bound to `f"v{int}"` subsumes +the same method bound to `"v1"` or `"v2"`, but not the one bound to `"x"`, which the template does +not produce + +```by +def _(flag: bool, s: "v1" | "v2" | "x", t: f"v{int}"): + # revealed: (Overload[() -> LiteralString, () -> str]) | (bound method f"v{int}".lower() -> str) + reveal_type(s.lower if flag else t.lower) + # revealed: (bound method f"v{int}".lower() -> str) | (Overload[() -> LiteralString, () -> str]) + reveal_type(t.lower if flag else s.lower) + reveal_type((s.lower if flag else t.lower).__self__) # revealed: "x" | f"v{int}" + reveal_type((t.lower if flag else s.lower).__self__) # revealed: f"v{int}" | "x" +``` + ## Do not erase `Unknown` ```py diff --git a/crates/ty_python_semantic/src/assumed.rs b/crates/ty_python_semantic/src/assumed.rs index 5b9b4618cb..7b855fab4a 100644 --- a/crates/ty_python_semantic/src/assumed.rs +++ b/crates/ty_python_semantic/src/assumed.rs @@ -474,16 +474,22 @@ pub(crate) fn seeded_place<'db>( scope: ScopeId<'db>, place_expr: PlaceExprRef, ) -> Option> { - scope.program(db).assumptions(db)?; - let seeds = seeds(db, scope); - if seeds.is_empty() { + let assumptions = scope.program(db).assumptions(db)?; + let index = semantic_index(db, scope.program_file(db)); + let table = index.place_table(scope.file_scope_id(db)); + let place_id = table.place_id(place_expr)?; + // only a place the debugger observed can have a seed, and that is asked before the seeds are + // built: building them types every observation, and typing an enum member reads its class's + // bases, which are name loads in this same scope — `class Color(Enum)` loading `Enum` would + // otherwise wait on the seeds it is part of building + if !assumptions + .observations(db) + .iter() + .any(|observation| place_id_of(table, &observation.name) == Some(place_id)) + { return None; } - let index = semantic_index(db, scope.program_file(db)); - let place_id = index - .place_table(scope.file_scope_id(db)) - .place_id(place_expr)?; - seeds.get(&place_id).copied() + seeds(db, scope).get(&place_id).copied() } /// whether a use is at or below the line the program is stopped on diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index 141e755c09..f058eb8d17 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -25,7 +25,7 @@ use ruff_db::source::{SourceTextError, source_text}; use rustc_hash::FxHasher; pub use semantic_model::{ Completion, DjangoLookupArgument, ExpectedStringLiteralCompletion, ExtensionOperatorRewrite, - HasDefinition, HasType, ImplicitReceiverReference, MemberDefinition, NameKind, + HasDefinition, HasType, ImplicitReceiverReference, LoweringModel, MemberDefinition, NameKind, PreludeDunderReceiver, SemanticModel, basedpython_augmented_assignment_type, basedpython_class_type, }; @@ -51,6 +51,7 @@ pub use types::conformance::{ConformanceRegistration, WitnessDispatch, WitnessKi pub use types::conversions::{ ConversionImport, ConversionInfo, ConversionRuntime, DISCARD_ADAPTER, }; +pub use types::deferred_imports::DeferrableImport; pub use types::extensions::{ ExtensionAttributeInfo, ExtensionMemberKind, decides_receiver_binding, extension_backing_function, extension_backing_functions, extension_member_kind, @@ -68,13 +69,15 @@ pub use types::reified_infer::{ ArgVariance, ErasedTargetReason, ErasedUnion, ParametricIsPlan, ProtocolMemberCheck, TargetSpelling, }; +pub use types::runtime_spelling::RuntimeSpelling; pub use types::static_resource::{ResourceError, render_as, resolve_static_resource}; pub use types::template::finite_string_set; pub use types::visibility::private_symbols; pub use types::{ - CollectedTest, CollectedTestClass, DisplaySettings, FixtureBinding, FixtureExposure, - FixtureNameSource, ProgramEnvironment, TypeQualifiers, collected_pytest_tests, - fixture_bindings_for_parameter, fixture_exposures_for_definition, pytest_global_plugin_files, + AnonNamedTupleClass, AnonNamedTupleId, CollectedTest, CollectedTestClass, DisplaySettings, + FixtureBinding, FixtureExposure, FixtureNameSource, ProgramEnvironment, TypeQualifiers, + collected_pytest_tests, fixture_bindings_for_parameter, fixture_exposures_for_definition, + pytest_global_plugin_files, }; pub mod api_lockfile; diff --git a/crates/ty_python_semantic/src/place.rs b/crates/ty_python_semantic/src/place.rs index 988909202b..93e01d5218 100644 --- a/crates/ty_python_semantic/src/place.rs +++ b/crates/ty_python_semantic/src/place.rs @@ -3,6 +3,7 @@ pub(crate) mod definitions; use crate::ProgramEnvironment; use itertools::Either; use ruff_index::IndexSlice; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; use ruff_python_ast::{self as ast, PythonVersion}; use rustc_hash::FxHashMap; use ty_module_resolver::{ @@ -18,6 +19,7 @@ use crate::reachability::{ NarrowingProjector, ReachabilityEvaluationCache, evaluate_reachability, evaluate_reachability_with_cache, }; +use crate::types::check_order::InferredInCheckOrder; use crate::types::{ DynamicType, KnownClass, MemberLookupPolicy, Type, TypeAndQualifiers, TypeQualifiers, UnionBuilder, UnionType, binding_type, inferred_declaration, is_discarded_dict_key_assignment, @@ -25,7 +27,7 @@ use crate::types::{ }; use crate::{Db, FxIndexSet, FxOrderSet}; use ruff_db::parsed::{ParsedModuleRef, parsed_module}; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextRange}; use ty_python_core::definition::{Definition, DefinitionKind, DefinitionState}; use ty_python_core::narrowing_constraints::ScopedNarrowingConstraint; use ty_python_core::place::{PlaceExpr, ScopedPlaceId}; @@ -33,7 +35,7 @@ use ty_python_core::predicate::{Predicate, ScopedPredicateId}; use ty_python_core::reachability_constraints::{ ReachabilityConstraints, ScopedReachabilityConstraintId, }; -use ty_python_core::scope::ScopeId; +use ty_python_core::scope::{FileScopeId, ScopeId}; use ty_python_core::{ BindingWithConstraints, BindingWithConstraintsIterator, BoundnessAnalysis, DeclarationWithConstraint, DeclarationsIterator, ProgramFile, SemanticIndex, Truthiness, @@ -545,6 +547,677 @@ pub(crate) fn global_symbol<'db>( }) } +/// basedpython: what a name the transpiled program reads holds when the program runs +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeBinding { + /// on every path to the read, a binding the program runs holds the name, and it is the + /// binding the checker resolved + Bound, + /// on some path to the read nothing the program runs has bound the name: it is bound only + /// under `if TYPE_CHECKING:`, only on some paths, or only further on + Unbound, + /// the program binds the name, but a binding the checker resolves never runs — it is made + /// under `if TYPE_CHECKING:` — so the value read need not be what the checker resolved + NotTheCheckers, +} + +impl RuntimeBinding { + /// why a runtime check naming `name` cannot run here, for a diagnostic's message + pub(crate) fn why_not_checkable(self, name: &str) -> String { + match self { + RuntimeBinding::Bound => format!("`{name}` is the class the checker resolved"), + RuntimeBinding::Unbound => format!("`{name}` is not bound when it runs"), + RuntimeBinding::NotTheCheckers => format!( + "the `{name}` bound when it runs need not be the class the checker resolved" + ), + } + } + + /// why `name`, read where a lowering would write it, cannot be read there + pub(crate) fn describe(self, name: &str) -> String { + match self { + RuntimeBinding::Bound => format!("`{name}` holds the binding the checker resolved"), + RuntimeBinding::Unbound => format!( + "`{name}` is not bound when this runs: it is bound only under \ + `if TYPE_CHECKING:`, only on some paths, or further on" + ), + RuntimeBinding::NotTheCheckers => format!( + "the `{name}` bound when this runs need not be the class the checker resolved: \ + the checker reads a binding of `{name}` made under `if TYPE_CHECKING:`, which \ + never runs" + ), + } + } +} + +/// a name the transpiled program reads, and where it reads it +#[derive(Clone, Copy)] +pub(crate) enum RuntimeRead<'a> { + /// a name the source writes, read where it is written + Written(ast::ExprRef<'a>), + /// a module-level name a lowering writes into code at `range` of `scope` + Global { name: &'a str, range: TextRange }, +} + +/// basedpython: what `read` holds when the transpiled program runs — the one question every +/// lowering that writes a name into code that runs asks first. +/// +/// a name the source writes is read where it is written, and holds what ty resolves there. a +/// module-level name a lowering writes is read inside one of the module's own statements, +/// which the program runs in order; that statement may be a `def` or a `class` whose body runs +/// later, but which code runs it, and when, is not known here, so the name is asked about at +/// the point the statement itself runs. it is bound there if every binding of it is made above +/// that statement, and either, by the end of the module, every path has bound it and none has +/// unbound it — a `del`, a binding made only in one branch of a conditional expression — or +/// every binding of it is one a statement at the top of the module always makes once it +/// completes, which a module the checker finds never ends, one that stops with `sys.exit()` +/// say, needs. a binding made under `if TYPE_CHECKING:` never runs, which is how an import +/// cycle is broken +/// +/// the answer reads the bindings the module makes after the read, and the conditions they +/// are made under, so it is asked only once the scopes around the read are inferred +/// ([`InferredInCheckOrder`]), never by inference itself: asked from inside the module's +/// inference, each later condition would be inferred inside the inference of the next. +/// inference records where to ask ([`crate::types::runtime_reads`]) +pub(crate) fn runtime_binding<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + read: RuntimeRead<'_>, + inferred: InferredInCheckOrder<'db>, +) -> RuntimeBinding { + // a question is asked of the file whose inference the proof is for, and of no other + let file = inferred.file(); + let index = semantic_index(db, file); + let module = parsed_module(db, file.python_file(db)).load(db); + match read { + RuntimeRead::Written(expr) => written_runtime_binding(db, env, file, index, expr), + RuntimeRead::Global { name, range, .. } => { + let global = global_scope(db, file); + let Some(symbol) = place_table(db, global).symbol_id(name) else { + return RuntimeBinding::Unbound; + }; + let Some(statement) = module + .suite() + .iter() + .find(|statement| statement.range().contains_range(range)) + else { + return RuntimeBinding::Unbound; + }; + let use_def = use_def_map(db, global); + match tally_bindings(db, index, use_def.end_of_scope_symbol_bindings(symbol)) + .runtime_binding() + { + RuntimeBinding::Bound => {} + _ if made_by_completed_statements(db, file, &module, name) => {} + other => return other, + } + let bindings = use_def.reachable_symbol_bindings(symbol); + let predicates = bindings.predicates(); + let reachability_constraints = bindings.reachability_constraints(); + let above = bindings.into_iter().all(|binding| { + let DefinitionState::Defined(definition) = binding.binding else { + return true; + }; + evaluate_reachability_with_cache( + db, + None, + reachability_constraints, + predicates, + binding.reachability_constraint, + ) + .is_always_false() + || definition.full_range(db, &module).range().end() <= statement.start() + }); + if above { + RuntimeBinding::Bound + } else { + RuntimeBinding::Unbound + } + } + } +} + +/// basedpython: whether every binding `file` makes of the module-level `name` binds one and +/// the same class, so that whenever the program finds the name bound it holds that class. a +/// name that is not yet bound, or bound only on some paths, passes: what this answers is what +/// the name holds once it is there +pub(crate) fn binds_one_class_wherever_bound<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + name: &str, +) -> bool { + let Some(bindings) = bindings_alone(db, file, name) else { + return false; + }; + let mut classes = bindings + .into_iter() + .map(|definition| binding_type(db, definition)); + let Some(Type::ClassLiteral(class)) = classes.next() else { + return false; + }; + classes.all(|bound| bound == Type::ClassLiteral(class)) +} + +/// whether every binding `file` makes of the module-level `name` is made by a statement at the +/// top of the module that always makes it once it completes — an import, a `class`, a `def`, +/// an assignment — rather than one inside an `if`, a `try` or an expression. the module runs +/// those statements in order, and one that raises stops it, so whatever runs below such a +/// binding runs after it was made +fn made_by_completed_statements<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + module: &ParsedModuleRef, + name: &str, +) -> bool { + bindings_alone(db, file, name).is_some_and(|bindings| { + !bindings.is_empty() + && bindings.into_iter().all(|definition| { + let range = definition.full_range(db, module).range(); + let Some(statement) = module + .suite() + .iter() + .find(|statement| statement.range().contains_range(range)) + else { + return false; + }; + matches!( + (definition.kind(db), statement), + (DefinitionKind::Import(_), ast::Stmt::Import(_)) + | (DefinitionKind::ImportFrom(_), ast::Stmt::ImportFrom(_)) + | (DefinitionKind::Class(_), ast::Stmt::ClassDef(_)) + | (DefinitionKind::Function(_), ast::Stmt::FunctionDef(_)) + | (DefinitionKind::TypeAlias(_), ast::Stmt::TypeAlias(_)) + | (DefinitionKind::Assignment(_), ast::Stmt::Assign(_)) + | ( + DefinitionKind::AnnotatedAssignment(_), + ast::Stmt::AnnAssign(_) + ) + ) + }) + }) +} + +/// every binding `file`'s global scope makes of the module-level `name`, reached by the checker +/// or not, when nothing else changes what the name holds: none is made only for the checker, +/// nothing unbinds the name, and no other scope declares it `global` +fn bindings_alone<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + name: &str, +) -> Option>> { + let global = global_scope(db, file); + let symbol = place_table(db, global).symbol_id(name)?; + if declared_global_elsewhere(db, file, name) { + return None; + } + let use_def = use_def_map(db, global); + let place = ScopedPlaceId::Symbol(symbol); + if use_def.deletions().any(|(_, deleted)| deleted == place) { + return None; + } + let module = parsed_module(db, file.python_file(db)).load(db); + let in_stub = file.file(db).is_stub(db); + let mut bindings = Vec::new(); + for (_, definition, _) in use_def.definitions_with_usage() { + if definition.place(db) != place + || !definition.kind(db).category(in_stub, &module).is_binding() + { + continue; + } + if made_only_for_the_checker(db, definition) { + return None; + } + bindings.push(definition); + } + Some(bindings) +} + +/// whether a scope of `file` other than its global one declares `name` `global`, which is how +/// a function rebinds a module-level name. the module's own bindings of it do not show that +fn declared_global_elsewhere(db: &dyn Db, file: ProgramFile<'_>, name: &str) -> bool { + let global = global_scope(db, file).file_scope_id(db); + semantic_index(db, file).scope_ids().any(|scope| { + let table = place_table(db, scope); + scope.file_scope_id(db) != global + && table + .symbol_id(name) + .is_some_and(|id| table.symbol(id).is_global()) + }) +} + +/// [`runtime_binding`] of a name the source writes, resolved where it is written +fn written_runtime_binding<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: ProgramFile<'db>, + index: &'db SemanticIndex<'db>, + expr: ast::ExprRef<'_>, +) -> RuntimeBinding { + // a name the index never saw (inside a string annotation, or a word basedpython reads + // as syntax) is not one the program reads as a name + let (Some(scope), Some(_)) = (index.try_expression_scope_id(&expr), index.try_use_id(expr)) + else { + return RuntimeBinding::Bound; + }; + let scope = scope.to_scope_id(db, file); + let Some(place) = PlaceExpr::try_from_expr(expr) else { + return RuntimeBinding::Bound; + }; + let mut resolution = + resolve_place_load(db, index, scope, place, PlaceLoadMode::AtExpression(expr)); + loop { + match resolution.next() { + Some(PlaceLoadResolutionStep::Source(source)) => { + let tally = runtime_tally(db, env, index, source); + if !tally.is_empty() { + return tally.runtime_binding(); + } + } + Some(PlaceLoadResolutionStep::MemberResolutionCondition(_)) => {} + Some(PlaceLoadResolutionStep::Exhausted(_)) | None => break, + } + } + // nothing binds it on the way here: either the program binds it only further on, or + // no binding in the program supplies it — a name basedpython resolves itself, which the + // transpiler makes available + match is_forward_reference(db, env, file, expr) { + Some(true) => RuntimeBinding::Unbound, + Some(false) | None => RuntimeBinding::Bound, + } +} + +/// basedpython: whether `definition` may be made only for the checker: the checker reads +/// `typing.TYPE_CHECKING` as `True` and the program reads it as `False`, so a binding whose +/// making, or whose value, depends on the constant need not be one the program makes. +/// +/// a binding depends on the constant when a condition deciding whether it is made reads it — +/// the test of an enclosing `if`, `elif` or `while` and those before it, the subject, patterns +/// and guards of an enclosing `match` case and those before it, or an expression around the +/// binding such as a conditional expression or `and` holding a walrus — +/// or when its value reads it, as `Foo = _F if TYPE_CHECKING else None` does. a condition +/// reads the constant when any name in it resolves to a binding that carries it: the +/// constant's declaration, imported under any name, or a binding whose value reads it. this +/// does not ask which way the condition goes: `if not TYPE_CHECKING:` counts, and so does +/// `if TYPE_CHECKING and x:`, whose body the program may never run either +#[salsa::tracked(returns(copy), cycle_initial=|_, _, _| false, heap_size=ruff_memory_usage::heap_size)] +pub(crate) fn made_only_for_the_checker<'db>(db: &'db dyn Db, definition: Definition<'db>) -> bool { + let file = definition.program_file(db); + let index = semantic_index(db, file); + let module = parsed_module(db, file.python_file(db)).load(db); + let range = definition.full_range(db, &module).range(); + if index.is_in_type_checking_block(definition.file_scope(db), range) { + return true; + } + let reader = TypeCheckingReader { db, file }; + statements_check_only(&reader, module.suite(), range) + || carries_type_checking(db, definition) + || match definition.kind(db) { + // a binding imported from another module is made only for the checker when the + // binding it imports is + DefinitionKind::Import(_) + | DefinitionKind::ImportFrom(_) + | DefinitionKind::StarImport(_) => { + resolved_definitions(db, definition).any(|resolved| match resolved { + ResolvedTarget::Definition(resolved) => { + resolved != definition && made_only_for_the_checker(db, resolved) + } + ResolvedTarget::Module(_) => false, + }) + } + _ => false, + } +} + +/// whether `range`, inside one of `body`'s statements, is reached only past a condition that +/// reads the `TYPE_CHECKING` constant +fn statements_check_only( + reader: &TypeCheckingReader<'_>, + body: &[ast::Stmt], + range: TextRange, +) -> bool { + let Some(statement) = body + .iter() + .find(|statement| statement.range().contains_range(range)) + else { + return false; + }; + let holds = |body: &[ast::Stmt]| body.iter().any(|s| s.range().contains_range(range)); + // the conditions decided before `range` is reached, and the body holding it, if any + let (guards, inner): (Vec, Option<&[ast::Stmt]>) = match statement { + ast::Stmt::If(node) => { + let clauses = std::iter::once((Some(&*node.test), &node.body[..], node.test.start())) + .chain( + node.elif_else_clauses + .iter() + .map(|clause| (clause.test.as_ref(), &clause.body[..], clause.start())), + ) + .collect::>(); + let taken = clauses + .iter() + .rposition(|(_, _, start)| *start <= range.start()) + .unwrap_or_default(); + let guards = clauses[..=taken] + .iter() + .filter_map(|(test, _, _)| test.map(Guard::Expr)) + .collect(); + let body = clauses[taken].1; + (guards, holds(body).then_some(body)) + } + ast::Stmt::While(node) => ( + vec![Guard::Expr(&node.test)], + [&node.body[..], &node.orelse[..]] + .into_iter() + .find(|body| holds(body)), + ), + // the checker never takes the body of a `for` or an `except` clause to run for + // certain, so a binding made only there is one it already finds possibly unbound + ast::Stmt::For(node) => ( + Vec::new(), + [&node.body[..], &node.orelse[..]] + .into_iter() + .find(|body| holds(body)), + ), + ast::Stmt::Try(node) => ( + Vec::new(), + std::iter::once(&node.body[..]) + .chain(node.handlers.iter().map(|handler| match handler { + ast::ExceptHandler::ExceptHandler(handler) => &handler.body[..], + })) + .chain([&node.orelse[..], &node.finalbody[..]]) + .find(|body| holds(body)), + ), + ast::Stmt::Match(node) => { + // a case is tried only once the subject is evaluated and every case before it + // has failed to match + let taken = node + .cases + .iter() + .position(|case| case.range().contains_range(range)); + let tried = taken.map_or(0, |taken| taken + 1); + let guards = std::iter::once(Guard::Expr(&node.subject)) + .chain(node.cases[..tried].iter().flat_map(|case| { + std::iter::once(Guard::Pattern(&case.pattern)) + .chain(case.guard.as_deref().map(Guard::Expr)) + })) + .collect(); + let body = taken.map(|taken| &node.cases[taken].body[..]); + (guards, body.filter(|body| holds(body))) + } + ast::Stmt::With(node) => (Vec::new(), holds(&node.body).then_some(&node.body[..])), + ast::Stmt::FunctionDef(node) => (Vec::new(), holds(&node.body).then_some(&node.body[..])), + ast::Stmt::ClassDef(node) => (Vec::new(), holds(&node.body).then_some(&node.body[..])), + _ => (Vec::new(), None), + }; + if guards.into_iter().any(|guard| reader.reads(guard)) { + return true; + } + match inner { + Some(body) => statements_check_only(reader, body, range), + // the binding is made in the statement itself, by an expression it may stand inside + None => { + let mut around = AroundBinding { + reader, + range, + found: false, + }; + source_order::walk_stmt(&mut around, statement); + around.found + } + } +} + +/// a condition deciding whether a binding is made +#[derive(Clone, Copy)] +enum Guard<'a> { + Expr(&'a ast::Expr), + Pattern(&'a ast::Pattern), +} + +/// finds an expression standing around a binding's range that reads the `TYPE_CHECKING` +/// constant, without entering the statement's nested bodies +struct AroundBinding<'r, 'db> { + reader: &'r TypeCheckingReader<'db>, + range: TextRange, + found: bool, +} + +impl<'a> SourceOrderVisitor<'a> for AroundBinding<'_, '_> { + fn visit_body(&mut self, _body: &'a [ast::Stmt]) {} + + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if self.found || !expr.range().contains_range(self.range) || expr.range() == self.range { + return; + } + if self.reader.reads(Guard::Expr(expr)) { + self.found = true; + } else { + source_order::walk_expr(self, expr); + } + } +} + +/// answers whether an expression of `file` reads the `TYPE_CHECKING` constant +struct TypeCheckingReader<'db> { + db: &'db dyn Db, + file: ProgramFile<'db>, +} + +impl<'db> TypeCheckingReader<'db> { + fn reads(&self, guard: Guard<'_>) -> bool { + let mut visitor = ReadsTypeChecking { + reader: self, + found: false, + }; + match guard { + Guard::Expr(expr) => visitor.visit_expr(expr), + Guard::Pattern(pattern) => visitor.visit_pattern(pattern), + } + visitor.found + } + + /// the bindings `expr` reads: those of a name that reach it, or the module members an + /// attribute of a module names + fn definitions(&self, expr: &ast::Expr) -> Vec> { + match expr { + ast::Expr::Name(_) => self.reaching_definitions(expr), + ast::Expr::Attribute(attribute) => self + .definitions(&attribute.value) + .into_iter() + .flat_map(|definition| resolved_definitions(self.db, definition)) + .collect::>() + .into_iter() + .flat_map(|module| match module { + ResolvedTarget::Module(module) => { + crate::types::definition_resolution::find_symbol_in_scope( + self.db, + global_scope(self.db, module), + attribute.attr.as_str(), + ) + .into_iter() + .collect() + } + ResolvedTarget::Definition(_) => Vec::new(), + }) + .collect(), + _ => Vec::new(), + } + } + + /// the bindings of a name the program reads that reach it + fn reaching_definitions(&self, expr: &ast::Expr) -> Vec> { + let db = self.db; + let index = semantic_index(db, self.file); + let expr = ast::ExprRef::from(expr); + let (Some(scope), Some(_)) = (index.try_expression_scope_id(&expr), index.try_use_id(expr)) + else { + return Vec::new(); + }; + let Some(place) = PlaceExpr::try_from_expr(expr) else { + return Vec::new(); + }; + let mut resolution = resolve_place_load( + db, + index, + scope.to_scope_id(db, self.file), + place, + PlaceLoadMode::AtExpression(expr), + ); + loop { + match resolution.next() { + Some(PlaceLoadResolutionStep::Source(PlaceLoadSource { + kind: PlaceLoadSourceKind::Bindings(bindings), + .. + })) => { + let predicates = bindings.predicates(); + let reachability_constraints = bindings.reachability_constraints(); + let definitions: Vec> = bindings + .filter(|binding| { + !evaluate_reachability_with_cache( + db, + None, + reachability_constraints, + predicates, + binding.reachability_constraint, + ) + .is_always_false() + }) + .filter_map(|binding| match binding.binding { + DefinitionState::Defined(definition) => Some(definition), + DefinitionState::Undefined | DefinitionState::Deleted => None, + }) + .collect(); + if !definitions.is_empty() { + return definitions; + } + } + Some( + PlaceLoadResolutionStep::Source(_) + | PlaceLoadResolutionStep::MemberResolutionCondition(_), + ) => {} + Some(PlaceLoadResolutionStep::Exhausted(_)) | None => return Vec::new(), + } + } + } +} + +struct ReadsTypeChecking<'r, 'db> { + reader: &'r TypeCheckingReader<'db>, + found: bool, +} + +impl<'a> SourceOrderVisitor<'a> for ReadsTypeChecking<'_, '_> { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if self.found { + return; + } + if matches!(expr, ast::Expr::Name(_) | ast::Expr::Attribute(_)) + && self + .reader + .definitions(expr) + .into_iter() + .any(|definition| carries_type_checking(self.reader.db, definition)) + { + self.found = true; + return; + } + source_order::walk_expr(self, expr); + } +} + +/// what an import resolves to: a binding in the module it imports from, or a module +enum ResolvedTarget<'db> { + Definition(Definition<'db>), + Module(ProgramFile<'db>), +} + +/// `definition`, or the bindings an import resolves to, through every alias +fn resolved_definitions<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> impl Iterator> + 'db { + let env = ProgramEnvironment::from_file(definition.program_file(db)); + crate::types::definition_resolution::resolve_definition( + db, + &env, + definition, + None, + crate::types::definition_resolution::ImportAliasResolution::ResolveAliases, + ) + .into_iter() + .filter_map(|resolved| match resolved { + crate::types::definition_resolution::ResolvedDefinition::Definition(definition) => { + Some(ResolvedTarget::Definition(definition)) + } + crate::types::definition_resolution::ResolvedDefinition::Module(module) => { + Some(ResolvedTarget::Module(module)) + } + crate::types::definition_resolution::ResolvedDefinition::FileWithRange(_) => None, + }) +} + +/// whether the value `definition` binds carries the `TYPE_CHECKING` constant: it is the +/// constant's declaration, reached through any import of it, or a binding computed from an +/// expression that reads the constant. +/// +/// ty reads every binding named `TYPE_CHECKING` as `True`, the constant's own declaration in +/// `typing` among them, and a module that cannot import `typing` declares its own as `False` +#[salsa::tracked(returns(copy), cycle_initial=|_, _, _| false, heap_size=ruff_memory_usage::heap_size)] +fn carries_type_checking<'db>(db: &'db dyn Db, definition: Definition<'db>) -> bool { + resolved_definitions(db, definition).any(|resolved| { + let ResolvedTarget::Definition(resolved) = resolved else { + return false; + }; + let file = resolved.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + let reader = TypeCheckingReader { db, file }; + let reads = |expr: &ast::Expr| reader.reads(Guard::Expr(expr)); + let names_the_constant = |target: &ast::Expr| { + target + .as_name_expr() + .is_some_and(|name| name.id == "TYPE_CHECKING") + }; + match resolved.kind(db) { + DefinitionKind::Assignment(assignment) => { + names_the_constant(assignment.target(&module)) || reads(assignment.value(&module)) + } + DefinitionKind::AnnotatedAssignment(assignment) => { + names_the_constant(assignment.target(&module)) + || assignment.value(&module).is_some_and(reads) + } + DefinitionKind::NamedExpression(named) => reads(&named.node(&module).value), + DefinitionKind::AugmentedAssignment(assignment) => { + reads(&assignment.node(&module).value) + } + _ => false, + } + }) +} + +/// basedpython: whether `definition` is the declaration of `typing.TYPE_CHECKING` (or of +/// `typing_extensions.TYPE_CHECKING`, which re-exports it) +pub(crate) fn declares_typing_type_checking<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> bool { + let file = definition.program_file(db); + let is_typing = file_to_module(db, file.resolver_file(db)).is_some_and(|module| { + module.is_known(db, KnownModule::Typing) + || module.is_known(db, KnownModule::TypingExtensions) + }); + if !is_typing { + return false; + } + let module = parsed_module(db, file.python_file(db)).load(db); + let target = match definition.kind(db) { + DefinitionKind::Assignment(assignment) => assignment.target(&module), + DefinitionKind::AnnotatedAssignment(assignment) => assignment.target(&module), + _ => return false, + }; + target + .as_name_expr() + .is_some_and(|name| name.id == "TYPE_CHECKING") +} + /// Infers the public type of an imported symbol. /// /// If `requires_explicit_reexport` is [`None`], it will be inferred from the file's source type. @@ -1781,7 +2454,6 @@ pub(crate) fn is_forward_reference<'db>( let index = semantic_index(db, file); let scope = index.try_expression_scope_id(&expr)?.to_scope_id(db, file); index.try_use_id(expr)?; - let module = parsed_module(db, file.python_file(db)).load(db); let mut resolution = resolve_place_load( db, @@ -1793,7 +2465,7 @@ pub(crate) fn is_forward_reference<'db>( let bound_where_written = loop { match resolution.next() { Some(PlaceLoadResolutionStep::Source(source)) => { - match runtime_definedness(db, env, index, &module, source) { + match runtime_definedness(db, env, index, source) { RuntimeDefinedness::Defined => break true, RuntimeDefinedness::PossiblyUndefined => break false, RuntimeDefinedness::Undefined => {} @@ -1856,6 +2528,39 @@ fn lexical_binding_exists<'db>(db: &'db dyn Db, source: PlaceLoadSource<'db>) -> } } +/// what the bindings of one source of a place load hold once the program is running +#[derive(Default, Clone, Copy)] +struct BindingTally { + /// a binding the program runs holds a value on some path + runs: bool, + /// on some path nothing holds a value + unbound: bool, + /// a binding the checker sees is made under `if TYPE_CHECKING:`, and never runs + checker_only: bool, +} + +impl BindingTally { + fn is_empty(self) -> bool { + !self.runs && !self.checker_only + } + + fn definedness(self) -> RuntimeDefinedness { + match (self.runs, self.unbound || self.checker_only) { + (true, false) => RuntimeDefinedness::Defined, + (true, true) => RuntimeDefinedness::PossiblyUndefined, + (false, _) => RuntimeDefinedness::Undefined, + } + } + + fn runtime_binding(self) -> RuntimeBinding { + match (self.runs, self.unbound, self.checker_only) { + (true, false, false) => RuntimeBinding::Bound, + (true, _, true) => RuntimeBinding::NotTheCheckers, + _ => RuntimeBinding::Unbound, + } + } +} + /// Whether one source of a place load holds a value once the program is running. enum RuntimeDefinedness { Defined, @@ -1869,33 +2574,33 @@ fn runtime_definedness<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, index: &'db SemanticIndex<'db>, - module: &ParsedModuleRef, source: PlaceLoadSource<'db>, ) -> RuntimeDefinedness { + runtime_tally(db, env, index, source).definedness() +} + +fn runtime_tally<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + index: &'db SemanticIndex<'db>, + source: PlaceLoadSource<'db>, +) -> BindingTally { match source.kind { - PlaceLoadSourceKind::Bindings(bindings) => { - bindings_runtime_definedness(db, index, module, bindings) - } + PlaceLoadSourceKind::Bindings(bindings) => tally_bindings(db, index, bindings), // a lazy scope reads the owner's value when it runs, by which point the owner has made // every binding it is going to PlaceLoadSourceKind::DefinitionsFromOwningScope { scope, id } => { - bindings_runtime_definedness( - db, - index, - module, - use_def_map(db, scope).end_of_scope_bindings(id), - ) + tally_bindings(db, index, use_def_map(db, scope).end_of_scope_bindings(id)) } PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::ExplicitGlobalSymbol { file, name }) => { let global = global_scope(db, file); match place_table(db, global).symbol_id(&name) { - Some(symbol) => bindings_runtime_definedness( + Some(symbol) => tally_bindings( db, index, - module, use_def_map(db, global).end_of_scope_symbol_bindings(symbol), ), - None => RuntimeDefinedness::Undefined, + None => BindingTally::default(), } } PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::Builtin(name)) => { @@ -1908,30 +2613,30 @@ fn runtime_definedness<'db>( defined_if(module_type_implicit_global_symbol(db, file, &name)) } PlaceLoadSourceKind::Observed(_) - | PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::DunderClass(_)) => { - RuntimeDefinedness::Defined - } + | PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::DunderClass(_)) => BindingTally { + runs: true, + ..BindingTally::default() + }, } } /// an implicit source holds a value only where its lookup finds one -fn defined_if(place: PlaceAndQualifiers<'_>) -> RuntimeDefinedness { - if place.place.is_definitely_bound() { - RuntimeDefinedness::Defined - } else { - RuntimeDefinedness::Undefined +fn defined_if(place: PlaceAndQualifiers<'_>) -> BindingTally { + BindingTally { + runs: place.place.is_definitely_bound(), + ..BindingTally::default() } } -fn bindings_runtime_definedness<'db>( +fn tally_bindings<'db>( db: &'db dyn Db, index: &'db SemanticIndex<'db>, - module: &ParsedModuleRef, bindings: BindingWithConstraintsIterator<'_, 'db>, -) -> RuntimeDefinedness { +) -> BindingTally { let predicates = bindings.predicates(); let reachability_constraints = bindings.reachability_constraints(); - let (mut defined, mut undefined) = (false, false); + let mut tally = BindingTally::default(); + let mut checker_only_place = None; for binding in bindings { if evaluate_reachability_with_cache( db, @@ -1945,22 +2650,52 @@ fn bindings_runtime_definedness<'db>( continue; } match binding.binding { - DefinitionState::Defined(definition) - if !index.is_in_type_checking_block( - definition.file_scope(db), - definition.full_range(db, module).range(), - ) => - { - defined = true; + DefinitionState::Defined(definition) if made_only_for_the_checker(db, definition) => { + tally.checker_only = true; + checker_only_place = Some((definition.file_scope(db), definition.place(db))); } - _ => undefined = true, + DefinitionState::Defined(_) => tally.runs = true, + DefinitionState::Undefined | DefinitionState::Deleted => tally.unbound = true, } } - match (defined, undefined) { - (true, false) => RuntimeDefinedness::Defined, - (true, true) => RuntimeDefinedness::PossiblyUndefined, - (false, _) => RuntimeDefinedness::Undefined, + // the checker reads every `if TYPE_CHECKING:` block as taken, so a binding the program + // makes in its `else:` instead is one it never sees reach anything. the program still + // makes it, so the name holds something when it runs — not what the checker resolved + if !tally.runs + && let Some((scope, place)) = checker_only_place + { + tally.runs = binds_where_the_checker_does_not_look(db, index, scope, place); } + tally +} + +/// whether `scope` binds `place` anywhere the checker does not vouch for as unmade: outside +/// every branch made only for the checker, or in one the checker never reaches. the checker +/// reads every `if TYPE_CHECKING:` block as taken, so a binding in its `else:` is one the +/// checker sees reach nothing, and one the program makes +fn binds_where_the_checker_does_not_look<'db>( + db: &'db dyn Db, + index: &'db SemanticIndex<'db>, + scope: FileScopeId, + place: ScopedPlaceId, +) -> bool { + let bindings = index.use_def_map(scope).reachable_bindings(place); + let predicates = bindings.predicates(); + let reachability_constraints = bindings.reachability_constraints(); + bindings.into_iter().any(|binding| { + let DefinitionState::Defined(definition) = binding.binding else { + return false; + }; + !made_only_for_the_checker(db, definition) + || evaluate_reachability_with_cache( + db, + None, + reachability_constraints, + predicates, + binding.reachability_constraint, + ) + .is_always_false() + }) } /// The type declared for the place a binding writes to. diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 0bc8ef0341..0c74355d2a 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -1,3 +1,5 @@ +use std::sync::OnceLock; + use compact_str::CompactString; use ruff_db::PythonFile; use ruff_db::files::{File, FilePath}; @@ -15,12 +17,19 @@ use ty_module_resolver::{ }; use crate::Db; +use crate::RuntimeSpelling; use crate::place::implicit_globals::all_implicit_module_globals; use crate::place::imported_symbol; use crate::place::{builtins_module_scope, implicit_builtins_symbol_scope}; +use crate::types::AnonNamedTupleClass; +use crate::types::TypeExpressionFlags; +use crate::types::check_order::{InferredInCheckOrder, infer_file_in_check_order}; use crate::types::ide_support::{ImportAliasResolution, definition_for_name}; use crate::types::implicit_names::implicit_name; use crate::types::list_members::{all_members, all_reachable_members}; +use crate::types::reified_infer::WrittenTarget; +use crate::types::runtime_spelling::RuntimeSite; +use crate::types::soundness::CastCheck; use crate::types::{ ClassPatternPositionalResult, ClassPatternPositionalSource, CycleDetector, ProgramEnvironment, SpecialFormType, Type, TypeQualifiers, binding_type, class_pattern_positional_result, @@ -144,6 +153,11 @@ impl<'db> SemanticModel<'db> { crate::types::visibility::resolves_to_module_scope(self.db, self.file, reference) } + /// basedpython: see `crate::types::visibility::reads_class_scope_binding` + pub fn reads_class_scope_binding(&self, reference: &ast::ExprName) -> Option { + crate::types::visibility::reads_class_scope_binding(self.db, self.file, reference) + } + /// basedpython: the module-level names this file declares `private` — the /// set `private-import` reads, and whose references the lowering renames pub fn private_module_symbols(&self) -> Vec { @@ -216,46 +230,6 @@ impl<'db> SemanticModel<'db> { .to_instance_approximation(self.db, &self.program_environment()) } - /// basedpython: the source text of the specialization step the transpiler - /// splices in after the callee of a bare reified-generic call (`f(1)` → - /// `"[int]"`). `None` when the call is not a bare reified-generic call or - /// no injectable spelling exists — the checker reports the latter as - /// `unspecialized-reified-generic` - pub fn reified_call_specialization(&self, call: &ast::ExprCall) -> Option { - let env = &self.program_environment(); - let db = self.db; - let callee_ty = call.func.inferred_type(self)?; - let function = match callee_ty { - Type::FunctionLiteral(function) => function, - Type::BoundMethod(method) => method.function(db), - _ => return None, - }; - if function.is_classmethod(db) || !function.is_unspecialized_reified(db) { - return None; - } - let mut positional = Vec::with_capacity(call.arguments.args.len()); - for argument in &call.arguments.args { - if argument.is_starred_expr() { - return None; - } - positional.push(argument.inferred_type(self)?); - } - let mut keywords = Vec::with_capacity(call.arguments.keywords.len()); - for keyword in &call.arguments.keywords { - let name = keyword.arg.as_ref()?; - keywords.push((name.as_str(), keyword.value.inferred_type(self)?)); - } - crate::types::reified_infer::injectable_call_specialization( - db, - env, - self.file(), - callee_ty, - function, - positional, - keywords, - ) - } - /// basedpython: the runtime registrations a conformance extension /// (`extension str(A):`) emits — one per interface in its header. /// @@ -329,21 +303,18 @@ impl<'db> SemanticModel<'db> { crate::types::conformance::eagerly_imported_modules(self.db, self.file.file(db)) } - /// basedpython: the names this file imports that must be bound to the real - /// object rather than to a lazy proxy — the exception classes and the special - /// forms. + /// basedpython: the names this file imports that the lazy-import polyfill binds where the + /// import is written rather than deferring — the exception classes and the special forms /// - /// `except` is one place cpython refuses a stand-in: it checks that what it - /// catches is a class inheriting `BaseException`, and never consults - /// `__instancecheck__`. A proxy there raises `TypeError` from the handler — - /// the line least likely to be covered by a happy-path test — so an - /// exception class is imported eagerly and the proxy never stands where it - /// cannot work. + /// A deferred name is resolved where it is first read, and for an exception class that is + /// a handler: an import that fails there would surface while another exception is being + /// handled, and replace it. So an exception class is imported where the import is written, + /// where a failure surfaces as it does without the polyfill /// /// A special form is the other. What reads one tells it apart by identity: /// `dataclasses` decides that a bare `ClassVar` annotation is not a field by /// asking whether it *is* `typing.ClassVar`, and a checker reading the emitted - /// python rejects `ClassVar[int]` once `ClassVar` names a variable. + /// python rejects `ClassVar[int]` once `ClassVar` is anything else pub fn eagerly_imported_names(&self) -> Vec { let db = self.db; let env = self.program_environment(); @@ -376,6 +347,13 @@ impl<'db> SemanticModel<'db> { names } + /// basedpython: the module-level `from` imports whose binding is the only thing that ever + /// binds its name in this file, each with every read of it — the imports the lazy-import + /// polyfill can defer by rewriting those reads. see `types::deferred_imports` + pub fn deferrable_from_imports(&self) -> Vec { + crate::types::deferred_imports::deferrable_from_imports(self.db, self.file) + } + /// basedpython: when an attribute access reads a *requirement* off an /// interface-typed receiver, how it dispatches through the witness table. /// @@ -742,6 +720,17 @@ impl<'db> SemanticModel<'db> { conversions } + /// basedpython: the anonymous named tuple class the checker read `tuple` as — + /// the one a type form or a value form declares, or the one a plain tuple + /// literal is built as where one was expected. `None` for any other tuple + pub fn anon_named_tuple_class( + &self, + tuple: &ast::ExprTuple, + ) -> Option> { + let ty = tuple.inferred_type(self)?; + AnonNamedTupleClass::of(self.db, &self.program_environment(), ty) + } + /// could any conversion apply at this call at all? /// /// Binding the call a second time to find out is the expensive part, and @@ -926,6 +915,45 @@ impl<'db> SemanticModel<'db> { .is_some_and(|ty| crate::types::subscript::is_runtime_subscript(self.db, env, ty)) } + /// basedpython: whether subscripting `value` specializes it, so that the checker infers + /// the slice as type expressions — the same answer inference acts on (see + /// `types::subscript::specializes_on_subscript`). `false` for a value the checker could + /// not resolve + pub fn subscript_specializes(&self, value: &Expr) -> bool { + value + .inferred_type(self) + .is_some_and(|ty| crate::types::subscript::specializes_on_subscript(self.db, ty)) + } + + /// basedpython: whether `expr` is `typing.Callable` or `collections.abc.Callable`, whatever + /// name it is reached by — an alias (`from typing import Callable as Fn`), an assignment + /// (`C = typing.Callable`) — and not a class of the program's own that is named `Callable` + pub fn is_callable_form(&self, expr: &Expr) -> bool { + matches!( + expr.inferred_type(self), + Some(Type::SpecialForm( + SpecialFormType::TypingCallable | SpecialFormType::CollectionsAbcCallable + )) + ) + } + + /// basedpython: whether inference reads the type expression `expr` as deferred, as it + /// does an annotation, though python evaluates it where it stands — the slice of a + /// specialization written as a value (`list[F]()`), or a type-expression argument of a + /// typing construct assigned to a name (`TypeVar("T", bound=F)`). a name in one may be + /// bound only further on, and the transpiler writes it as the forward reference python + /// reads lazily + pub fn is_read_deferred(&self, expr: &Expr) -> bool { + let file = self.program_file(); + let index = semantic_index(self.db, file); + let Some(file_scope) = index.try_expression_scope_id(&ast::ExprRef::from(expr)) else { + return false; + }; + infer_complete_scope_types(self.db, file_scope.to_scope_id(self.db, file)) + .type_expression_flags(expr) + .contains(TypeExpressionFlags::READ_DEFERRED) + } + /// basedpython: whether an attribute access resolves through an *implicit /// receiver* — `x.fn` where `fn` names a receiver callable (`int.() -> str`) /// in scope rather than a member of `x`. The transpiler rewrites those to @@ -1046,26 +1074,6 @@ impl<'db> SemanticModel<'db> { .map(|member| member.qualifier(self.db).to_string()) } - /// basedpython: the bracketed type-argument spelling the transpiler - /// injects at a bare constructor call of a generic class (`A(1)` → - /// `"int"`), read from the inferred type of the constructed instance. - /// `None` when the callee is not a generic class literal or no runtime - /// spelling exists — reification of constructors is best-effort, never - /// an error - pub fn reified_constructor_type_arguments(&self, call: &ast::ExprCall) -> Option { - let env = &self.program_environment(); - let callee_ty = call.func.inferred_type(self)?; - let class_literal = callee_ty.as_class_literal()?; - let constructed = call.inferred_type(self)?; - crate::types::reified_infer::constructor_specialization_display( - self.db, - env, - self.file(), - class_literal, - constructed, - ) - } - /// basedpython: the keyword the transpiler passes a trailing lambda block /// with — the name of the callee's last declared parameter. `None` means /// the lambda is appended as a positional argument instead (unknown @@ -1147,29 +1155,6 @@ impl<'db> SemanticModel<'db> { } } - /// basedpython: how the type test `lhs is rhs` resolves, from the value's - /// inferred type and the type its right-hand side names. - /// - /// The right-hand side is a type expression, so `rhs` may name its target - /// however a type expression can: directly (`list[int]`), through an - /// implicit alias (`X = list[int]`), or through a PEP 695 `type` alias. - /// `None` only when inference has no type for one of the operands. - pub fn parametric_is_plan( - &self, - lhs: &ast::Expr, - rhs: &ast::Expr, - ) -> Option { - let env = &self.program_environment(); - Some(crate::types::reified_infer::type_test_plan( - self.db, - env, - self.file(), - lhs.inferred_type(self)?, - rhs.inferred_type(self)?, - Some(rhs), - )) - } - /// basedpython: when `annotation` is a type expression denoting a union of /// specializations of one erased origin (`list[int] | list[str]`), how the /// arms differ. the transpiler uses this to give the parameter a reified @@ -2012,6 +1997,164 @@ impl<'db> SemanticModel<'db> { } } +/// basedpython: a [`SemanticModel`] for code that writes the checked program out — the +/// transpiler, the native backend, and what the checker reads once a file is checked. it can +/// also ask what a name holds when the program runs, which reads the inference of every scope +/// around the name, those after it included, so it infers the file in the order checking does +/// before the first such question. inference builds plain [`SemanticModel`]s, which cannot ask +/// it: asked from inside inference, the scopes it reads would be inferred inside the scope +/// being inferred +pub struct LoweringModel<'db> { + model: SemanticModel<'db>, + in_check_order: OnceLock>, +} + +impl<'db> std::ops::Deref for LoweringModel<'db> { + type Target = SemanticModel<'db>; + + fn deref(&self) -> &Self::Target { + &self.model + } +} + +impl<'db> LoweringModel<'db> { + pub fn new(db: &'db dyn Db, file: ProgramFile<'db>) -> Self { + Self { + model: SemanticModel::new(db, file), + in_check_order: OnceLock::new(), + } + } + + /// infers every scope of the file, each before the scopes inside it, as checking the file + /// does. a reader about to ask for the types of most of the file asks this first: a scope + /// inferred before the scope around it reads that scope's names before that scope has + /// inferred the conditions deciding whether they are bound there, and each of those + /// conditions is then inferred inside the inference of the next + pub fn infer_in_check_order(&self) { + self.inferred_in_check_order(); + } + + /// [`Self::infer_in_check_order`], run once, with the proof it hands out + pub(crate) fn inferred_in_check_order(&self) -> InferredInCheckOrder<'db> { + *self + .in_check_order + .get_or_init(|| infer_file_in_check_order(self.db, self.file, |_, _| {})) + } + + /// basedpython: the specialization step the transpiler splices in after the + /// callee of a bare reified-generic call (`f(1)` → `"[int]"`). `None` when + /// the call is not a bare reified-generic call or no injectable spelling + /// exists — the checker reports the latter as `unspecialized-reified-generic` + pub fn reified_call_specialization(&self, call: &ast::ExprCall) -> Option { + let env = &self.program_environment(); + let db = self.db; + let callee_ty = call.func.inferred_type(self)?; + let function = match callee_ty { + Type::FunctionLiteral(function) => function, + Type::BoundMethod(method) => method.function(db), + _ => return None, + }; + if function.is_classmethod(db) || !function.is_unspecialized_reified(db) { + return None; + } + let mut positional = Vec::with_capacity(call.arguments.args.len()); + for argument in &call.arguments.args { + if argument.is_starred_expr() { + return None; + } + positional.push(argument.inferred_type(self)?); + } + let mut keywords = Vec::with_capacity(call.arguments.keywords.len()); + for keyword in &call.arguments.keywords { + let name = keyword.arg.as_ref()?; + keywords.push((name.as_str(), keyword.value.inferred_type(self)?)); + } + crate::types::reified_infer::injectable_call_specialization( + db, + env, + self.file(), + callee_ty, + function, + positional, + keywords, + Some(RuntimeSite::at(call, self.inferred_in_check_order())), + ) + } + + /// basedpython: whether every name `expr` reads holds, where it is written, the binding the + /// checker resolved when the program runs — so writing `expr` into code that runs there + /// evaluates what the checker saw. a class imported only under `if TYPE_CHECKING:`, bound + /// only on some paths or only further on does not + pub fn reads_names_bound_where_written(&self, expr: &ast::Expr) -> bool { + crate::types::reified_infer::written_unbound_read( + self.db, + &self.program_environment(), + expr, + self.inferred_in_check_order(), + ) + .is_none() + } + + /// basedpython: the runtime check a checked cast to `type_expr` applies where it is + /// written, or `None` when the cast cannot check its value there + pub fn checked_cast_plan(&self, type_expr: &ast::Expr) -> Option { + crate::types::soundness::cast_check_plan( + self.db, + &self.program_environment(), + self.file(), + type_expr.inferred_type(self)?, + Some(RuntimeSite::at(type_expr, self.inferred_in_check_order())), + ) + } + + /// basedpython: the bracketed type-argument spelling the transpiler + /// injects at a bare constructor call of a generic class (`A(1)` → + /// `"int"`), read from the inferred type of the constructed instance. + /// `None` when the callee is not a generic class literal or no runtime + /// spelling exists — reification of constructors is best-effort, never + /// an error + pub fn reified_constructor_type_arguments( + &self, + call: &ast::ExprCall, + ) -> Option { + let env = &self.program_environment(); + let callee_ty = call.func.inferred_type(self)?; + let class_literal = callee_ty.as_class_literal()?; + let constructed = call.inferred_type(self)?; + crate::types::reified_infer::constructor_specialization_display( + self.db, + env, + self.file(), + class_literal, + constructed, + Some(RuntimeSite::at(call, self.inferred_in_check_order())), + ) + } + + /// basedpython: how the type test `lhs is rhs` resolves, from the value's + /// inferred type and the type its right-hand side names. + /// + /// The right-hand side is a type expression, so `rhs` may name its target + /// however a type expression can: directly (`list[int]`), through an + /// implicit alias (`X = list[int]`), or through a PEP 695 `type` alias. + /// `None` only when inference has no type for one of the operands. + pub fn parametric_is_plan( + &self, + lhs: &ast::Expr, + rhs: &ast::Expr, + ) -> Option { + let env = &self.program_environment(); + Some(crate::types::reified_infer::type_test_plan( + self.db, + env, + self.file(), + lhs.inferred_type(self)?, + rhs.inferred_type(self)?, + Some(WrittenTarget::checked(rhs, self.inferred_in_check_order())), + )) + } +} + /// The type and definition of a symbol. #[derive(Clone, Debug)] pub struct MemberDefinition<'db> { diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index e08e009fc1..1e1bd4dc78 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -25,6 +25,7 @@ use ty_module_resolver::{ ImportingFile, KnownModule, Module, ModuleName, file_to_module, resolve_module, }; +pub use self::anon_named_tuple::{AnonNamedTupleClass, AnonNamedTupleId}; pub(crate) use self::callable::UpcastPolicy; use self::class::ClassInstanceFlags; pub use self::cyclic::CycleDetector; @@ -43,9 +44,10 @@ pub use self::diagnostic::{ }; use self::infer::infer_function_default_types; pub(crate) use self::infer::{ - ArgumentContextOrigin, InferredDeclaration, TypeContext, infer_complete_scope_types, - infer_deferred_types, infer_definition_types, infer_expression_type, infer_expression_types, - infer_same_file_expression_type, infer_scope_types, is_discarded_dict_key_assignment, + ArgumentContextOrigin, InferredDeclaration, TypeContext, TypeExpressionFlags, + infer_complete_scope_types, infer_deferred_types, infer_definition_types, + infer_expression_type, infer_expression_types, infer_same_file_expression_type, + infer_scope_types, is_discarded_dict_key_assignment, }; pub(crate) use self::iteration::{ extract_fixed_length_iterable_element_types, report_iteration_over_character, @@ -170,6 +172,7 @@ use ty_python_core::place::ScopedPlaceId; use ty_python_core::scope::ScopeId; use ty_python_core::{ProgramFile, Truthiness, place_table, semantic_index, use_def_map}; +mod anon_named_tuple; mod attribute_write; mod bool; mod bound_super; @@ -178,6 +181,7 @@ mod call; pub mod call_type_forms; mod callable; pub mod character; +pub(crate) mod check_order; mod class; mod class_base; pub mod classinfo_spelling; @@ -192,6 +196,7 @@ pub(crate) mod conversions; mod cyclic; pub(crate) mod dedicated; mod deferred; +pub(crate) mod deferred_imports; mod diagnostic; pub(crate) mod display; pub(crate) mod enums; @@ -213,6 +218,7 @@ mod known_instance; mod lifetimes; pub mod list_members; mod literal; +mod lowering_reads; mod match_pattern; pub(crate) mod match_type; mod member; @@ -234,10 +240,13 @@ pub(crate) mod relation; mod relation_error; pub mod repeated_underscore; mod restricted; +pub(crate) mod runtime_reads; +pub(crate) mod runtime_spelling; mod safe_variance; mod set_theoretic; mod signatures; pub mod soundness; +pub mod soundness_sites; mod special_form; pub(crate) mod state_invalidations; pub(crate) mod state_reads; @@ -258,6 +267,7 @@ mod typevar; pub(crate) mod underscore_names; mod unpacker; mod unsafe_union; +mod unused_imports; mod variance; pub(crate) mod visibility; mod visitor; @@ -290,20 +300,23 @@ pub(crate) fn check_types_with( let index = semantic_index(db, file); let mut diagnostics = TypeCheckDiagnostics::default(); - for scope_id in index.scope_ids() { - // Scopes that may require type context are inferred during the inference of - // their outer scope. - if scope_id.accepts_type_context(db) { - continue; - } - - let result = infer_scope_types(db, scope_id, TypeContext::default()); - + check_order::infer_file_in_check_order(db, file, |_, result| { if let Some(scope_diagnostics) = result.diagnostics() { diagnostics.extend(scope_diagnostics); } + }); + // basedpython: whether a name a lowering writes into code holds there the binding the + // checker resolved is asked once the file is inferred + for checks in runtime_reads::file_runtime_read_checks(db, file) { + diagnostics.extend(&checks.diagnostics); } + // basedpython: an import may be used only through an extension it makes applicable, + // which only the whole file's inference knows + let mut unused_imports = TypeCheckDiagnostics::default(); + unused_imports::report_unused_imports(db, file, &diagnostics, &mut unused_imports); + diagnostics.extend(&unused_imports); + diagnostics.extend_diagnostics( index .semantic_syntax_errors() @@ -3536,16 +3549,7 @@ impl<'db> Type<'db> { .elements(db) .iter() .all(|ty| ty.is_literal_or_union_of_literals(db, env)), - Type::LiteralValue(literal) => match literal.kind() { - LiteralValueTypeKind::String(_) - | LiteralValueTypeKind::Bytes(_) - | LiteralValueTypeKind::Int(_) - | LiteralValueTypeKind::Bool(_) - | LiteralValueTypeKind::Enum(_) - | LiteralValueTypeKind::Float(_) - | LiteralValueTypeKind::Complex(_) => true, - LiteralValueTypeKind::LiteralString | LiteralValueTypeKind::Template(_) => false, - }, + Type::LiteralValue(literal) => literal.kind().holds_one_value(), Type::NominalInstance(_) => { self.is_none(db) || self.is_bool(db) || self.is_enum(db, env) } @@ -3923,20 +3927,28 @@ impl<'db> Type<'db> { self.apply_type_mapping( db, env, - &TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular), + &TypeMapping::Promote( + PromotionMode::On, + PromotionKind::regular( + NumericPromotion::SpecialCase, + FunctionPromotion::ToCallable, + ), + ), TypeContext::default(), ) } - /// As [`Type::promote`], but honouring the numeric model of the file the type came from. + /// As [`Type::promote`], but honouring the model of the file the type came from. /// /// An *inferred* type is widened through the same promotion that applies the numeric /// special case, so a module that opted out of the special case in its annotations would /// otherwise get it back through inference — `self.x = x` with `x: float` would declare /// the attribute `int | float` however strict the parameter was. /// - /// A `.by` file has that model by definition rather than by configuration, so it takes - /// the strict path whatever `strict-float` says. + /// A `.by` file has that numeric model by definition rather than by configuration, so it + /// takes the strict path whatever `strict-float` says. It also keeps a function's own type + /// ([`FunctionPromotion::Keep`]), which is a property of the language rather than of its + /// numbers, so `strict-float` has no say in it. #[must_use] fn promote_in( self, @@ -3944,16 +3956,24 @@ impl<'db> Type<'db> { env: &ProgramEnvironment<'db>, file: ruff_db::files::File, ) -> Type<'db> { - let kind = - if file.source_type(db).is_basedpython() || db.analysis_settings(file).strict_float { - PromotionKind::RegularStrictNumeric - } else { - PromotionKind::Regular - }; + let basedpython = file.source_type(db).is_basedpython(); + let numeric = if basedpython || db.analysis_settings(file).strict_float { + NumericPromotion::Exact + } else { + NumericPromotion::SpecialCase + }; + let functions = if basedpython { + FunctionPromotion::Keep + } else { + FunctionPromotion::ToCallable + }; self.apply_type_mapping( db, env, - &TypeMapping::Promote(PromotionMode::On, kind), + &TypeMapping::Promote( + PromotionMode::On, + PromotionKind::regular(numeric, functions), + ), TypeContext::default(), ) } @@ -10305,7 +10325,10 @@ impl<'db> Type<'db> { // callable form. TypeMapping::Promote( PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularKeepingLiterals, + PromotionKind::Regular { + functions: FunctionPromotion::ToCallable, + .. + }, ) => mapped.promote_impl(db, visitor.env), _ => mapped, } @@ -10332,14 +10355,15 @@ impl<'db> Type<'db> { ), )), - // `RegularStrictNumeric` is deliberately absent: it promotes everything - // else and leaves `float` and `complex` exact Type::NominalInstance(instance) if matches!( type_mapping, TypeMapping::Promote( PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularKeepingLiterals + PromotionKind::Regular { + numeric: NumericPromotion::SpecialCase, + .. + } ) ) => { @@ -10492,10 +10516,7 @@ impl<'db> Type<'db> { // so we don't preserve them here when regular promotion is enabled. if !matches!( type_mapping, - TypeMapping::Promote( - PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularKeepingLiterals - ) + TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular { .. }) ) { for negative in intersection.negative(db) { builder.add_negative_in_place(negative.apply_type_mapping_impl( @@ -10567,7 +10588,13 @@ impl<'db> Type<'db> { if restricted.modifier(db) == TypeModifier::Final && matches!( type_mapping, - TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular) + TypeMapping::Promote( + PromotionMode::On, + PromotionKind::Regular { + literals: LiteralPromotion::Widen, + .. + } + ) ) { return inner; @@ -10699,11 +10726,18 @@ impl<'db> Type<'db> { ) => self, TypeMapping::Promote( PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularStrictNumeric, + PromotionKind::Regular { + literals: LiteralPromotion::Widen, + .. + }, ) => self.promote_impl(db, env), - TypeMapping::Promote(PromotionMode::On, PromotionKind::RegularKeepingLiterals) => { - self.promote_impl_keeping_literals(db, env) - } + TypeMapping::Promote( + PromotionMode::On, + PromotionKind::Regular { + literals: LiteralPromotion::Keep, + .. + }, + ) => self.promote_impl_keeping_literals(db, env), }, Type::Dynamic(_) => match type_mapping { @@ -11859,19 +11893,13 @@ impl PromotionMode { #[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] pub enum PromotionKind { - /// Default promotion behaviour: recurse into nested types - Regular, - /// Like [`PromotionKind::Regular`], but literal values are left as they are. - /// - /// This is what a covariant type argument gets: widening it is sound, but nothing reads a - /// different type back out of that position, so the literal it was inferred from stands. - RegularKeepingLiterals, - /// Like [`PromotionKind::Regular`], but `float` and `complex` are left exact. - /// - /// Widening a literal is one thing; widening `float` to `int | float` is the typing spec's - /// numeric special case, and a module that set `strict-float` has opted out of it. The two - /// travel together in [`PromotionKind::Regular`] because they are usually wanted together. - RegularStrictNumeric, + /// Default promotion behaviour: recurse into nested types. what it does with a literal + /// and with `float` / `complex` are two independent choices + Regular { + literals: LiteralPromotion, + numeric: NumericPromotion, + functions: FunctionPromotion, + }, /// Promote class literals recursively without promoting other literal types. ClassLiteralsOnly, /// Singleton-only promotion recursively descends through nominal instances @@ -11879,6 +11907,54 @@ pub enum PromotionKind { SingletonsOnly, } +impl PromotionKind { + /// regular promotion that widens literals, under the given numeric model and treatment of + /// functions + const fn regular(numeric: NumericPromotion, functions: FunctionPromotion) -> Self { + Self::Regular { + literals: LiteralPromotion::Widen, + numeric, + functions, + } + } +} + +/// what regular promotion does with a literal value +#[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] +pub enum LiteralPromotion { + /// widen it to the class it is an instance of (`Literal[1]` → `int`), and drop the exactness + /// a constructor call infers (`final A` → `A`), which is the same kind of precision + Widen, + /// leave it as it is. this is what a covariant type argument gets: widening it is sound, + /// but nothing reads a different type back out of that position, so the literal it was + /// inferred from stands + Keep, +} + +/// what regular promotion does with `float` and `complex` +#[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] +pub enum NumericPromotion { + /// the typing spec's numeric special case: `float` widens to `int | float` and `complex` + /// to `int | float | complex` + SpecialCase, + /// `float` and `complex` are left exact. a `.by` module has this model by definition, and + /// a module that set `strict-float` has opted into it + Exact, +} + +/// what regular promotion does with a function +#[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] +pub enum FunctionPromotion { + /// widen it to its callable type (`def f() -> int` → `() -> int`) + ToCallable, + /// leave it as it is. a function declared in a `.by` module can carry what no callable + /// type expresses — reified type parameters a call specializes by subscript, `context` + /// parameters a call fills, `local` / `once` parameters, a `raises` clause — and a call + /// through the stored value relies on them, so a `.by` module keeps the function's own + /// type + Keep, +} + /// Returns the [`ClassLiteral`] that "owns" a `Self` typevar (i.e., the class from its upper bound). fn self_typevar_owner_class_literal<'db>( db: &'db dyn Db, diff --git a/crates/ty_python_semantic/src/types/anon_named_tuple.rs b/crates/ty_python_semantic/src/types/anon_named_tuple.rs new file mode 100644 index 0000000000..941d26669e --- /dev/null +++ b/crates/ty_python_semantic/src/types/anon_named_tuple.rs @@ -0,0 +1,120 @@ +//! basedpython anonymous named tuples, `(name: T, ...)` +//! +//! each one is a synthesized `NamedTuple` class. a plain tuple display standing +//! where one is expected — `take((1, 2))` with `def take(p: (a: int, b: int))` — +//! is read as an instance of that class, and the transpiler constructs it there. +//! both sides read the decision off the display's inferred type, so the checker +//! never accepts field access on a value the emitted python leaves a plain tuple + +use ruff_python_ast::name::Name; +use salsa::plumbing::AsId; + +use crate::Db; +use crate::types::class::{ClassLiteral, DynamicNamedTupleLiteral, NamedTupleSpec}; +use crate::types::{ProgramEnvironment, Type, UnionType}; + +/// the class and field specification of an anonymous named tuple `ty` is an instance +/// of, or (for a type form in a value position) the class itself +pub(crate) fn anon_named_tuple_class<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, +) -> Option<(DynamicNamedTupleLiteral<'db>, NamedTupleSpec<'db>)> { + let literal = match ty { + Type::NominalInstance(instance) => instance.class(db, env).class_literal(db), + Type::ClassLiteral(literal) => literal, + _ => return None, + }; + let ClassLiteral::DynamicNamedTuple(class) = literal else { + return None; + }; + let spec = class.anchor(db).anonymous_spec()?; + Some((class, spec)) +} + +/// every anonymous named tuple instance `expected` allows, in the order its union +/// lists them. a type alias is read through, so this follows what an annotation +/// means rather than how it is spelled +pub(crate) fn expected_anon_named_tuples<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + expected: Type<'db>, +) -> Vec<(Type<'db>, NamedTupleSpec<'db>)> { + fn collect<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, + found: &mut Vec<(Type<'db>, NamedTupleSpec<'db>)>, + ) { + let ty = ty.resolve_type_alias(db); + if let Type::Union(union) = ty { + for member in union.elements(db) { + collect(db, env, *member, found); + } + return; + } + if let Type::NominalInstance(_) = ty + && let Some((_, spec)) = anon_named_tuple_class(db, env, ty) + && !found.iter().any(|(seen, _)| *seen == ty) + { + found.push((ty, spec)); + } + } + let mut found = Vec::new(); + collect(db, env, expected, &mut found); + found +} + +/// `expected` without its anonymous named tuple arms, or `None` when nothing is left. +/// +/// a plain tuple literal is an instance of an anonymous named tuple only where it is built +/// as one, so one that is not cannot use such an arm as the shape it is inferred against — +/// a `(a: list[int])` arm would make `([1, "s"],)` fail a `tuple[list[object]]` arm beside it +pub(crate) fn without_anon_named_tuples<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + expected: Type<'db>, +) -> Option> { + let resolved = expected.resolve_type_alias(db); + let arms: Vec> = match resolved { + Type::Union(union) => union.elements(db).to_vec(), + other => vec![other], + }; + let kept: Vec> = arms + .into_iter() + .filter(|arm| { + !(matches!(arm, Type::NominalInstance(_)) + && anon_named_tuple_class(db, env, *arm).is_some()) + }) + .collect(); + (!kept.is_empty()).then(|| UnionType::from_elements(db, env, kept)) +} + +/// one anonymous named tuple class, as the checker identifies it. two type forms +/// with the same identity are the same type, whatever their spelling +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct AnonNamedTupleId(salsa::Id); + +/// an anonymous named tuple class the checker read an expression as +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AnonNamedTupleClass<'db> { + pub id: AnonNamedTupleId, + /// each field's name and declared type, in order + pub fields: Vec<(Name, Type<'db>)>, +} + +impl<'db> AnonNamedTupleClass<'db> { + /// the anonymous named tuple `ty` is an instance of, or the class itself for a type + /// form standing in a value position + pub fn of(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>) -> Option { + let (class, spec) = anon_named_tuple_class(db, env, ty)?; + Some(Self { + id: AnonNamedTupleId(class.as_id()), + fields: spec + .fields(db) + .iter() + .map(|field| (field.name.clone(), field.ty)) + .collect(), + }) + } +} diff --git a/crates/ty_python_semantic/src/types/attribute_write.rs b/crates/ty_python_semantic/src/types/attribute_write.rs index 4ad942b4f9..3adb1ff866 100644 --- a/crates/ty_python_semantic/src/types/attribute_write.rs +++ b/crates/ty_python_semantic/src/types/attribute_write.rs @@ -147,6 +147,113 @@ pub(super) enum ExplicitAttributeWriteRequirement<'db> { }, } +impl<'db> AttributeWriteRequirement<'db> { + /// the one type the assigned value is checked against, where it is checked against one: a + /// declared type, or the value a descriptor's setter takes + pub(super) fn write_type( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ) -> Option> { + match self { + Self::Instance { object_ty, member } => member.write_type(db, env, *object_ty), + Self::Class { object_ty, member } => match member { + ClassAttributeWriteMember::Explicit { member, .. } => { + member.write_type(db, env, *object_ty) + } + ClassAttributeWriteMember::ClassAttribute( + FallbackAttributeWriteRequirement::AssignableTo { ty, .. }, + ) => Some(*ty), + ClassAttributeWriteMember::ClassAttribute( + FallbackAttributeWriteRequirement::PossiblyMissing, + ) + | ClassAttributeWriteMember::Unresolved { .. } => None, + }, + Self::Module(ty) => *ty, + Self::ProtocolMember { + write: Some(ProtocolMemberWriteRequirement::AssignableTo(ty)), + .. + } => Some(*ty), + Self::ProtocolMember { + write: + Some(ProtocolMemberWriteRequirement::Descriptor { + domain: Some(domain), + .. + }), + .. + } => Some(*domain), + Self::ProtocolMember { .. } + | Self::All { .. } + | Self::Any { .. } + | Self::Unconstrained + | Self::CannotAssign => None, + } + } +} + +impl<'db> InstanceAttributeWriteMember<'db> { + /// the type the assigned value is checked against on `object_ty`, where it is checked + /// against one + pub(super) fn write_type( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + object_ty: Type<'db>, + ) -> Option> { + match self { + Self::Explicit { member, .. } => member.write_type(db, env, object_ty), + Self::Instance(FallbackAttributeWriteRequirement::AssignableTo { ty, .. }) => Some(*ty), + Self::Instance(FallbackAttributeWriteRequirement::PossiblyMissing) + | Self::ClassVar + | Self::SetAttr => None, + } + } +} + +impl<'db> ExplicitAttributeWriteRequirement<'db> { + /// the type the assigned value is checked against on `object_ty`, where it is checked + /// against one: the declared type, or the value the descriptor's setter takes + pub(super) fn write_type( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + object_ty: Type<'db>, + ) -> Option> { + match self { + Self::AssignableTo { ty, .. } => Some(*ty), + Self::Descriptor { descriptor_ty, .. } => { + // a property's own `__set__` takes any value; what its setter takes is what + // the write is checked against + if let Some(property) = descriptor_ty.as_property_instance() { + return property + .setter(db) + .and_then(|setter| setter_value_type(db, setter)); + } + match descriptor_setter_domain(db, env, *descriptor_ty, object_ty) { + DescriptorSetterDomain::Known(domain) => Some(domain), + DescriptorSetterDomain::Missing | DescriptorSetterDomain::Deferred => None, + } + } + } + } +} + +/// the value a property setter takes: the type of its second parameter, where it has one +/// signature +fn setter_value_type<'db>(db: &'db dyn Db, setter: Type<'db>) -> Option> { + let Type::FunctionLiteral(function) = setter else { + return None; + }; + let [overload] = function.signature(db).overloads.as_slice() else { + return None; + }; + overload + .parameters() + .iter() + .nth(1) + .map(Parameter::annotated_type) +} + impl ExplicitAttributeWriteRequirement<'_> { pub(super) fn qualifiers(&self) -> TypeQualifiers { match self { diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index 536c456b17..00a03de383 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -10741,7 +10741,7 @@ impl<'db> BindingError<'db> { Self::get_argument_node(node, *argument_index).map(ArgOrKeyword::value), ) { - crate::types::conversions::report_ambiguous_conversion(context, range, &repair); + crate::types::conversions::accept_conversion(context, range, &repair); return; } @@ -10929,6 +10929,24 @@ impl<'db> BindingError<'db> { } add_invariant_generic_hints(db, env, &mut diag, *expected_ty, *provided_ty); + + // a `.by` file builds a plain tuple standing where an anonymous named tuple + // is expected as one, so a caller written in python is where that stops + if !context.file().source_type(db).is_basedpython() + && provided_ty.tuple_instance_spec(db, env).is_some() + && !crate::types::anon_named_tuple::expected_anon_named_tuples( + db, + env, + *expected_ty, + ) + .is_empty() + { + diag.info( + "A `.py` file does not build an anonymous named tuple from a plain \ + tuple: construct it through a name its module binds the type to, \ + as in `P(1, 2)` where the module declares `P = (a: int, b: int)`", + ); + } } Self::InvalidKeyType { diff --git a/crates/ty_python_semantic/src/types/check_order.rs b/crates/ty_python_semantic/src/types/check_order.rs new file mode 100644 index 0000000000..11455959f0 --- /dev/null +++ b/crates/ty_python_semantic/src/types/check_order.rs @@ -0,0 +1,70 @@ +//! basedpython: inferring a file's scopes in the order checking it does, each before the scopes +//! inside it +//! +//! a scope inferred before the scope around it reads that scope's names before that scope has +//! inferred the conditions deciding whether they are bound there, and each of those conditions +//! is then inferred inside the inference of the next. inferred in order, every condition a +//! question reads is already inferred when it is asked + +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::{ProgramFile, semantic_index}; + +use crate::Db; +use crate::types::TypeContext; +use crate::types::infer::{ScopeInference, infer_complete_scope_types, infer_scope_types}; + +/// that the scopes a question about the running program reads are inferred, each before the +/// scopes inside it: the bindings a name has when the program runs read every binding the +/// module makes and every condition deciding whether each is made, those after the read +/// included ([`crate::place::runtime_binding`]). +/// +/// only [`infer_file_in_check_order`] and [`infer_scope_in_check_order`] make one, so +/// inference, which those run, cannot ask such a question of the scope it is inferring. one +/// is for one file, and a question asked with it is asked of that file +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct InferredInCheckOrder<'db> { + /// the file whose scopes are inferred, and so the only one a question may be asked of + file: ProgramFile<'db>, +} + +impl<'db> InferredInCheckOrder<'db> { + pub(crate) fn file(self) -> ProgramFile<'db> { + self.file + } +} + +/// infers every scope of `file` in the order checking it does, handing each scope's inference to +/// `each` +pub(crate) fn infer_file_in_check_order<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + mut each: impl FnMut(ScopeId<'db>, &'db ScopeInference<'db>), +) -> InferredInCheckOrder<'db> { + for scope in semantic_index(db, file).scope_ids() { + // a scope that takes type context is inferred inside the scope around it + if !scope.accepts_type_context(db) { + each(scope, infer_scope_types(db, scope, TypeContext::default())); + } + } + InferredInCheckOrder { file } +} + +/// infers every scope around `scope`, outermost first, and then `scope` +pub(crate) fn infer_scope_in_check_order<'db>( + db: &'db dyn Db, + scope: ScopeId<'db>, +) -> (&'db ScopeInference<'db>, InferredInCheckOrder<'db>) { + let file = scope.program_file(db); + let enclosing: Vec = semantic_index(db, file) + .ancestor_scopes(scope.file_scope_id(db)) + .skip(1) + .map(|(ancestor, _)| ancestor) + .collect(); + for ancestor in enclosing.into_iter().rev() { + infer_complete_scope_types(db, ancestor.to_scope_id(db, file)); + } + ( + infer_complete_scope_types(db, scope), + InferredInCheckOrder { file }, + ) +} diff --git a/crates/ty_python_semantic/src/types/class/named_tuple.rs b/crates/ty_python_semantic/src/types/class/named_tuple.rs index 6045953c08..87c56988d1 100644 --- a/crates/ty_python_semantic/src/types/class/named_tuple.rs +++ b/crates/ty_python_semantic/src/types/class/named_tuple.rs @@ -193,7 +193,8 @@ impl<'db> DynamicNamedTupleLiteral<'db> { match self.anchor(db) { DynamicNamedTupleAnchor::CollectionsDefinition { definition, .. } | DynamicNamedTupleAnchor::TypingDefinition(definition) => Some(*definition), - DynamicNamedTupleAnchor::ScopeOffset { .. } => None, + DynamicNamedTupleAnchor::ScopeOffset { .. } + | DynamicNamedTupleAnchor::Anonymous { .. } => None, } } @@ -202,7 +203,8 @@ impl<'db> DynamicNamedTupleLiteral<'db> { match self.anchor(db) { DynamicNamedTupleAnchor::CollectionsDefinition { definition, .. } | DynamicNamedTupleAnchor::TypingDefinition(definition) => definition.scope(db), - DynamicNamedTupleAnchor::ScopeOffset { scope, .. } => *scope, + DynamicNamedTupleAnchor::ScopeOffset { scope, .. } + | DynamicNamedTupleAnchor::Anonymous { scope, .. } => *scope, } } @@ -221,6 +223,10 @@ impl<'db> DynamicNamedTupleLiteral<'db> { DynamicNamedTupleAnchor::ScopeOffset { offset, .. } => { DynamicClassHeaderAnchor::ScopeOffset(*offset) } + // every form of this shape makes the class, so no one of them is its header + DynamicNamedTupleAnchor::Anonymous { .. } => { + DynamicClassHeaderAnchor::ScopeOffset(DynamicClassScopeOffset::Node(0)) + } }; dynamic_class_header_range(db, self.scope(db), anchor) } @@ -463,7 +469,8 @@ impl<'db> DynamicNamedTupleLiteral<'db> { match self.anchor(db) { DynamicNamedTupleAnchor::CollectionsDefinition { spec, .. } - | DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => *spec, + | DynamicNamedTupleAnchor::ScopeOffset { spec, .. } + | DynamicNamedTupleAnchor::Anonymous { spec, .. } => *spec, DynamicNamedTupleAnchor::TypingDefinition(definition) => deferred_spec(db, *definition), } } @@ -537,6 +544,15 @@ pub enum DynamicNamedTupleAnchor<'db> { offset: DynamicClassScopeOffset, spec: NamedTupleSpec<'db>, }, + + /// basedpython: the class an anonymous named tuple form `(name: T, ...)` or + /// `(name=v, ...)` makes, which no call the program makes can produce. its identity + /// is its shape: every form in a module with the same spec is the same class, anchored + /// at the module's scope + Anonymous { + scope: ScopeId<'db>, + spec: NamedTupleSpec<'db>, + }, } impl<'db> DynamicNamedTupleAnchor<'db> { @@ -562,6 +578,21 @@ impl<'db> DynamicNamedTupleAnchor<'db> { offset: *offset, spec: spec.recursive_type_normalized_impl(db, env, div, nested)?, }), + Self::Anonymous { scope, spec } => Some(Self::Anonymous { + scope: *scope, + spec: spec.recursive_type_normalized_impl(db, env, div, nested)?, + }), + } + } + + /// the field specification an anonymous named tuple form made this class from, or + /// `None` for a named tuple the program makes itself + pub(crate) fn anonymous_spec(&self) -> Option> { + match self { + Self::Anonymous { spec, .. } => Some(*spec), + Self::CollectionsDefinition { .. } + | Self::TypingDefinition(_) + | Self::ScopeOffset { .. } => None, } } } diff --git a/crates/ty_python_semantic/src/types/conformance.rs b/crates/ty_python_semantic/src/types/conformance.rs index f4be840436..aa3babd84d 100644 --- a/crates/ty_python_semantic/src/types/conformance.rs +++ b/crates/ty_python_semantic/src/types/conformance.rs @@ -155,10 +155,10 @@ fn imported_module_names(db: &dyn Db, file: File) -> Vec { .collect() } -/// the protocol a conformance extension in `file` makes `receiver_class` -/// conform to, when that protocol's class literal is `protocol`. +/// the conformance extension visible in `file` that makes `receiver_class` conform to +/// `protocol`, with the protocol it declares. /// -/// The answer is the protocol *as the conformance declared it*, so a specialized +/// The protocol is the one *as the conformance declared it*, so a specialized /// conformance (`extension MyBox(Container[int]):`) hands back `Container[int]` /// — which is what a protocol extension's members must be specialized at when /// they are reached through the conforming type @@ -168,20 +168,21 @@ pub(crate) fn conformance_for<'db>( file: File, receiver_class: ClassType<'db>, protocol: ClassLiteral<'db>, -) -> Option> { +) -> Option<(StaticClassLiteral<'db>, ClassType<'db>)> { for &(extension, declared) in visible_conformances(db, file) { if declared.class_literal(db) != protocol { continue; } if extension_applies(db, env, extension, receiver_class).is_some() { - return Some(declared); + return Some((extension, declared)); } } None } /// would a visible conformance make `source` acceptable where `target` is asked -/// for? the protocol it conforms to, for the diagnostic that reports two routes. +/// for? the conformance, and the protocol it conforms to for the diagnostic that +/// reports two routes. /// /// Unlike a conversion this materializes nothing: the value already answers /// every requirement, through the witness table its conformance registered at @@ -194,7 +195,7 @@ pub(crate) fn repair_with_conformance<'db>( file: File, source: Type<'db>, target: Type<'db>, -) -> Option> { +) -> Option<(StaticClassLiteral<'db>, ClassType<'db>)> { if !file.source_type(db).is_basedpython() { return None; } @@ -215,7 +216,7 @@ pub(crate) fn repair_with_conformance<'db>( continue; } if Type::instance(db, env, protocol).is_assignable_to(db, env, target) { - return Some(protocol); + return Some((extension, protocol)); } } None @@ -452,8 +453,14 @@ pub(crate) fn validate_conformance_declaration<'db>( } // two conformances of one pair would register two witness tables against - // the same key, and which one survives would depend on import order - if let Some(other) = conflicting_conformance(db, extension, target, *interface) + // the same key, and which one survives would depend on import order. the + // import bringing the other one into scope is what puts the pair in that + // state, so it is relied on until one of the two is dropped + let conflicting = conflicting_conformance(db, extension, target, *interface); + if let Some(other) = conflicting { + context.record_extension_use(other); + } + if let Some(other) = conflicting && let Some(builder) = context.report_lint(&INVALID_CONFORMANCE, node.range()) { let mut diagnostic = builder.into_diagnostic(format_args!( @@ -587,6 +594,23 @@ pub(crate) fn validate_conformance_declaration<'db>( } } + // a default the protocol's own extension supplies goes into the witness + // table this conformance registers, so the import that makes that extension + // applicable here is relied on + for requirement in interface_requirements(db, *interface) { + let name = requirement.as_str(); + if own_member(db, extension, name).is_none() + && let Some(default) = protocol_extension_supplying( + db, + extension.file(db), + interface.class_literal(db), + name, + ) + { + context.record_extension_use(default); + } + } + // every requirement has to be answered by something: the block itself, a // default on the protocol's own extension, or a member the type already // has. an unanswered one is an `AttributeError` the moment anything diff --git a/crates/ty_python_semantic/src/types/context.rs b/crates/ty_python_semantic/src/types/context.rs index bbe77164ad..efc73d3065 100644 --- a/crates/ty_python_semantic/src/types/context.rs +++ b/crates/ty_python_semantic/src/types/context.rs @@ -17,9 +17,12 @@ use super::{Type, TypeCheckDiagnostics, infer_definition_types}; use crate::diagnostic::DiagnosticGuard; use crate::lint::LintSource; use crate::reachability::is_range_reachable; +use crate::types::class::StaticClassLiteral; use crate::types::diagnostic::{INVALID_TYPE_FORM, UNBOUND_TYPE_VARIABLE}; use crate::types::function::FunctionDecorators; use crate::types::infer::InferenceFlags; +use crate::types::runtime_reads::{RuntimeReadCheck, RuntimeReadSite}; +use crate::types::runtime_spelling::RuntimeSpelling; use crate::{ Db, Program, lint::{LintId, LintMetadata}, @@ -27,8 +30,9 @@ use crate::{ }; use ty_module_resolver::ResolverEnvironment; use ty_python_core::definition::Definition; -use ty_python_core::scope::ScopeId; -use ty_python_core::{ProgramFile, semantic_index}; +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::symbol::ScopedSymbolId; +use ty_python_core::{ProgramFile, global_scope, place_table, semantic_index}; /// The lazily resolved program used by a semantic operation. #[derive(Clone)] @@ -235,7 +239,58 @@ impl<'db, 'ast> InferContext<'db, 'ast> { } pub(super) fn has_diagnostics(&self) -> bool { - !self.diagnostics.borrow().is_empty() + self.diagnostics.borrow().has_reports() + } + + /// basedpython: records that `extension` supplied something this inference relied on, so + /// the import that makes it applicable is a used one. + /// + /// Recorded even where diagnostics are suppressed: a speculative inference whose result is + /// kept relied on the extension just the same, and one whose result is dropped takes its + /// record with it + pub(crate) fn record_extension_use(&self, extension: StaticClassLiteral<'db>) { + self.record_extension_module_use(extension.file(self.db)); + } + + /// basedpython: records that inference read every binding `symbol` has in `scope`, at a + /// site the semantic index records no read for + pub(crate) fn record_scope_read(&self, scope: FileScopeId, symbol: ScopedSymbolId) { + self.diagnostics.borrow_mut().mark_scope_read(scope, symbol); + } + + /// basedpython: [`Self::record_scope_read`], for the symbol named `name` + pub(crate) fn record_scope_read_of_name(&self, scope: ScopeId<'db>, name: &str) { + let db = self.db; + if let Some(symbol) = place_table(db, scope).symbol_id(name) { + self.record_scope_read(scope.file_scope_id(db), symbol); + } + } + + /// basedpython: records the bindings of the module's global scope a spelling the + /// transpiled program writes reads. the checker spells it while it checks, exactly as + /// the lowering will, and the source spells none of those reads + pub(crate) fn record_spelling(&self, spelling: &RuntimeSpelling) { + let global = global_scope(self.db, self.program_file); + for name in spelling.global_reads() { + self.record_scope_read_of_name(global, name); + } + } + + /// basedpython: records a site whose module-level names are checked once the file is + /// inferred, reported there only if this inference may report diagnostics + pub(crate) fn record_runtime_read_check(&self, site: RuntimeReadSite) { + let reports = !self.diagnostics_suppressed && !self.is_in_no_type_check(); + self.diagnostics + .borrow_mut() + .record_runtime_read_check(RuntimeReadCheck { site, reports }); + } + + /// basedpython: [`Self::record_extension_use`], for a use known only by the module that + /// declares the extension + pub(crate) fn record_extension_module_use(&self, module: File) { + self.diagnostics + .borrow_mut() + .mark_extension_module_used(module); } /// Prevents diagnostic construction for this inference context. diff --git a/crates/ty_python_semantic/src/types/context_sensitive.rs b/crates/ty_python_semantic/src/types/context_sensitive.rs index c462f7feda..589eef4df7 100644 --- a/crates/ty_python_semantic/src/types/context_sensitive.rs +++ b/crates/ty_python_semantic/src/types/context_sensitive.rs @@ -28,7 +28,8 @@ use std::ops::ControlFlow; use ruff_db::files::File; use ruff_python_ast::name::Name; use ty_python_core::predicate::CaseNamePredicateKind; -use ty_python_core::scope::ScopeId; +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::symbol::ScopedSymbolId; use ty_python_core::{place_table, semantic_index}; use crate::Db; @@ -52,6 +53,10 @@ pub(crate) struct ContextSensitiveMember<'db> { enum_class: ClassLiteral<'db>, /// the type of `.` pub(crate) ty: Type<'db>, + /// the binding the qualifier is read through once the name is lowered to + /// `.`: the symbol, and the scope that binds it. set once the member is + /// known to be nameable + qualifier_binding: Option<(FileScopeId, ScopedSymbolId)>, } impl<'db> ContextSensitiveMember<'db> { @@ -65,6 +70,20 @@ impl<'db> ContextSensitiveMember<'db> { pub(crate) fn enum_class(self) -> ClassLiteral<'db> { self.enum_class } + + /// the binding the lowered `.` reads the qualifier through. the checker + /// records it, because the source spells no read of it + pub(crate) fn qualifier_binding(self) -> Option<(FileScopeId, ScopedSymbolId)> { + self.qualifier_binding + } + + /// this member, once the enum is known to be nameable here through `binding` + fn named_through(self, binding: (FileScopeId, ScopedSymbolId)) -> Self { + Self { + qualifier_binding: Some(binding), + ..self + } + } } /// what the expected type has to say about `name` before the resolution rules @@ -132,7 +151,8 @@ pub(crate) fn resolve_in_context<'db>( let Search::Found(member) = search(db, env, target, name) else { return None; }; - is_nameable(db, file, scope, member.enum_class).then_some(member) + nameable_through(db, file, scope, member.enum_class) + .map(|binding| member.named_through(binding)) } /// the member a bare `case A:` names, or `None` when it is the capture python @@ -157,7 +177,8 @@ pub(crate) fn resolve_case_name<'db>( let Search::Found(member) = search(db, env, subject_ty, name) else { return None; }; - is_nameable(db, file, scope, member.enum_class).then_some(member) + nameable_through(db, file, scope, member.enum_class) + .map(|binding| member.named_through(binding)) } /// the type a bare `case A:` matches, or `None` when the name is an ordinary @@ -200,7 +221,7 @@ pub(crate) fn explain_case_name_miss<'db>( { Some(Miss::Shadowed(member.enum_class)) } - Search::Found(member) if !is_nameable(db, file, scope, member.enum_class) => { + Search::Found(member) if nameable_through(db, file, scope, member.enum_class).is_none() => { Some(Miss::Unnameable(member.enum_class)) } Search::Found(_) | Search::Nothing => None, @@ -221,7 +242,7 @@ pub(crate) fn explain_miss<'db>( Search::Found(member) if claimed_by_lexical_scope(db, env, file, scope, name) => { Some(Miss::Shadowed(member.enum_class)) } - Search::Found(member) if !is_nameable(db, file, scope, member.enum_class) => { + Search::Found(member) if nameable_through(db, file, scope, member.enum_class).is_none() => { Some(Miss::Unnameable(member.enum_class)) } Search::Found(_) | Search::Nothing => None, @@ -272,6 +293,7 @@ fn member_of<'db>( return Some(ContextSensitiveMember { enum_class: literal, ty: Type::enum_literal(EnumLiteralType::new(db, enum_class, member.clone())), + qualifier_binding: None, }); } // a based enum's payload variant, whose class is declared in the enum body @@ -283,6 +305,7 @@ fn member_of<'db>( return Some(ContextSensitiveMember { enum_class: literal, ty, + qualifier_binding: None, }); } } @@ -296,28 +319,31 @@ fn is_variant_class<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { .is_some_and(|class| class.is_enum_variant(db)) } -/// whether the enum is reachable from `scope` under its own name. the transpiler -/// qualifies the resolved name with it (`Red` → `Color.Red`), so a spelling that -/// is not in scope has no lowering — and must therefore not resolve here either -fn is_nameable<'db>( +/// the binding the enum is reachable through from `scope` under its own name: the +/// symbol, and the scope that binds it. the transpiler qualifies the resolved name with +/// it (`Red` → `Color.Red`), so a spelling that is not in scope has no lowering — and +/// must therefore not resolve here either +fn nameable_through<'db>( db: &'db dyn Db, file: File, scope: ScopeId<'db>, enum_class: ClassLiteral<'db>, -) -> bool { +) -> Option<(FileScopeId, ScopedSymbolId)> { let name = enum_class.name(db); let index = semantic_index(db, db.program_file(file)); for (ancestor_id, _) in index.visible_ancestor_scopes(scope.file_scope_id(db)) { let ancestor_scope = ancestor_id.to_scope_id(db, db.program_file(file)); - let Some(place) = place_table(db, ancestor_scope).symbol_by_name(name) else { + let table = place_table(db, ancestor_scope); + let Some(symbol_id) = table.symbol_id(name) else { continue; }; + let place = table.symbol(symbol_id); if !(place.is_bound() || place.is_declared()) { continue; } // the first scope that gives the name a value decides it, exactly as a // name load would - return symbol( + let names_enum = symbol( db, ancestor_scope, name, @@ -327,8 +353,9 @@ fn is_nameable<'db>( .ignore_possibly_undefined() .and_then(Type::as_class_literal) == Some(enum_class); + return names_enum.then_some((ancestor_id, symbol_id)); } - false + None } /// shared checker/transpiler contract: the enum a name that resolved to `ty` @@ -381,7 +408,7 @@ pub(crate) fn enum_class_for_unbound_name<'db>( if !declares_member(db, env, enum_class, name) { return None; } - is_nameable(db, file, scope, enum_class).then_some(enum_class) + nameable_through(db, file, scope, enum_class).map(|_| enum_class) } /// the enum an already-resolved member type belongs to diff --git a/crates/ty_python_semantic/src/types/conversions.rs b/crates/ty_python_semantic/src/types/conversions.rs index 6bc670d61c..02df942a60 100644 --- a/crates/ty_python_semantic/src/types/conversions.rs +++ b/crates/ty_python_semantic/src/types/conversions.rs @@ -79,7 +79,10 @@ pub(crate) enum Route<'db> { /// a visible conformance extension makes the source's class conform to the /// protocol the target asks for. nothing is emitted: the value is already /// the object the protocol dispatches on - Conformance(ClassType<'db>), + Conformance { + extension: StaticClassLiteral<'db>, + protocol: ClassType<'db>, + }, /// `T.__from__(value)`, where `T` is the target class From(ClassType<'db>, DunderSource<'db>), /// `T.__of__(value)`, where `T` is the target class @@ -94,6 +97,19 @@ pub(crate) enum Route<'db> { } impl<'db> Route<'db> { + /// the extension the route is supplied by, when one is + fn extension(self) -> Option> { + match self { + Route::Conformance { extension, .. } + | Route::From(_, DunderSource::Extension(extension)) + | Route::Of(_, DunderSource::Extension(extension)) => Some(extension), + Route::From(_, DunderSource::Declared) + | Route::Of(_, DunderSource::Declared) + | Route::Into(_) + | Route::DiscardReturn => None, + } + } + /// how the route reads in a diagnostic fn describe(self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> String { let dunder = |class: ClassType<'db>, source: DunderSource<'db>, dunder: &str| match source { @@ -107,7 +123,9 @@ impl<'db> Route<'db> { ), }; match self { - Route::Conformance(protocol) => format!("conformance to `{}`", protocol.name(db)), + Route::Conformance { protocol, .. } => { + format!("conformance to `{}`", protocol.name(db)) + } Route::From(class, source) => dunder(class, source, FROM), Route::Of(class, source) => dunder(class, source, OF), Route::Into(source) => format!("{}.{INTO}", source.display(db, env)), @@ -156,10 +174,13 @@ pub(crate) fn repair_conversion<'db>( let source = source.erase_restriction(db); let mut routes: Vec> = Vec::new(); - if let Some(protocol) = + if let Some((extension, protocol)) = super::conformance::repair_with_conformance(db, env, file, source, target) { - routes.push(Route::Conformance(protocol)); + routes.push(Route::Conformance { + extension, + protocol, + }); } dunder_routes(db, env, file, source, target, value, &mut routes); if discards_return(db, env, source, target) { @@ -897,14 +918,15 @@ pub(crate) fn call_parameter_types<'db>( bindings.plain_callee_parameter_types(arguments.len()) } -/// report a conversion site served by more than one route. the site is otherwise -/// accepted — any of the conversions would work — but which one runs must not -/// depend on ordering -pub(crate) fn report_ambiguous_conversion<'db>( +/// accept a conversion site's repair: record the extensions it relies on, and report +/// a site served by more than one route. the site is otherwise accepted — any of the +/// conversions would work — but which one runs must not depend on ordering +pub(crate) fn accept_conversion<'db>( context: &InferContext<'db, '_>, node: impl Ranged, repair: &ConversionRepair<'db>, ) { + record_conversion_uses(context, repair); let env = context.program_environment(); let db = context.db(); if repair.ambiguous_with.is_empty() { @@ -922,6 +944,21 @@ pub(crate) fn report_ambiguous_conversion<'db>( diagnostic.help("Remove all but one of them, or write the conversion you want explicitly"); } +/// record the extensions a repair relies on: the one supplying its route, and every rival +/// route's. a site served by two is reported rather than resolved, and each import keeps +/// the site in that state, so neither is removable on its own +pub(crate) fn record_conversion_uses<'db>( + context: &InferContext<'db, '_>, + repair: &ConversionRepair<'db>, +) { + for extension in std::iter::once(repair.route) + .chain(repair.ambiguous_with.iter().copied()) + .filter_map(Route::extension) + { + context.record_extension_use(extension); + } +} + /// the conversions the value expression `value` needs so that it satisfies /// `declared`, as `(range to wrap, conversion)` in source order. /// @@ -1130,7 +1167,7 @@ pub(crate) fn conversion_info<'db>( // conformance is not a conversion at runtime: the value already answers // every member the protocol asks for, through the witness table its // conformance registered. the site emits nothing at all - Route::Conformance(_) => ConversionInfo::Call { + Route::Conformance { .. } => ConversionInfo::Call { prefix: String::new(), suffix: String::new(), replaces_value: false, diff --git a/crates/ty_python_semantic/src/types/deferred_imports.rs b/crates/ty_python_semantic/src/types/deferred_imports.rs new file mode 100644 index 0000000000..cb16b8347f --- /dev/null +++ b/crates/ty_python_semantic/src/types/deferred_imports.rs @@ -0,0 +1,451 @@ +//! basedpython: which module-level `from` imports can be deferred by rewriting their reads +//! +//! below python 3.15 there is no `lazy` keyword, so the transpiler defers a `from m import x` +//! by binding nothing at the import and writing every read of `x` as a read through an object +//! that resolves `m.x` on first use. that is only the program python would run when every +//! read the rewrite reaches is a read of that import, and every read of that import is +//! rewritten: [`deferrable_from_imports`] answers which imports that holds for, and with +//! which reads. an import it leaves out stays an ordinary import + +use ruff_db::parsed::parsed_module; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk_stmt}; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange}; +use rustc_hash::FxHashMap; + +use ty_python_core::ast_ids::HasScopedUseId; +use ty_python_core::definition::{Definition, DefinitionState}; +use ty_python_core::place::ScopedPlaceId; +use ty_python_core::scope::{FileScopeId, ScopeKind}; +use ty_python_core::{ProgramFile, semantic_index}; + +use crate::Db; + +/// a module-level `from` import name whose every read can be rewritten +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeferrableImport { + /// the `ast::Alias` the name is imported by + pub alias: TextRange, + /// every read of the binding the alias makes, as the range of its `ast::ExprName` + pub reads: Vec, +} + +/// where a name read or written in a module resolves +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Resolution { + /// the module's own namespace + Module, + /// a binding of some scope inside the module + Elsewhere, + /// either, depending on what has run: a class body that binds the name on some paths + /// reads its own binding where it has made one and the module's where it has not + Either, +} + +/// basedpython: the module-level `from` imports of `file` whose binding is the only thing that +/// ever binds its name, with every read of it +/// +/// an import qualifies when, across the whole module: +/// +/// - nothing else binds, declares or deletes the name at module scope, nor from a scope that +/// declares it `global`: the index records such a binding among the module scope's own, for +/// the reads after the nested scope, and a `del` under `global` is a name node the read walk +/// finds +/// - every read that can reach the module's binding reaches nothing else: a class body that +/// binds the name on some paths to a read and not on others disqualifies it +/// - the module has no `from … import *`, which can bind any name unseen, and binds neither +/// `__getattr__` nor `__dir__`, which a deferred import is served through +/// +/// a name inside a string is not a read here: in `.by` a string in a type expression is a +/// literal type, and a forward reference the transpiler writes as a string is its business +pub(crate) fn deferrable_from_imports<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, +) -> Vec { + let module = parsed_module(db, file.python_file(db)).load(db); + let index = semantic_index(db, file); + let global = FileScopeId::global(); + let global_table = index.place_table(global); + + let mut statements = ModuleStatements::default(); + statements.visit_body(module.suite()); + if statements.star_import { + return Vec::new(); + } + let binds = |name: &str| { + let Some(symbol) = global_table.symbol_id(name) else { + return false; + }; + let symbol = global_table.symbol(symbol); + symbol.is_bound() || symbol.is_declared() + }; + if binds("__getattr__") || binds("__dir__") { + return Vec::new(); + } + + // every definition the module scope makes, by the place it makes it of. a deletion is + // a `del`, whose target is a name node the read walk below finds, or the end of an + // `except … as` clause, which bound the name first + let use_def = index.use_def_map(global); + let mut definitions: FxHashMap>> = FxHashMap::default(); + for (_, definition, _) in use_def.definitions_with_usage() { + definitions + .entry(definition.place(db)) + .or_default() + .push(definition); + } + + let mut candidates: FxHashMap> = FxHashMap::default(); + for alias in statements.aliases { + let bound = alias.asname.as_ref().unwrap_or(&alias.name).id.clone(); + let sole = global_table.symbol_id(&bound).is_some_and(|symbol| { + let place = ScopedPlaceId::Symbol(symbol); + index + .try_definitions(AnyNodeRef::from(alias)) + .is_some_and(|own| { + own.len() == 1 + && definitions + .get(&place) + .is_some_and(|made| made.as_slice() == own) + }) + }); + // a second import of the name is a second binding, so neither import qualifies + candidates + .entry(bound) + .and_modify(|slot| *slot = None) + .or_insert(sole.then_some(alias.range())); + } + candidates.retain(|_, alias| alias.is_some()); + if candidates.is_empty() { + return Vec::new(); + } + + let mut names = NameReferences { + names: &candidates, + found: Vec::new(), + }; + names.visit_body(module.suite()); + let mut reads: FxHashMap> = FxHashMap::default(); + for reference in names.found { + let Some(Some(_)) = candidates.get(&reference.id) else { + continue; + }; + match (resolve(db, file, reference), reference.ctx) { + (Resolution::Elsewhere, _) => {} + (Resolution::Module, ast::ExprContext::Load) => { + reads + .entry(reference.id.clone()) + .or_default() + .push(reference.range()); + } + // a write the module's own table does not show — a walrus in a comprehension — + // or a read that may find another binding + _ => { + candidates.insert(reference.id.clone(), None); + } + } + } + + let mut deferrable: Vec = candidates + .into_iter() + .filter_map(|(name, alias)| { + Some(DeferrableImport { + alias: alias?, + reads: reads.remove(&name).unwrap_or_default(), + }) + }) + .collect(); + deferrable.sort_by_key(|import| import.alias.start()); + deferrable +} + +/// where `reference` resolves, as python resolves it +/// +/// a class body is where this differs from a lexical walk. a name the body binds anywhere is +/// read with `LOAD_NAME`: the class namespace, and then the module's — never an enclosing +/// function's — so a read the body has not bound on any path resolves to the module even +/// when a function around the class binds the name +fn resolve<'db>(db: &'db dyn Db, file: ProgramFile<'db>, reference: &ast::ExprName) -> Resolution { + let index = semantic_index(db, file); + let Some(own_scope) = index.try_expression_scope_id(&ast::ExprRef::from(reference)) else { + return Resolution::Either; + }; + for (scope_id, scope) in index.visible_ancestor_scopes(own_scope) { + if scope_id.is_global() { + return Resolution::Module; + } + let table = index.place_table(scope_id); + let Some(symbol) = table.symbol_by_name(&reference.id) else { + continue; + }; + if symbol.is_global() { + return Resolution::Module; + } + if symbol.is_nonlocal() { + return Resolution::Elsewhere; + } + if !(symbol.is_bound() || symbol.is_declared()) { + continue; + } + if scope.kind() != ScopeKind::Class { + return Resolution::Elsewhere; + } + // an annotation scope sees the class it is defined in through the class namespace, + // falling back to the names outside it when the class has not bound the name yet + if scope_id != own_scope || !reference.ctx.is_load() { + return if reference.ctx.is_load() { + Resolution::Either + } else { + Resolution::Elsewhere + }; + } + let scope = scope_id.to_scope_id(db, file); + let use_id = reference.scoped_use_id(db, file); + let (mut defined, mut undefined) = (false, false); + for binding in ty_python_core::use_def_map(db, scope).bindings_at_use(use_id) { + match binding.binding { + DefinitionState::Defined(_) => defined = true, + DefinitionState::Undefined | DefinitionState::Deleted => undefined = true, + } + } + return match (defined, undefined) { + (true, false) => Resolution::Elsewhere, + (false, _) => Resolution::Module, + (true, true) => Resolution::Either, + }; + } + Resolution::Module +} + +/// the module-level `from` imports, and whether there is a star import anywhere +#[derive(Default)] +struct ModuleStatements<'ast> { + aliases: Vec<&'ast ast::Alias>, + star_import: bool, +} + +impl<'ast> SourceOrderVisitor<'ast> for ModuleStatements<'ast> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + match stmt { + Stmt::ImportFrom(import) => { + if import.names.iter().any(|alias| &alias.name == "*") { + self.star_import = true; + } else { + self.aliases.extend(import.names.iter()); + } + } + // a function's or a class's imports bind in that scope + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} + _ => walk_stmt(self, stmt), + } + } + + fn visit_expr(&mut self, _expr: &'ast Expr) {} +} + +/// every name node spelling one of `names`, whatever it does and wherever it is +struct NameReferences<'a, 'ast> { + names: &'a FxHashMap>, + found: Vec<&'ast ast::ExprName>, +} + +impl<'ast> SourceOrderVisitor<'ast> for NameReferences<'_, 'ast> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && self.names.contains_key(&name.id) + { + self.found.push(name); + } + walk_expr(self, expr); + } +} + +#[cfg(test)] +mod tests { + use ruff_db::files::system_path_to_file; + use ty_python_core::ProgramFile; + + use crate::db::tests::TestDbBuilder; + + /// each deferrable import of `/src/main.py`, as `source` spells its alias, with the text of + /// every read of it + fn deferrable(source: &str) -> Vec<(String, Vec)> { + let db = TestDbBuilder::new() + .with_file("/src/main.py", source) + .build() + .expect("the db builds"); + let file = system_path_to_file(&db, "/src/main.py").expect("the file exists"); + let file = ProgramFile::new(&db, file, db.program_environment().program(&db)); + super::deferrable_from_imports(&db, file) + .into_iter() + .map(|import| { + ( + source[import.alias].to_owned(), + import + .reads + .iter() + .map(|read| source[*read].to_owned()) + .collect(), + ) + }) + .collect() + } + + /// whether `source`'s import of `dumps` is deferred + fn defers_dumps(source: &str) -> bool { + deferrable(source).iter().any(|(alias, _)| alias == "dumps") + } + + #[test] + fn a_name_bound_only_by_its_import_is_deferred_with_every_read() { + let found = deferrable( + "from json import dumps, loads as parse\n\ + def f():\n return dumps(parse('1'))\n\ + print(dumps)\n", + ); + assert_eq!( + found, + vec![ + ( + "dumps".to_owned(), + vec!["dumps".to_owned(), "dumps".to_owned()] + ), + ("loads as parse".to_owned(), vec!["parse".to_owned()]), + ] + ); + } + + /// every way python binds, declares or deletes a name, at module scope or from a scope + /// that declares it `global`, is a second binding + #[test] + fn any_other_binding_of_the_name_keeps_the_import() { + for rebinding in [ + "dumps = 5", + "dumps += 1", + "dumps: int = 1", + "dumps: int", + "def dumps(): pass", + "class dumps: pass", + "for dumps in []: pass", + "with open('x') as dumps: pass", + "try:\n pass\nexcept Exception as dumps:\n pass", + "if (dumps := 1): pass", + "print([(dumps := i) for i in []])", + "del dumps", + "from json import dumps", + "from pickle import dumps", + "import dumps", + "import pickle as dumps", + "match 1:\n case dumps:\n pass", + "match [1]:\n case [*dumps]:\n pass", + "def f():\n global dumps\n dumps = 1", + "def f():\n global dumps\n del dumps", + "def f():\n global dumps\n def dumps(): pass", + "def f():\n global dumps\n import json as dumps", + "def f():\n global dumps\n for dumps in []: pass", + "class C:\n global dumps\n dumps = 1", + ] { + let source = format!("from json import dumps\n{rebinding}\nprint(dumps)\n"); + assert!(!defers_dumps(&source), "deferred:\n{source}"); + } + } + + /// a binding in a scope of its own is that scope's, and leaves the module's alone + #[test] + fn a_local_binding_of_the_name_is_not_the_module_s() { + let found = deferrable( + "from json import dumps\n\ + def f(dumps):\n return dumps\n\ + def g():\n dumps = 1\n return dumps\n\ + lambda dumps: dumps\n\ + [dumps for dumps in []]\n\ + class C:\n dumps = 1\n seen = dumps\n\ + def h():\n return dumps\n", + ); + assert_eq!(found, vec![("dumps".to_owned(), vec!["dumps".to_owned()])]); + } + + /// a read reaches the module's binding from any scope between it and the module that does + /// not bind the name + #[test] + fn a_read_from_any_scope_that_reaches_the_module_is_a_read() { + let found = deferrable( + "from json import dumps\n\ + def nested():\n def inner():\n return dumps\n return inner\n\ + class Body:\n value = dumps\n\ + def default(f=dumps):\n return f\n\ + def reads_global():\n global dumps\n return dumps\n\ + later = lambda: dumps\n\ + listed = [dumps for _ in []]\n\ + text = f'{dumps}'\n\ + @dumps\n\ + def decorated(): pass\n", + ); + assert_eq!(found.len(), 1); + assert_eq!(found[0].1.len(), 8, "{found:?}"); + } + + /// a class body reads a name it binds with `LOAD_NAME`: the class namespace, and then the + /// module's — never a function's around it + #[test] + fn a_class_body_reads_the_module_until_it_binds_the_name() { + let found = deferrable( + "from json import dumps\n\ + def outer():\n dumps = 1\n class Body:\n seen = dumps\n dumps = 2\n", + ); + assert_eq!(found, vec![("dumps".to_owned(), vec!["dumps".to_owned()])]); + } + + /// a read the body has bound on some paths finds either binding + #[test] + fn a_class_body_read_that_may_find_its_own_binding_keeps_the_import() { + assert!(!defers_dumps( + "from json import dumps\nimport random\nclass Body:\n if random.random():\n dumps = 1\n seen = dumps\n" + )); + } + + /// a star import can bind any name unseen, and a module's own `__getattr__` or `__dir__` + /// is what a deferred name would be served through + #[test] + fn a_module_that_could_rebind_unseen_defers_nothing() { + for source in [ + "from os import *\nfrom json import dumps\nprint(dumps)\n", + "from json import dumps\ndef __getattr__(name): return dumps\n", + "from json import dumps\ndef __dir__(): return []\n", + "from json import dumps\n__getattr__ = print\n", + ] { + assert!(!defers_dumps(source), "deferred:\n{source}"); + } + } + + /// an import inside a function or a class binds there + #[test] + fn an_import_in_a_scope_of_its_own_is_not_a_module_import() { + assert!(deferrable("def f():\n from json import dumps\n return dumps\n").is_empty()); + } + + /// a name an import binds only on some paths is still bound by nothing else, and reads of + /// it are reads of the import. what a read finds where the import did not run is the + /// runtime's business: python's lookup falls back to the builtins + #[test] + fn an_import_on_some_paths_is_still_the_only_binding() { + let found = deferrable( + "import sys\nif sys.version_info < (3, 0):\n from itertools import izip as zip\nprint(zip)\n", + ); + assert_eq!( + found, + vec![("izip as zip".to_owned(), vec!["zip".to_owned()])] + ); + } + + /// a package importing its own submodule, by relative or absolute name, is an ordinary + /// import of a name nothing else binds + #[test] + fn a_package_importing_its_own_submodule_is_deferrable() { + assert_eq!( + deferrable("from . import helper\ndef use():\n return helper.NAME\n"), + vec![("helper".to_owned(), vec!["helper".to_owned()])] + ); + } +} diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 27330e1130..014b260512 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -23,7 +23,9 @@ use crate::types::function::{FunctionDecorators, FunctionType, KnownFunction, Ov use crate::types::infer::UnsupportedComparisonError; use crate::types::overrides::MethodKind; use crate::types::protocol_class::ProtocolMember; +use crate::types::reified_infer::{ErasedTargetReason, ReifiedInferenceError}; use crate::types::repeated_underscore::UnderscoreRefusal; +use crate::types::runtime_reads::RuntimeReadCheck; use crate::types::special_form::TypeQualifier; use crate::types::string_annotation::{ ESCAPE_CHARACTER_IN_FORWARD_ANNOTATION, IMPLICIT_CONCATENATED_STRING_TYPE_ANNOTATION, @@ -63,6 +65,8 @@ use ty_module_resolver::{KnownModule, Module, ModuleName, SearchPath, file_to_mo use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; use ty_python_core::place::{PlaceTable, ScopedPlaceId}; use ty_python_core::predicate::CaseNamePredicateKind; +use ty_python_core::scope::FileScopeId; +use ty_python_core::symbol::ScopedSymbolId; use ty_python_core::{ProgramFile, global_scope, place_table, use_def_map}; const RUNTIME_CHECKABLE_DOCS_URL: &str = @@ -211,6 +215,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&OPTIONAL_OBJECT_CONVERSION); registry.register_lint(&BOOL_AS_INT); registry.register_lint(&UNUSED_RETURN_VALUE); + registry.register_lint(&UNUSED_IMPORT); registry.register_lint(&MISSING_CONTEXT_ARGUMENT); registry.register_lint(&AMBIGUOUS_CONTEXT_ARGUMENT); registry.register_lint(&REIFIED_WITHOUT_RECEIVER); @@ -1670,12 +1675,17 @@ declare_lint! { /// ## What it does /// Checks for conversion sites where more than one conversion applies — two /// dunders, a dunder and an in-scope conformance, or two applicable - /// `implementation`s of the same interface and type. + /// `implementation`s of the same interface and type. A plain tuple literal + /// that more than one expected anonymous named tuple accepts is one too: + /// `(1,)` where `(a: int) | (b: int)` is expected. /// /// ## Why is this bad? /// `__from__` and `__into__` are hand-written bodies that can disagree, so /// which one runs must not depend on arbitrary ordering. Remove one of them, - /// or write the conversion you want explicitly. + /// or write the conversion you want explicitly. A plain tuple literal is + /// constructed as the anonymous named tuple it stands for, which decides the + /// field names the value has, and nothing in the literal says which one was + /// meant: name the fields, as in `(a=1)`. /// /// ## Example /// @@ -2860,6 +2870,80 @@ declare_lint! { } } +declare_lint! { + /// ## What it does + /// Checks a basedpython file for an import that nothing uses: no name it binds + /// is read, and no `extension` or conformance it brings into scope supplies + /// anything the file relies on. + /// + /// ## Why is this bad? + /// An unused import costs time when the module is imported, can create an + /// import cycle, and tells a reader the module depends on something it does + /// not. + /// + /// The extension members a module declares are only applicable in the files + /// that import it, so an import can be used without its name ever being read. + /// This rule counts that as a use, which a linter working one file at a time + /// cannot — so in a `.by` file it takes the place of buff's `F401`, which + /// leaves `.by` files alone. + /// + /// Otherwise the rule is `F401`'s: + /// + /// - a read of a binding counts for every later binding of that name in the + /// scope. An import nothing reads before another import rebinds the name + /// is reported too, as `F811` reports the redefinition, so one round of + /// fixes removes every one. An import inside a class body is never + /// reported + /// - `import a as a`, `from a import b as b` and `from a export b` are + /// re-exports, a name listed in `__all__` is exported, and + /// `from __future__ import …` is a directive — none of them is reported + /// - in a package's `__init__.by`, and inside a `try` that handles + /// `ImportError`, an unused import is reported without a fix: it may be a + /// re-export, or a test for whether the module is installed + /// + /// ## Examples + /// ```by + /// # colours.by + /// from enum import Enum + /// + /// extension Enum: + /// let lower_name: str + /// get() = self.name.lower() + /// ``` + /// + /// ```by + /// # main.by + /// import json # error: `json` imported but unused + /// import colours # ok — the extension it declares is used below + /// from enum import Enum + /// + /// class Colour(Enum): + /// RED = 1 + /// + /// def describe(colour: Colour) -> str: + /// return colour.lower_name + /// ``` + /// + /// Use instead: + /// + /// ```by + /// import colours + /// from enum import Enum + /// ``` + /// + /// ## Fix safety + /// The fix removes the import, and is safe: nothing in the file reads it. + /// Removing an import also stops the imported module from being run, which + /// matters only if the module's import has a side effect the program relies + /// on. + pub(crate) static UNUSED_IMPORT = { + summary: "detects an import that nothing uses", + status: LintStatus::stable("0.0.1-beta.2"), + default_level: Level::Warn, + ty_compat: TyCompat::BasedPython, + } +} + declare_lint! { /// ## What it does /// Checks for a call, written as a statement of its own, whose result is @@ -4117,6 +4201,254 @@ declare_lint! { pub struct TypeCheckDiagnostics { diagnostics: Vec, used_suppressions: FxHashSet, + /// basedpython: allocated by the first use recorded. every inference builder holds its + /// diagnostics by value, and inference recurses through builders on the stack, so the + /// record is kept out of line + uses: Option>, +} + +/// basedpython: what an inference relied on that the semantic index has no record of +#[derive(Default, Eq, PartialEq, get_size2::GetSize)] +struct InferenceUses { + /// the modules whose `extension` or conformance supplied something inference relied on — + /// a member, an operator's dunder, a conversion, a conformance. none of these reads a name + /// the module's import binds, so the import that makes the module's extensions applicable + /// is used only through this record (see `unused-import`) + extension_modules: FxHashSet, + /// the names inference read as every binding their scope makes, where the semantic index + /// records no read — a receiver callable read as a method (`x.fn()`), a name in an + /// annotation, which is evaluated lazily, a name read through `global`, the enum a + /// context-sensitive name or case name is lowered to qualify (`Color.Red`), and each + /// module-level name a runtime spelling the transpiled program writes reads. each is a + /// symbol of the scope that owns it + scope_reads: FxHashSet<(FileScopeId, ScopedSymbolId)>, + /// the sites that write module-level names into code that runs, whose names are checked + /// once the file is inferred ([`crate::types::runtime_reads`]) + runtime_read_checks: Vec, +} + +/// reports a bare call of the reified generic function `name` whose specialization cannot be +/// written into the call, saying why — `None` when it would have to pass through unpacked +/// arguments +pub(crate) fn report_unspecialized_reified_call<'db>( + context: &InferContext<'db, '_>, + call: TextRange, + name: &str, + failure: Option>, +) { + let Some(builder) = context.report_lint(&UNSPECIALIZED_REIFIED_GENERIC, call) else { + return; + }; + let (db, env) = (context.db(), context.program_environment()); + let mut diagnostic = builder.into_diagnostic(format_args!( + "Cannot call reified generic function `{name}` without explicit specialization" + )); + match failure { + None => diagnostic.info(format_args!( + "the inferred specialization cannot be injected through unpacked arguments — \ + specialize with `{name}[...]`" + )), + Some(ReifiedInferenceError::Unsolved(parameter)) => diagnostic.info(format_args!( + "reified type parameter `{parameter}` cannot be inferred from the arguments — \ + specialize with `{name}[...]`" + )), + Some(ReifiedInferenceError::Unspellable(parameter, ty)) => { + diagnostic.info(format_args!( + "inferred type `{}` for type parameter `{parameter}` has no runtime spelling — \ + specialize with `{name}[...]`", + ty.display(db, env), + )); + } + Some(ReifiedInferenceError::NotAtRuntime(read, binding)) => { + diagnostic.info(binding.describe(&read)); + } + Some(ReifiedInferenceError::NoBinding) => diagnostic.info(format_args!( + "the specialization cannot be inferred from this call — specialize with \ + `{name}[...]`" + )), + } +} + +/// reports a bare construction of the reified generic class `name` whose specialization +/// cannot be written into the call, saying why +pub(crate) fn report_unspecialized_reified_construction<'db>( + context: &InferContext<'db, '_>, + call: TextRange, + name: &str, + failure: ReifiedInferenceError<'db>, +) { + let Some(builder) = context.report_lint(&UNSPECIALIZED_REIFIED_GENERIC, call) else { + return; + }; + let (db, env) = (context.db(), context.program_environment()); + let mut diagnostic = builder.into_diagnostic(format_args!( + "Cannot construct reified generic class `{name}` without a specialization" + )); + match failure { + ReifiedInferenceError::Unsolved(parameter) => diagnostic.info(format_args!( + "reified type parameter `{parameter}` cannot be inferred here — construct a \ + specialization, as in `{name}[int](...)`" + )), + ReifiedInferenceError::Unspellable(parameter, ty) => diagnostic.info(format_args!( + "inferred type `{}` for type parameter `{parameter}` has no runtime spelling — \ + construct a specialization, as in `{name}[int](...)`", + ty.display(db, env), + )), + ReifiedInferenceError::NotAtRuntime(read, binding) => { + diagnostic.info(binding.describe(&read)); + } + ReifiedInferenceError::NoBinding => diagnostic.info(format_args!( + "the specialization cannot be inferred from this call — construct one, as in \ + `{name}[int](...)`" + )), + } +} + +pub(crate) fn report_erased_type_check( + context: &InferContext<'_, '_>, + primary: TextRange, + target: &str, + reason: ErasedTargetReason, +) { + let Some(builder) = context.report_lint(&ERASED_TYPE_CHECK, primary) else { + return; + }; + match reason { + ErasedTargetReason::NotAtRuntime(name, binding) => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: {}", + binding.why_not_checkable(&name) + )); + diagnostic.info(binding.describe(&name)); + } + ErasedTargetReason::Protocol => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a protocol's instances don't \ + record which specialization they satisfy" + )); + diagnostic.info(format_args!( + "an instance's `__orig_class__` names its concrete class, never the protocol, \ + and a structural `isinstance` check can't see type arguments" + )); + diagnostic.info(format_args!( + "reify the type parameter (`def f[T](x: T)`), or test against a concrete class \ + that records the specialization on its instances or across its mro" + )); + } + ErasedTargetReason::NotSubscriptable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: the target class cannot be \ + subscripted" + )); + diagnostic.info(format_args!( + "the class has no `__class_getitem__` on this python version, so the test's \ + runtime check would raise `TypeError` evaluating `{target}`" + )); + diagnostic.info(format_args!( + "drop the type arguments and test against the bare class, which is all the \ + runtime records anyway" + )); + } + ErasedTargetReason::BuiltinCollection => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a builtin collection's instances \ + don't record their type arguments" + )); + diagnostic.info(format_args!( + "test against a subclass that fixes the arguments (`class B(list[int])`), \ + whose `__orig_bases__` the probe can unwind" + )); + } + ErasedTargetReason::Dynamic => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` admits every value, \ + so there is nothing for the test to look for" + )); + diagnostic.info(format_args!( + "a test that can only answer `True` narrows nothing; drop it, or name the \ + type the value is expected to have" + )); + } + ErasedTargetReason::Callable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a callable's parameter and \ + return types are not recorded on the value" + )); + diagnostic.info(format_args!( + "the runtime can see that a value is callable and nothing more, so the \ + signature would be assumed rather than checked" + )); + diagnostic.info(format_args!( + "test against a bare `Callable`, which asks only what the runtime records, \ + or take the value's type from the call site" + )); + } + ErasedTargetReason::TypedDict => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a `TypedDict`'s instances are \ + plain dicts" + )); + diagnostic.info(format_args!( + "the test could only ask whether the value is a `dict`, assuming every key \ + and value type `{target}` declares" + )); + diagnostic.info(format_args!( + "`isinstance` refuses a `TypedDict` for the same reason" + )); + } + ErasedTargetReason::NonRuntimeCheckableProtocol => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` has a member with no \ + runtime spelling" + )); + diagnostic.info(format_args!( + "a member whose type the emitted python cannot name leaves nothing to check \ + the value's own annotation against" + )); + diagnostic.info(format_args!( + "decorate `{target}` with `@typing.runtime_checkable` for the check python \ + itself performs, which asks only that the members are present" + )); + } + ErasedTargetReason::Intersection => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: an intersection has no single \ + runtime form" + )); + diagnostic.info(format_args!( + "test each part separately and join the tests with `and`" + )); + } + ErasedTargetReason::Subclass => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `issubclass` cannot take what \ + `{target}` is parameterized by" + )); + diagnostic.info(format_args!( + "the test can see that the value is a class and nothing more; name the class \ + it must be a subclass of" + )); + } + ErasedTargetReason::UncomparableLiteral => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: equality does not decide \ + membership of `{target}`" + )); + diagnostic.info(format_args!( + "`0.0 == -0.0` and `nan != nan`, so a `float` or `complex` literal names a \ + type the runtime cannot be asked about; `LiteralString` is a property of how \ + a value was written rather than of the value" + )); + } + ErasedTargetReason::Unspellable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` has no runtime form" + )); + diagnostic.info(format_args!( + "the emitted python has no expression to evaluate for this target" + )); + } + } } pub(crate) fn report_mismatched_type_name<'db>( @@ -4154,6 +4486,13 @@ impl TypeCheckDiagnostics { pub(super) fn extend(&mut self, other: &TypeCheckDiagnostics) { self.diagnostics.extend_from_slice(&other.diagnostics); self.used_suppressions.extend(&other.used_suppressions); + if let Some(other_uses) = &other.uses { + let uses = self.uses_mut(); + uses.extension_modules.extend(&other_uses.extension_modules); + uses.scope_reads.extend(&other_uses.scope_reads); + uses.runtime_read_checks + .extend_from_slice(&other_uses.runtime_read_checks); + } } pub(super) fn extend_diagnostics(&mut self, diagnostics: impl IntoIterator) { @@ -4172,8 +4511,52 @@ impl TypeCheckDiagnostics { self.used_suppressions.len() } + /// basedpython: records that an extension or conformance declared in `module` supplied + /// something inference relied on + pub(crate) fn mark_extension_module_used(&mut self, module: File) { + self.uses_mut().extension_modules.insert(module); + } + + pub(crate) fn is_extension_module_used(&self, module: File) -> bool { + self.uses + .as_ref() + .is_some_and(|uses| uses.extension_modules.contains(&module)) + } + + /// basedpython: records that inference read every binding `symbol` has in `scope` + pub(crate) fn mark_scope_read(&mut self, scope: FileScopeId, symbol: ScopedSymbolId) { + self.uses_mut().scope_reads.insert((scope, symbol)); + } + + pub(crate) fn scope_reads(&self) -> impl Iterator { + self.uses + .iter() + .flat_map(|uses| uses.scope_reads.iter().copied()) + } + + /// basedpython: records a site whose module-level names are checked once the file is + /// inferred + pub(crate) fn record_runtime_read_check(&mut self, check: RuntimeReadCheck) { + self.uses_mut().runtime_read_checks.push(check); + } + + pub(crate) fn runtime_read_checks(&self) -> &[RuntimeReadCheck] { + self.uses + .as_ref() + .map_or(&[], |uses| &uses.runtime_read_checks) + } + + fn uses_mut(&mut self) -> &mut InferenceUses { + self.uses.get_or_insert_with(Box::default) + } + pub(crate) fn shrink_to_fit(&mut self) { self.used_suppressions.shrink_to_fit(); + if let Some(uses) = &mut self.uses { + uses.extension_modules.shrink_to_fit(); + uses.scope_reads.shrink_to_fit(); + uses.runtime_read_checks.shrink_to_fit(); + } self.diagnostics.shrink_to_fit(); } @@ -4182,7 +4565,13 @@ impl TypeCheckDiagnostics { } pub(crate) fn is_empty(&self) -> bool { - self.diagnostics.is_empty() && self.used_suppressions.is_empty() + !self.has_reports() && self.uses.is_none() + } + + /// whether anything was reported, suppressed or not. a recorded extension use is not a + /// report + pub(crate) fn has_reports(&self) -> bool { + !self.diagnostics.is_empty() || !self.used_suppressions.is_empty() } fn iter(&self) -> std::slice::Iter<'_, Diagnostic> { @@ -4907,8 +5296,11 @@ pub(super) fn report_invalid_assignment<'db>( value_node, target_ty, ); - if let Some((range, repair)) = conversions.first() { - crate::types::conversions::report_ambiguous_conversion(context, *range, repair); + if let Some(((range, repair), rest)) = conversions.split_first() { + crate::types::conversions::accept_conversion(context, *range, repair); + for (_, repair) in rest { + crate::types::conversions::record_conversion_uses(context, repair); + } return; } } diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 41c76ad0d9..22ce39e910 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -22,7 +22,7 @@ use ty_module_resolver::{KnownModule, Module, file_to_module}; use crate::Db; use crate::place::{DefinedPlace, Place, builtins_symbol, global_symbol}; use crate::types::callable::CallableTypeKind; -use crate::types::class::{ClassLiteral, ClassType, DynamicNamedTupleAnchor, GenericAlias}; +use crate::types::class::{ClassLiteral, ClassType, GenericAlias}; use crate::types::constraints::ConstraintSetBuilder; use crate::types::function::{FunctionType, OverloadLiteral}; use crate::types::generics::{GenericContext, Specialization}; @@ -32,6 +32,7 @@ use crate::types::signatures::{ }; use crate::types::tuple::{RepeatedUnit, TupleSpec, VariableLengthTuple, VariableSegment}; use crate::types::typevar::BoundTypeVarIdentity; +use crate::types::visibility::private_symbols; use crate::types::visitor::TypeVisitor; use crate::types::{ CallableType, DeferredOperation, DeferredType, DynamicType, IntersectionType, @@ -986,6 +987,9 @@ pub struct SourceSpelling { /// the outer one, `Outer.Inner`. such a path names the outer class, which is unbound while /// it is being made, so it resolves only where it is not evaluated as it is written pub reads_nested_class: bool, + /// those of [`Self::bare_classes`] that are module-level `private` symbols of the file, + /// which the transpiler emits under a name of its own + pub module_private_classes: Vec, } /// Collects every class a type names, so each can be spelled for a particular file. @@ -1153,6 +1157,23 @@ impl<'db> Type<'db> { return None; } } + // a bare class the file's global scope binds under a name the file declares + // `private` is that `private` symbol + let private = private_symbols(db, file.file(db)); + let module_private_classes = bare + .iter() + .filter(|name| { + private.contains(**name) + && matches!( + global_symbol(db, file, name).place, + Place::Defined(defined) if defined + .ty + .as_class_literal() + .is_some_and(|class| classes.contains(&class)) + ) + }) + .map(|name| (*name).to_owned()) + .collect(); let bare_classes = bare.into_iter().map(str::to_owned).collect(); let settings = DisplaySettings { @@ -1164,6 +1185,7 @@ impl<'db> Type<'db> { modules: modules.into_iter().collect(), bare_classes, reads_nested_class, + module_private_classes, }) } } @@ -1349,34 +1371,27 @@ impl<'db> FmtDetailed<'db> for ClassDisplay<'_, 'db> { // class name. Positional fields use the synthetic `arg` name and // are rendered without a label if let ClassLiteral::DynamicNamedTuple(nt) = self.class - && nt.name(self.db).as_str().starts_with("_AnonNamedTuple_") + && let Some(spec) = nt.anchor(self.db).anonymous_spec() { - let spec = match nt.anchor(self.db) { - DynamicNamedTupleAnchor::CollectionsDefinition { spec, .. } - | DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => Some(*spec), - DynamicNamedTupleAnchor::TypingDefinition(_) => None, - }; - if let Some(spec) = spec { - let fields = spec.fields(self.db); - let ty = Type::ClassLiteral(self.class); - let mut f = f.with_type(ty); - f.write_char('(')?; - for (i, field) in fields.iter().enumerate() { - if i > 0 { - f.write_str(", ")?; - } - let synthetic_pos = field.name.as_str() == format!("arg{i}"); - if !synthetic_pos { - write!(f, "{}: ", field.name)?; - } - field - .ty - .display_with(self.db, env, self.settings.clone()) - .fmt_detailed(&mut f)?; + let fields = spec.fields(self.db); + let ty = Type::ClassLiteral(self.class); + let mut f = f.with_type(ty); + f.write_char('(')?; + for (i, field) in fields.iter().enumerate() { + if i > 0 { + f.write_str(", ")?; } - f.write_char(')')?; - return Ok(()); + let synthetic_pos = field.name.as_str() == format!("arg{i}"); + if !synthetic_pos { + write!(f, "{}: ", field.name)?; + } + field + .ty + .display_with(self.db, env, self.settings.clone()) + .fmt_detailed(&mut f)?; } + f.write_char(')')?; + return Ok(()); } let qualification_level = self.settings.qualified.get(&**self.class.name(self.db)); diff --git a/crates/ty_python_semantic/src/types/exceptions.rs b/crates/ty_python_semantic/src/types/exceptions.rs index db211d07df..85048929cc 100644 --- a/crates/ty_python_semantic/src/types/exceptions.rs +++ b/crates/ty_python_semantic/src/types/exceptions.rs @@ -63,6 +63,7 @@ use crate::types::diagnostic::{ }; use crate::types::function::{FunctionLiteral, FunctionType, OverloadLiteral}; use crate::types::generics::{ApplySpecialization, Specialization, enclosing_binding_contexts}; +use crate::types::runtime_spelling::RuntimeSpelling; use crate::types::typevar::{BindingContext, BoundTypeVarInstance, TypeVarBoundOrConstraints}; use crate::types::visitor::any_over_type; use crate::types::{ @@ -1137,7 +1138,7 @@ pub struct RaisesRuntimeTarget { /// The `isinstance` target with every type parameter widened to its declared /// ceiling — what the guard tests when nothing says which type the caller /// chose. - pub ceiling: String, + pub ceiling: RuntimeSpelling, /// How to build the exact target at the call, when the clause names a type /// parameter that carries a runtime value. pub resolved: Option, @@ -1148,7 +1149,7 @@ pub struct RaisesRuntimeTarget { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResolvedRaisesTarget { /// The target, with each of those parameters spelled by its name. - pub expression: String, + pub expression: RuntimeSpelling, /// The guarded function's own reified parameters, read off the specialization /// it is called through. pub own: Vec, @@ -1187,7 +1188,7 @@ pub fn declared_raises_runtime_target<'db>( } if declared.is_never() { return Some(RaisesRuntimeTarget { - ceiling: "()".to_string(), + ceiling: RuntimeSpelling::text("()"), resolved: None, }); } @@ -1249,7 +1250,7 @@ fn resolved_runtime_target<'db>( let mut own = Vec::new(); let mut receiver = Vec::new(); let mut reads_any = false; - let mut parts: Vec = Vec::new(); + let mut parts: Vec = Vec::new(); for element in union_elements(db, declared) { let part = match element { Type::TypeVar(bound_typevar) if let Some(source) = source_of(bound_typevar) => { @@ -1265,7 +1266,7 @@ fn resolved_runtime_target<'db>( names.push(name.clone()); } reads_any = true; - name + RuntimeSpelling::text(name) } element => crate::types::soundness::runtime_check_target( db, @@ -1286,7 +1287,12 @@ fn resolved_runtime_target<'db>( // flattening let expression = match parts.len() { 1 => parts.pop()?, - _ => format!("({})", parts.join(", ")), + _ => { + let mut expression = RuntimeSpelling::text("("); + expression.push(&RuntimeSpelling::join(parts, ", ")); + expression.push_str(")"); + expression + } }; Some(ResolvedRaisesTarget { expression, diff --git a/crates/ty_python_semantic/src/types/extensions.rs b/crates/ty_python_semantic/src/types/extensions.rs index 7f5ca2e7f4..c9936dc7c4 100644 --- a/crates/ty_python_semantic/src/types/extensions.rs +++ b/crates/ty_python_semantic/src/types/extensions.rs @@ -24,11 +24,13 @@ //! [`ClassDef`]: ruff_python_ast::StmtClassDef use ruff_db::files::File; +use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; use ruff_python_ast as ast; use ruff_python_ast::name::Name; use rustc_hash::{FxHashMap, FxHashSet}; use ty_module_resolver::{ModuleName, resolve_module}; +use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_core::{global_scope, place_table, semantic_index}; use crate::Db; @@ -181,6 +183,74 @@ pub(crate) fn applicable_extensions(db: &dyn Db, file: File) -> Box<[StaticClass extensions.into_boxed_slice() } +/// an import in a file that makes another module's extensions applicable there +#[derive(Debug, Clone, Copy, PartialEq, Eq, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) struct ExtensionGrant<'db> { + /// the module whose extensions the import makes applicable + pub(crate) module: File, + /// the import's definition: one alias of an `import` or a `from … import` + pub(crate) import: Definition<'db>, +} + +/// every import in `file` that makes another module's extensions applicable, with +/// the module each one grants. +/// +/// This is [`applicable_extensions`]'s visibility rule read backwards, so it has to +/// grant exactly what that one does: `import a.b` grants `a.b` and `a`, and +/// `from a import x` grants `a`, wherever in the file the statement is written. A +/// module declaring no extension is left out, because no extension use can name it +#[salsa::tracked(returns(deref), heap_size = ruff_memory_usage::heap_size)] +pub(crate) fn extension_granting_imports(db: &dyn Db, file: File) -> Box<[ExtensionGrant<'_>]> { + if !file.source_type(db).is_basedpython() { + return Box::default(); + } + let program_file = db.program_file(file); + let index = semantic_index(db, program_file); + let parsed = parsed_module(db, program_file.python_file(db)).load(db); + let importing = ImportingFile::File(file, program_file.resolver_environment(db)); + let mut grants = Vec::new(); + for scope in index.scope_ids() { + let use_def = index.use_def_map(scope.file_scope_id(db)); + for (_, definition, _) in use_def.definitions_with_usage() { + let modules: Vec = match definition.kind(db) { + DefinitionKind::Import(import) => { + let alias = import.alias(&parsed); + if alias.is_resource { + continue; + } + let Some(name) = ModuleName::new(&alias.name) else { + continue; + }; + name.ancestors().collect() + } + DefinitionKind::ImportFrom(import) => { + let Ok(name) = + ModuleName::from_import_statement(db, importing, import.import(&parsed)) + else { + continue; + }; + vec![name] + } + _ => continue, + }; + for name in modules { + let Some(module) = + resolve_module(db, importing, &name).and_then(|module| module.file(db)) + else { + continue; + }; + if module != file && !extensions_in_module(db, module).is_empty() { + grants.push(ExtensionGrant { + module, + import: definition, + }); + } + } + } + } + grants.into_boxed_slice() +} + /// the classes an applicable extension supplies a conversion dunder for. /// /// The transpiler's hot-path gate asks whether a type might convert before doing @@ -563,6 +633,21 @@ pub(crate) struct ExtensionMemberResolution<'db> { /// another applicable extension that also supplies the member — an /// ambiguity the checker reports at the access site pub(crate) ambiguous_with: Option>, + /// the conformance the receiver reached the member through, when the member is a + /// protocol extension's and the receiver is not an instance of the protocol + pub(crate) conformance: Option>, +} + +impl<'db> ExtensionMemberResolution<'db> { + /// every extension the resolution relied on: the one supplying the member, the + /// conformance it was reached through, and a rival that supplies it too. a file's + /// import of each one's module is what makes it applicable, so each makes that import + /// a used one + pub(crate) fn relied_on(&self) -> impl Iterator> { + std::iter::once(self.extension) + .chain(self.conformance) + .chain(self.ambiguous_with) + } } /// resolve an attribute access that found no regular member: consult the @@ -663,9 +748,13 @@ pub(crate) fn resolve_extension_members<'db>( } resolved.push(ExtensionMemberResolution { extension, - ty: bind(member, applicable.conformed_as), + ty: bind( + member, + applicable.conformed_as.map(|(_, protocol)| protocol), + ), kind, ambiguous_with: None, + conformance: applicable.conformed_as.map(|(conformance, _)| conformance), }); } @@ -947,10 +1036,10 @@ pub(crate) fn own_member<'db>( /// an extension member that applies to one receiver struct ApplicableMember<'db> { member: Type<'db>, - /// the interface the receiver reached this member through, when it was - /// reached by conformance rather than by being an instance of the extended + /// the conformance and the interface the receiver reached this member through, when + /// it was reached by conformance rather than by being an instance of the extended /// type. that interface is what the member's `self` is written as - conformed_as: Option>, + conformed_as: Option<(StaticClassLiteral<'db>, ClassType<'db>)>, } /// if `extension` applies to `receiver_class` and declares `name`, return the @@ -1000,12 +1089,13 @@ fn applicable_member<'db>( }); } let target = extended_class(db, extension)?; - let conformed = conformance::conformance_for(db, env, file, receiver_class, target)?; + let (conformance, conformed) = + conformance::conformance_for(db, env, file, receiver_class, target)?; let application = applied_at(db, env, extension, target, conformed)?; let member = own_member(db, extension, name)?; Some(ApplicableMember { member: application.apply(db, env, extension, member), - conformed_as: Some(conformed), + conformed_as: Some((conformance, conformed)), }) } diff --git a/crates/ty_python_semantic/src/types/generics.rs b/crates/ty_python_semantic/src/types/generics.rs index 13e0c31b1b..e9df8bab33 100644 --- a/crates/ty_python_semantic/src/types/generics.rs +++ b/crates/ty_python_semantic/src/types/generics.rs @@ -38,10 +38,10 @@ use crate::types::visitor::{ use crate::types::{ ApplyTypeMappingVisitor, BindingContext, BoundTypeVarInstance, CallableType, CallableTypes, ClassLiteral, ErrorContext, FindLegacyTypeVarsVisitor, IntersectionType, KnownClass, - KnownInstanceType, MaterializationKind, PromotionKind, PromotionMode, SubclassOfInner, Type, - TypeAliasType, TypeContext, TypeMapping, TypeRecursionContext, TypeVarBoundOrConstraints, - TypeVarKind, TypeVarVariance, UnionAccumulator, UnionType, binding_type, - infer_definition_types, inferred_declaration, + KnownInstanceType, LiteralPromotion, MaterializationKind, PromotionKind, PromotionMode, + SubclassOfInner, Type, TypeAliasType, TypeContext, TypeMapping, TypeRecursionContext, + TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, UnionAccumulator, UnionType, + binding_type, infer_definition_types, inferred_declaration, }; use crate::{Db, FxIndexMap, FxOrderMap, FxOrderSet}; use ty_python_core::definition::{Definition, DefinitionKind}; @@ -1759,20 +1759,30 @@ impl<'db> Specialization<'db> { // reads a different type back out of it, so widening its literals only loses // precision. (an invariant or contravariant argument keeps its literals too — the // arm below flips promotion off, because widening it would be unsound) - (variance, TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular)) - if variance.is_covariant() => - { - ty.apply_type_mapping_impl( - db, - env, - &TypeMapping::Promote( - PromotionMode::On, - PromotionKind::RegularKeepingLiterals, - ), - tcx, - visitor, - ) - } + ( + variance, + TypeMapping::Promote( + PromotionMode::On, + PromotionKind::Regular { + literals: LiteralPromotion::Widen, + numeric, + functions, + }, + ), + ) if variance.is_covariant() => ty.apply_type_mapping_impl( + db, + env, + &TypeMapping::Promote( + PromotionMode::On, + PromotionKind::Regular { + literals: LiteralPromotion::Keep, + numeric: *numeric, + functions: *functions, + }, + ), + tcx, + visitor, + ), (variance, _) if variance.is_covariant() => { ty.apply_type_mapping_impl(db, env, type_mapping, tcx, visitor) } diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index 0760fb8e0f..268478f11e 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -97,6 +97,14 @@ bitflags::bitflags! { /// The expression refers to a `TypeVarTuple` without unpacking it. const INVALID_BARE_TYPE_VAR_TUPLE = 1 << 2; + + /// basedpython: a type expression python evaluates where it stands, which inference + /// reads as deferred all the same, as it does an annotation: the type arguments of a + /// specialization written as a value (`list[F]()`), and a type-expression argument of + /// a typing construct assigned to a name (`T = TypeVar("T", bound=F)`). a name in one + /// may be bound only further on, and the transpiler writes it as the forward reference + /// python reads lazily + const READ_DEFERRED = 1 << 3; } } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index c6ec47017f..c11b6f40c7 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -56,6 +56,7 @@ use crate::reachability::{ }; use crate::subscript::PyIndex; use crate::types::add_inferred_python_version_hint_to_diagnostic; +use crate::types::anon_named_tuple; use crate::types::attribute_write::{AssignmentAttributeMembers, assignment_attribute_members}; use crate::types::call::bind::{ ArgumentTypeContext, CallableDescription, CheckTypesMode, OverloadSet, @@ -65,20 +66,20 @@ use crate::types::call::{Argument, Binding, Bindings, CallArguments, CallError, use crate::types::call_type_forms::CallTypeForm; use crate::types::callable::CallableTypeKind; use crate::types::class::{ - ClassLiteral, CodeGeneratorKind, DynamicClassScopeOffset, DynamicNamedTupleAnchor, - DynamicNamedTupleLiteral, DynamicTypedDictAnchor, DynamicTypedDictLiteral, - FrozenDataclassDispatch, MethodDecorator, NamedTupleField, NamedTupleSpec, StaticClassLiteral, + ClassLiteral, CodeGeneratorKind, DynamicNamedTupleAnchor, DynamicNamedTupleLiteral, + DynamicTypedDictAnchor, DynamicTypedDictLiteral, FrozenDataclassDispatch, MethodDecorator, + NamedTupleField, NamedTupleSpec, StaticClassLiteral, }; use crate::types::constraints::{ConstraintSetBuilder, PathBounds, Solutions}; use crate::types::context::InferContext; -use crate::types::context_sensitive::{self, case_name_pattern_type}; +use crate::types::context_sensitive; use crate::types::dedicated::{django, pydantic}; use crate::types::deferred::{is_integer_operand, is_symbolic_operand}; use crate::types::definition_resolution::AttributeRead; use crate::types::diagnostic::{ - self, AMBIGUOUS_EXTENSION_MEMBER, CALL_NON_CALLABLE, CONFLICTING_DECLARATIONS, - CYCLIC_TYPE_ALIAS_DEFINITION, DYNAMIC_FUNCTION_DECORATOR_RETURN, ERASED_CAST_ARGUMENT, - ERASED_TYPE_CHECK, FINAL_ON_VARIABLE, GeneratorMismatchKind, IMPLICIT_DECLARATION, + self, AMBIGUOUS_CONVERSION, AMBIGUOUS_EXTENSION_MEMBER, CALL_NON_CALLABLE, + CONFLICTING_DECLARATIONS, CYCLIC_TYPE_ALIAS_DEFINITION, DYNAMIC_FUNCTION_DECORATOR_RETURN, + ERASED_CAST_ARGUMENT, FINAL_ON_VARIABLE, GeneratorMismatchKind, IMPLICIT_DECLARATION, INACCESSIBLE_MEMBER, INEFFECTIVE_FINAL, INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_DECLARATION, INVALID_ENUM_MEMBER_ANNOTATION, INVALID_FIELD_LOOKUP, INVALID_LEGACY_TYPE_VARIABLE, INVALID_NEWTYPE, INVALID_PARAMSPEC, @@ -90,14 +91,14 @@ use crate::types::diagnostic::{ TRAILING_LAMBDA_PARAMETERS, TypeCheckDiagnostics, UNANNOTATED_MODEL_FIELD, UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, UNRESOLVED_REFERENCE, UNSOUND_ASSIGNMENT, UNSOUND_CAST, UNSOUND_YIELD, - UNSPECIALIZED_REIFIED_GENERIC, UNSUPPORTED_OPERATOR, UNUSED_AWAITABLE, USED_UNDERSCORE_NAME, - YieldKind, autofix_with_notimplementederror, display_required_elements, + UNSUPPORTED_OPERATOR, UNUSED_AWAITABLE, USED_UNDERSCORE_NAME, YieldKind, + autofix_with_notimplementederror, display_required_elements, hint_if_stdlib_attribute_exists_on_other_versions, refutable_unpacking_applies, report_attempted_protocol_instantiation, report_bad_dunder_delattr_call, report_bad_dunder_delete_call, report_bool_as_int, report_bool_as_int_assignment, report_call_to_abstract_method, report_cannot_pop_required_field_on_typed_dict, report_capturing_case_name, report_capturing_case_name_alternative, - report_dynamic_function_decorator_return, report_invalid_assignment, + report_dynamic_function_decorator_return, report_erased_type_check, report_invalid_assignment, report_invalid_class_match_pattern, report_invalid_exception_caught, report_invalid_exception_cause, report_invalid_exception_raised, report_invalid_exception_tuple_caught, report_invalid_generator_yield_type, @@ -108,7 +109,8 @@ use crate::types::diagnostic::{ report_possibly_missing_attribute, report_possibly_unresolved_reference, report_restricted_constructor, report_too_many_positional_patterns_for_class_pattern, report_unfilled_decoration_context_parameters, report_unplaceable_starred_class_pattern, - report_unsound_assignment, report_unsound_yield, report_unsupported_augmented_assignment, + report_unsound_assignment, report_unsound_yield, report_unspecialized_reified_call, + report_unspecialized_reified_construction, report_unsupported_augmented_assignment, report_unsupported_comparison, }; use crate::types::enums::{enum_ignored_names, is_enum_class_by_inheritance}; @@ -140,10 +142,11 @@ use crate::types::narrow::{ use crate::types::newtype::NewType; use crate::types::receivers; use crate::types::regex; -use crate::types::reified_infer::{self, ErasedTargetReason, ReifiedInferenceError}; +use crate::types::reified_infer; use crate::types::repeated_underscore::{ LoweredParameters, WrittenNames, lower_repeated_underscores, parameter_slots, }; +use crate::types::runtime_reads::{ReifiedGeneric, RuntimeReadSite}; use crate::types::set_theoretic::RecursivelyDefined; use crate::types::signatures::{CallableSignature, ReturnCallableTypeVarScope}; use crate::types::soundness::{ @@ -527,6 +530,12 @@ pub(super) struct TypeInferenceBuilder<'db, 'ast> { /// is a stub file but we're still in a non-deferred region. deferred_state: DeferredExpressionState, + /// basedpython: the state a specialization written as a value is itself evaluated in, while + /// its type arguments are inferred as deferred (see `infer_subscript_load`). what is a value + /// inside them, `Literal[E.A]` or the metadata of `Annotated[T, m]`, python still evaluates + /// where it is written, so it is inferred in this state instead + specialization_evaluation_state: Option, + /// For decorated function or class definitions, the type before applying decorators. undecorated_type: Option>, @@ -672,6 +681,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return_types_and_ranges: vec![], called_functions: FxIndexSet::default(), deferred_state: DeferredExpressionState::None, + specialization_evaluation_state: None, expressions: FxHashMap::default(), comparison_truthiness: FxHashMap::default(), expression_cache: None, @@ -1216,6 +1226,19 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { previous } + /// infers what is a value inside a specialization's type arguments in the state the + /// specialization itself is evaluated in, when those arguments are being read as deferred + fn infer_as_evaluated(&mut self, infer: impl FnOnce(&mut Self) -> T) -> T { + let Some(state) = self.specialization_evaluation_state.take() else { + return infer(self); + }; + let previous = self.replace_deferred_state(state); + let result = infer(self); + self.deferred_state = previous; + self.specialization_evaluation_state = Some(state); + result + } + /// Returns `true` if `expr` is a call to a known diagnostic function /// (e.g., `reveal_type` or `assert_type`) whose return value should not /// trigger the `unused-awaitable` lint. @@ -3438,8 +3461,13 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let Some(case_name) = self.index.case_name(NodeKey::from_node(identifier)) else { return; }; - if case_name_pattern_type(self.db(), self.program_environment(), case_name).is_none() { - captures.push((alternative, identifier, case_name)); + match context_sensitive::resolve_case_name( + self.db(), + self.program_environment(), + case_name, + ) { + Some(member) => self.record_context_sensitive_qualifier(member), + None => captures.push((alternative, identifier, case_name)), } }); for (alternative, identifier, case_name) in captures { @@ -4803,6 +4831,37 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } fn infer_assignment_deferred(&mut self, target: &ast::Expr, value: &'ast ast::Expr) { + // basedpython: the type expressions a typing construct takes as call arguments — + // `TypeVar("T", bound=F)` — are read as deferred, as an annotation is, so they may name + // a class the module binds further on: the transpiler writes such a name as the forward + // reference python reads lazily. the bases of `type("C", bases, ns)` are classes the + // call needs, so they are not + let deferred_form = if self.is_basedpython_file() + && let ast::Expr::Call(call) = value + && let callee = self + .try_expression_type(&call.func) + .unwrap_or_else(|| self.infer_expression(&call.func, TypeContext::default())) + && let Some(form) = CallTypeForm::of(self.db(), callee) + { + Some((form, call)) + } else { + None + }; + let previous_deferred_state = deferred_form + .is_some() + .then(|| self.replace_deferred_state(DeferredExpressionState::Deferred)); + self.infer_assignment_deferred_impl(target, value); + if let Some(previous) = previous_deferred_state { + self.deferred_state = previous; + } + if let Some((form, call)) = deferred_form { + for argument in form.type_expressions(&call.arguments) { + self.store_type_expression_flags(argument, TypeExpressionFlags::READ_DEFERRED); + } + } + } + + fn infer_assignment_deferred_impl(&mut self, target: &ast::Expr, value: &'ast ast::Expr) { let db = self.db(); let env = self.program_environment(); // Infer deferred bounds/constraints/defaults of a legacy TypeVar / ParamSpec / NewType, @@ -8756,6 +8815,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Type::inline_protocol(self.db(), env, members, packs.into_boxed_slice()) } + /// the class of the anonymous named tuple form `tuple`, which the parser flagged as one — + /// `is_anon_named_tuple` for a type, `is_anon_named_tuple_value` for a value. it is the only + /// way the checker makes an anonymous named tuple class, so whether a type is one is read + /// off its [`DynamicNamedTupleAnchor::Anonymous`] anchor fn synthesize_anon_named_tuple_class( &mut self, tuple: &ast::ExprTuple, @@ -8822,14 +8885,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let class_name = ruff_python_ast::name::Name::new(format!("_AnonNamedTuple_{truncated:08x}")); - // Anchor at the module-scope with a constant offset of 0 so structural - // identity (driven entirely by `spec`) determines the synthesized - // class. Different shapes produce different specs; identical shapes - // unify across the file. + // Anchor at the module-scope so structural identity (driven entirely + // by `spec`) determines the synthesized class. Different shapes produce + // different specs; identical shapes unify across the file. let module_scope = global_scope(db, db.program_file(self.file())); - let anchor = DynamicNamedTupleAnchor::ScopeOffset { + let anchor = DynamicNamedTupleAnchor::Anonymous { scope: module_scope, - offset: DynamicClassScopeOffset::Node(0), spec, }; @@ -8837,6 +8898,87 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Type::ClassLiteral(ClassLiteral::DynamicNamedTuple(nt)) } + /// every plain tuple literal in `value` this builder has read as an anonymous named + /// tuple, with the class it is built as + fn anon_named_tuple_constructions( + &self, + value: &ast::Expr, + ) -> Vec<(TextRange, DynamicNamedTupleLiteral<'db>)> { + struct Literals<'a, 'db, 'ast> { + builder: &'a TypeInferenceBuilder<'db, 'ast>, + found: Vec<(TextRange, DynamicNamedTupleLiteral<'db>)>, + } + impl<'b> ast::visitor::Visitor<'b> for Literals<'_, '_, '_> { + fn visit_expr(&mut self, expr: &'b ast::Expr) { + if let ast::Expr::Tuple(tuple) = expr + && !tuple.is_anon_named_tuple + && !tuple.is_anon_named_tuple_value + && let Some(ty) = self.builder.try_expression_type(expr) + && let Some((class, _)) = anon_named_tuple::anon_named_tuple_class( + self.builder.db(), + self.builder.program_environment(), + ty, + ) + { + self.found.push((tuple.range, class)); + } + ast::visitor::walk_expr(self, expr); + } + } + let mut literals = Literals { + builder: self, + found: Vec::new(), + }; + ast::visitor::Visitor::visit_expr(&mut literals, value); + literals.found + } + + /// whether each element of the plain tuple literal `tuple`, inferred against its + /// field's type, is assignable to it. inferred speculatively, so nothing is + /// recorded for the literal + fn anon_named_tuple_fields_accept( + &self, + tuple: &ast::ExprTuple, + spec: NamedTupleSpec<'db>, + ) -> bool { + let env = self.program_environment(); + let db = self.db(); + let mut speculative = self.speculate_without_diagnostics(); + tuple + .elts + .iter() + .zip(spec.fields(db).iter()) + .all(|(elt, field)| { + speculative + .infer_expression(elt, TypeContext::new(Some(field.ty))) + .is_assignable_to(db, env, field.ty) + }) + } + + /// report a plain tuple literal that more than one expected anonymous named + /// tuple accepts + fn report_ambiguous_anon_named_tuple( + &self, + tuple: &ast::ExprTuple, + accepting: &[(Type<'db>, NamedTupleSpec<'db>)], + ) { + let env = self.program_environment(); + let db = self.db(); + let Some(builder) = self.context.report_lint(&AMBIGUOUS_CONVERSION, tuple) else { + return; + }; + let mut diagnostic = + builder.into_diagnostic("More than one anonymous named tuple accepts this tuple"); + let names = accepting + .iter() + .map(|(target, _)| format!("`{}`", target.display(db, env))) + .join(", "); + diagnostic.info(format_args!( + "{names} all accept it, and a plain tuple does not say which one to build" + )); + diagnostic.help("Name the fields, or construct the one you want explicitly"); + } + fn infer_tuple_expression( &mut self, tuple: &ast::ExprTuple, @@ -8867,13 +9009,17 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // same coercion the plain-tuple spelling gets below: `b: P = (name="a")` // and `b: P = ("a",)` construct the one class the transpiler emits, whose // fields carry the *declared* types — so `b._replace(name="x")` is a call - // the runtime accepts - if let Some(target) = tcx - .annotation() - .and_then(|annotation| find_anon_nt_target(self.db(), env, annotation)) - && literal_instance.is_assignable_to(self.db(), env, target) + // the runtime accepts. with several that accept it, none is chosen: the + // literal one is still assignable to each + if let Some(annotation) = tcx.annotation() + && let [(target, _)] = + anon_named_tuple::expected_anon_named_tuples(db, env, annotation) + .into_iter() + .filter(|(target, _)| literal_instance.is_assignable_to(db, env, *target)) + .collect::>() + .as_slice() { - return target; + return *target; } return literal_instance; } @@ -8891,83 +9037,54 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } return Type::unknown(); } - // basedpython: a *plain* tuple literal in a position whose expected - // type (via bidirectional inference) is an anonymous-named-tuple - // instance gets inferred AS that instance — matching the transpiler's - // implicit constructor coercion. This makes `def f() -> (name: str, - // age: int): return ("a", 1)` type-check, since at runtime the - // returned value is wrapped as `_AnonNamedTuple_xxx("a", 1)`. - // Find an anon-NT instance candidate in the expected type. Accept the - // direct case (`x: anon-NT`) and any union member (`x: anon-NT | None`, - // `x: anon-NT | Other`, …). The first matching union member wins. A - // `type P = (name: str)` alias is resolved through, so the coercion - // follows what the annotation means rather than how it is spelled — - // which is also what the transpiler wraps on. - #[expect(clippy::items_after_statements, reason = "helper colocated with use")] - fn find_anon_nt_target<'db>( - db: &'db dyn crate::Db, - env: &ProgramEnvironment<'db>, - ty: Type<'db>, - ) -> Option> { - let ty = ty.resolve_type_alias(db); - if let Type::NominalInstance(instance) = ty - && let crate::types::class::ClassLiteral::DynamicNamedTuple(nt) = - instance.class(db, env).class_literal(db) - && nt.name(db).as_str().starts_with("_AnonNamedTuple_") - { - return Some(ty); - } - if let Type::Union(union) = ty { - for member in union.elements(db).iter().copied() { - if let Some(found) = find_anon_nt_target(db, env, member) { - return Some(found); - } - } - } - None - } - if let Some(target_instance) = tcx - .annotation() - .and_then(|a| find_anon_nt_target(self.db(), env, a)) - && let Type::NominalInstance(instance) = target_instance - && let crate::types::class::ClassLiteral::DynamicNamedTuple(nt) = - instance.class(self.db(), env).class_literal(self.db()) - && nt.name(self.db()).as_str().starts_with("_AnonNamedTuple_") + // basedpython: a *plain* tuple literal in a position whose expected type + // (via bidirectional inference) is an anonymous named tuple is inferred as + // an instance of it, and the transpiler constructs that class around it — + // so `take((1, 2))` with `def take(p: (a: int, b: int))` reads `p.a` off a + // value that has it. the transpiler finds the sites by this inferred type, + // so whatever position gives the literal its expected type is one. + // + // the expected type may allow several (`(a: int) | (b: str)`), or one alongside + // other arms (`(a: list[int]) | tuple[list[object]]`). those whose field count + // matches are candidates, and a candidate is built only where its fields accept + // the elements — otherwise the literal is inferred as a tuple, against the rest + // of the expected type. a literal more than one candidate accepts is reported + // rather than given to whichever the union lists first: the class decides which + // field names exist at runtime, and nothing in the literal says which. a `.py` + // file has no such construction, so a plain tuple stays a plain tuple there + let mut tcx = tcx; + if self.is_basedpython_file() + && !tuple.elts.iter().any(ast::Expr::is_starred_expr) + && let Some(annotation) = tcx.annotation() { - let spec = match nt.anchor(self.db()) { - crate::types::class::DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => { - Some(*spec) + let expected = anon_named_tuple::expected_anon_named_tuples(db, env, annotation); + let accepting: Vec<_> = expected + .iter() + .copied() + .filter(|(_, spec)| { + spec.fields(db).len() == tuple.elts.len() + && self.anon_named_tuple_fields_accept(tuple, *spec) + }) + .collect(); + match accepting.as_slice() { + [] if expected.is_empty() => {} + [] => { + tcx = TypeContext::new(anon_named_tuple::without_anon_named_tuples( + db, env, annotation, + )); } - _ => None, - }; - if let Some(spec) = spec { - let fields = spec.fields(self.db()); - if fields.len() == tuple.elts.len() - && tuple - .elts - .iter() - .all(|e| !matches!(e, ast::Expr::Starred(_))) - { - // type-check element-wise against the expected field - // types. when every element is assignable, return the - // anon-NT instance so attribute access works; otherwise - // build a plain heterogeneous tuple from the inferred - // types (each element has already been inferred, so we - // can't re-call `infer_expression` in a fall-through - // branch) - let mut elt_tys: Vec> = Vec::with_capacity(tuple.elts.len()); - let mut all_assignable = true; - for (elt, field) in tuple.elts.iter().zip(fields.iter()) { - let elt_ty = self.infer_expression(elt, TypeContext::new(Some(field.ty))); - if !elt_ty.is_assignable_to(self.db(), env, field.ty) { - all_assignable = false; - } - elt_tys.push(elt_ty); + [(target, spec)] => { + for (elt, field) in tuple.elts.iter().zip(spec.fields(db).iter()) { + self.infer_expression(elt, TypeContext::new(Some(field.ty))); } - if all_assignable { - return target_instance; + return *target; + } + ambiguous => { + self.report_ambiguous_anon_named_tuple(tuple, ambiguous); + for elt in &tuple.elts { + self.infer_expression(elt, TypeContext::default()); } - return Type::heterogeneous_tuple(self.db(), env, elt_tys); + return Type::unknown(); } } } @@ -11126,7 +11243,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.file(), value_ty, alias, - Some(type_arg), + Some(crate::types::reified_infer::WrittenTarget::unchecked( + type_arg + )), ), crate::types::reified_infer::ParametricIsPlan::TokenEq(_) | crate::types::reified_infer::ParametricIsPlan::Fold(_) @@ -11164,7 +11283,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { )); match runtime_check_target(db, env, self.file(), target) { Some(shallow) => diagnostic.info(format_args!( - "a runtime check can only test `{shallow}`; the type arguments are assumed" + "a runtime check can only test `{}`; the type arguments are assumed", + shallow.source() )), None => diagnostic.info("the type arguments are assumed"), } @@ -11409,6 +11529,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // claim; the plain `cast` never reaches a check in the first place if cast_kind.verifies_at_runtime() { self.report_erased_cast_argument(type_arg, value_ty, target); + self.context + .record_runtime_read_check(RuntimeReadSite::CheckedCast { + value: value_arg.node_index().load(), + target: type_arg.node_index().load(), + }); } // an unsound plain `cast` is already the sharper report — a // non-overlapping one would only repeat that the value is not the @@ -12903,8 +13028,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .keywords .iter() .any(|keyword| keyword.arg.is_none()); - let inference_failure = if has_unpacked_arguments { - Some(None) + let inference = if has_unpacked_arguments { + Err(None) } else { reified_infer::inferred_call_type_arguments( self.db(), @@ -12913,45 +13038,30 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { callable_type, function, &call_arguments, + None, ) - .err() - .map(Some) + .map_err(Some) }; - if let Some(failure) = inference_failure - && let Some(builder) = self - .context - .report_lint(&UNSPECIALIZED_REIFIED_GENERIC, call_expression) - { - let name = function.name(self.db()); - let mut diagnostic = builder.into_diagnostic(format_args!( - "Cannot call reified generic function `{name}` \ - without explicit specialization" - )); - match failure { - None => diagnostic.info(format_args!( - "the inferred specialization cannot be injected through \ - unpacked arguments — specialize with `{name}[...]`" - )), - Some(ReifiedInferenceError::Unsolved(parameter)) => { - diagnostic.info(format_args!( - "reified type parameter `{parameter}` cannot be inferred \ - from the arguments — specialize with `{name}[...]`" - )); - } - Some(ReifiedInferenceError::Unspellable(parameter, ty)) => { - diagnostic.info(format_args!( - "inferred type `{}` for type parameter `{parameter}` has \ - no runtime spelling — specialize with `{name}[...]`", - ty.display(self.db(), env), - )); - } - Some(ReifiedInferenceError::NoBinding) => { - diagnostic.info(format_args!( - "the specialization cannot be inferred from this call — \ - specialize with `{name}[...]`" - )); - } + let name = function.name(self.db()); + match inference { + Ok(arguments) => { + self.context + .record_runtime_read_check(RuntimeReadSite::Reified { + call: call_expression.node_index().load(), + generic: ReifiedGeneric::Function(name.clone()), + reads: arguments + .iter() + .flat_map(reified_infer::TypeArgument::global_reads) + .cloned() + .collect(), + }); } + Err(failure) => report_unspecialized_reified_call( + &self.context, + call_expression.range(), + name, + failure, + ), } } @@ -13111,39 +13221,28 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return; } let env = self.program_environment(); - let Some(failure) = reified_infer::reified_construction_error( + match reified_infer::constructor_specialization( db, env, self.file(), class_literal, return_ty, - ) else { - return; - }; - let Some(builder) = self - .context - .report_lint(&UNSPECIALIZED_REIFIED_GENERIC, call_expression) - else { - return; - }; - let name = class.name(db); - let mut diagnostic = builder.into_diagnostic(format_args!( - "Cannot construct reified generic class `{name}` without a specialization" - )); - match failure { - ReifiedInferenceError::Unsolved(parameter) => diagnostic.info(format_args!( - "reified type parameter `{parameter}` cannot be inferred here — \ - construct a specialization, as in `{name}[int](...)`" - )), - ReifiedInferenceError::Unspellable(parameter, ty) => diagnostic.info(format_args!( - "inferred type `{}` for type parameter `{parameter}` has no runtime \ - spelling — construct a specialization, as in `{name}[int](...)`", - ty.display(db, env), - )), - ReifiedInferenceError::NoBinding => diagnostic.info(format_args!( - "the specialization cannot be inferred from this call — construct \ - one, as in `{name}[int](...)`" - )), + None, + ) { + Ok(spelling) => { + self.context + .record_runtime_read_check(RuntimeReadSite::Reified { + call: call_expression.node_index().load(), + generic: ReifiedGeneric::Class(class.name(db).clone()), + reads: spelling.global_reads().cloned().collect(), + }); + } + Err(failure) => report_unspecialized_reified_construction( + &self.context, + call_expression.range(), + class.name(db), + failure, + ), } } @@ -13928,6 +14027,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Some(name_node), ) { + if let receivers::ImplicitReceiverName::ExtensionMember { resolution, .. } = &resolved { + self.record_extension_resolution(resolution); + } return resolved.ty(); } @@ -14051,6 +14153,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { symbol_name, ) { + self.record_context_sensitive_qualifier(member); Place::bound(member.ty).into() } else { Place::Undefined.into() @@ -14067,6 +14170,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { && let Some(case_name) = self.index.case_name(NodeKey::from_node(name_node)) && let Some(member) = context_sensitive::resolve_case_name(db, env, case_name) { + self.record_context_sensitive_qualifier(member); Place::bound(member.ty).into() } else { Place::Undefined.into() @@ -14247,6 +14351,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let env = self.program_environment(); let is_class_body_global_fallback = source.is_class_body_global_fallback(); + self.record_whole_scope_read(&source.kind); + let place = match source.kind { PlaceLoadSourceKind::Bindings(bindings) => { let mut place = place_from_bindings_with_reachability_cache( @@ -14316,6 +14422,38 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + /// basedpython: records a read the semantic index has no record of — one that reads every + /// binding a scope makes rather than the bindings live at the reading expression. a name in + /// an annotation is evaluated lazily, and a name declared `global` is read from the module. + /// a nested scope's read of an enclosing one is not among them: the index records that + /// itself + fn record_whole_scope_read(&self, source: &PlaceLoadSourceKind<'db>) { + if !self.is_basedpython_file() { + return; + } + let db = self.db(); + match source { + PlaceLoadSourceKind::Bindings(bindings) if self.is_deferred() => { + for binding in bindings.clone() { + if let Some(definition) = binding.binding.definition() + && let Some(symbol) = definition.place(db).as_symbol() + { + self.context + .record_scope_read(definition.file_scope(db), symbol); + } + } + } + PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::ExplicitGlobalSymbol { + file, + name, + }) if *file == self.program_file() => { + self.context + .record_scope_read_of_name(ty_python_core::global_scope(db, *file), name); + } + _ => {} + } + } + /// Applies ty's convenience fallback for an unimported `reveal_type`. fn infer_unimported_reveal_type_fallback( &self, @@ -15044,6 +15182,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if let Some(resolution) = extensions::resolve_extension_member(db, env, self.file(), value_type, &attr.id) { + self.record_extension_resolution(&resolution); if let Some(other) = resolution.ambiguous_with && let Some(builder) = self .context @@ -15065,7 +15204,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // extensions, this never shadows a declared member if self.is_basedpython_file() && fallback_place.place.is_undefined() - && let Some(bound) = receivers::resolve_receiver_attribute( + && let Some((declaring_scope, bound)) = receivers::resolve_receiver_attribute_in_scope( db, env, self.file(), @@ -15074,6 +15213,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { &attr.id, ) { + // the access reads the name, which no name expression spells + self.context + .record_scope_read_of_name(declaring_scope, &attr.id); fallback_place = Place::bound(bound).into(); } @@ -15758,11 +15900,30 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return None; } let db = self.db(); - extensions::unary_extension_operator(db, env, self.file(), op, operand)?.return_type( - db, - env, - &CallArguments::none(), - ) + let operator = extensions::unary_extension_operator(db, env, self.file(), op, operand)?; + let returned = operator.return_type(db, env, &CallArguments::none())?; + self.record_extension_resolution(&operator.resolution); + Some(returned) + } + + /// basedpython: records the binding a context-sensitive name's lowering reads its + /// qualifier through — `Red` is written `Color.Red`, which reads `Color` although the + /// source never spells it + fn record_context_sensitive_qualifier( + &self, + member: context_sensitive::ContextSensitiveMember<'db>, + ) { + if let Some((scope, symbol)) = member.qualifier_binding() { + self.context.record_scope_read(scope, symbol); + } + } + + /// basedpython: records every extension an extension-member resolution relied on, so + /// the imports that make them applicable count as used + fn record_extension_resolution(&self, resolution: &extensions::ExtensionMemberResolution<'db>) { + for extension in resolution.relied_on() { + self.context.record_extension_use(extension); + } } /// basedpython: the type a binary operator evaluates to when an applicable @@ -15781,7 +15942,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let operator = extensions::binary_extension_operator(db, env, self.file(), left, op, right)?; let argument = if operator.reflected { left } else { right }; - operator.return_type(db, env, &CallArguments::positional([argument])) + let returned = operator.return_type(db, env, &CallArguments::positional([argument]))?; + self.record_extension_resolution(&operator.resolution); + Some(returned) } /// basedpython: the type a comparison evaluates to when an applicable @@ -15803,6 +15966,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { extensions::comparison_extension_operator(db, env, self.file(), left, op, right)?; let argument = if operator.reflected { left } else { right }; let returned = operator.return_type(db, env, &CallArguments::positional([argument]))?; + self.record_extension_resolution(&operator.resolution); Some(match op { ast::CmpOp::In | ast::CmpOp::NotIn => KnownClass::Bool.to_instance(db, env), _ => returned, @@ -16575,16 +16739,27 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.file(), left_ty, target, - Some(right), + Some(reified_infer::WrittenTarget::unchecked(right)), ); - if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = plan { + for module in plan.conformance_modules() { + self.context.record_extension_module_use(module); + } + if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = &plan { let source = ruff_db::source::source_text(db, self.file()); - self.report_erased_type_check( + report_erased_type_check( + &self.context, TextRange::new(left.start(), right.end()), &source[right.range()], - reason, + reason.clone(), ); } else { + // whether the names the test is written with hold where it runs is asked once + // the module is inferred + self.context + .record_runtime_read_check(RuntimeReadSite::TypeTest { + value: left.node_index().load(), + target: right.node_index().load(), + }); // a target with no runtime test has already been rejected, and // saying it can also never hold would pile a second report on one // mistake @@ -16645,146 +16820,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { )); } - fn report_erased_type_check( - &self, - primary: TextRange, - target: &str, - reason: ErasedTargetReason, - ) { - let Some(builder) = self.context.report_lint(&ERASED_TYPE_CHECK, primary) else { - return; - }; - match reason { - ErasedTargetReason::Protocol => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: a protocol's instances don't \ - record which specialization they satisfy" - )); - diagnostic.info(format_args!( - "an instance's `__orig_class__` names its concrete class, never the protocol, \ - and a structural `isinstance` check can't see type arguments" - )); - diagnostic.info(format_args!( - "reify the type parameter (`def f[T](x: T)`), or test against a concrete class \ - that records the specialization on its instances or across its mro" - )); - } - ErasedTargetReason::NotSubscriptable => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: the target class cannot be \ - subscripted" - )); - diagnostic.info(format_args!( - "the class has no `__class_getitem__` on this python version, so the test's \ - runtime check would raise `TypeError` evaluating `{target}`" - )); - diagnostic.info(format_args!( - "drop the type arguments and test against the bare class, which is all the \ - runtime records anyway" - )); - } - ErasedTargetReason::BuiltinCollection => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: a builtin collection's instances \ - don't record their type arguments" - )); - diagnostic.info(format_args!( - "test against a subclass that fixes the arguments (`class B(list[int])`), \ - whose `__orig_bases__` the probe can unwind" - )); - } - ErasedTargetReason::Dynamic => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: `{target}` admits every value, \ - so there is nothing for the test to look for" - )); - diagnostic.info(format_args!( - "a test that can only answer `True` narrows nothing; drop it, or name the \ - type the value is expected to have" - )); - } - ErasedTargetReason::Callable => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: a callable's parameter and \ - return types are not recorded on the value" - )); - diagnostic.info(format_args!( - "the runtime can see that a value is callable and nothing more, so the \ - signature would be assumed rather than checked" - )); - diagnostic.info(format_args!( - "test against a bare `Callable`, which asks only what the runtime records, \ - or take the value's type from the call site" - )); - } - ErasedTargetReason::TypedDict => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: a `TypedDict`'s instances are \ - plain dicts" - )); - diagnostic.info(format_args!( - "the test could only ask whether the value is a `dict`, assuming every key \ - and value type `{target}` declares" - )); - diagnostic.info(format_args!( - "`isinstance` refuses a `TypedDict` for the same reason" - )); - } - ErasedTargetReason::NonRuntimeCheckableProtocol => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: `{target}` has a member with no \ - runtime spelling" - )); - diagnostic.info(format_args!( - "a member whose type the emitted python cannot name leaves nothing to check \ - the value's own annotation against" - )); - diagnostic.info(format_args!( - "decorate `{target}` with `@typing.runtime_checkable` for the check python \ - itself performs, which asks only that the members are present" - )); - } - ErasedTargetReason::Intersection => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: an intersection has no single \ - runtime form" - )); - diagnostic.info(format_args!( - "test each part separately and join the tests with `and`" - )); - } - ErasedTargetReason::Subclass => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: `issubclass` cannot take what \ - `{target}` is parameterized by" - )); - diagnostic.info(format_args!( - "the test can see that the value is a class and nothing more; name the class \ - it must be a subclass of" - )); - } - ErasedTargetReason::UncomparableLiteral => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: equality does not decide \ - membership of `{target}`" - )); - diagnostic.info(format_args!( - "`0.0 == -0.0` and `nan != nan`, so a `float` or `complex` literal names a \ - type the runtime cannot be asked about; `LiteralString` is a property of how \ - a value was written rather than of the value" - )); - } - ErasedTargetReason::Unspellable => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: `{target}` has no runtime form" - )); - diagnostic.info(format_args!( - "the emitted python has no expression to evaluate for this target" - )); - } - } - } - /// `reification` names the owner of this list, which decides whether a /// basedpython `reified` parameter — or a declared variance — can take /// effect; one that cannot is reported here rather than silently dropped. @@ -16886,6 +16921,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { reachability_cache: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, called_functions, index: _, region: _, @@ -16958,6 +16994,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { slice_materialization: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, index: _, region: _, } = self; @@ -17086,6 +17123,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { discards_dict_key_assignments: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, index: _, region: _, cycle_recovery: _, @@ -17150,6 +17188,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { slice_materialization: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, index: _, region: _, return_types_and_ranges: _, @@ -17313,6 +17352,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { slice_materialization: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, called_functions: _, index: _, region: _, @@ -17370,6 +17410,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { index, cycle_recovery, deferred_state, + specialization_evaluation_state, typevar_binding_context, ref expression_cache, ref reachability_cache, @@ -17421,6 +17462,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // Ensure the speculative builder has the same inference context as the current one. builder.cycle_recovery = cycle_recovery; builder.deferred_state = deferred_state; + builder.specialization_evaluation_state = specialization_evaluation_state; builder.typevar_binding_context = typevar_binding_context; builder.context.inference_flags = self.inference_flags(); builder.expression_cache.clone_from(expression_cache); @@ -17482,6 +17524,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { reachability_cache: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, called_functions, index: _, region: _, @@ -17805,6 +17848,19 @@ impl<'db, 'ast, 'infer> MultiInferenceGuard<'db, 'ast, 'infer> { (self.infer_expr)(&mut builder.speculate_without_diagnostics(), tcx) } + /// infer the expression silently with `tcx`, and read the result off the speculative + /// builder it was inferred in. the context is not remembered as the last one tried + fn infer_silent_with( + &mut self, + builder: &mut TypeInferenceBuilder<'db, 'ast>, + tcx: TypeContext<'db>, + inspect: impl FnOnce(&TypeInferenceBuilder<'db, 'ast>) -> R, + ) -> R { + let mut speculative = builder.speculate_without_diagnostics(); + (self.infer_expr)(&mut speculative, tcx); + inspect(&speculative) + } + fn last_tcx(&self) -> TypeContext<'db> { self.last_tcx.unwrap_or_default() } diff --git a/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs b/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs index 220d6a65c0..9ad4932a5e 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs @@ -1,5 +1,7 @@ +use itertools::Itertools; use ruff_python_ast as ast; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextRange}; +use rustc_hash::FxHashMap; use super::{ArgumentsIter, MultiInferenceGuard, TypeInferenceBuilder}; use crate::place::{DefinedPlace, Place, PlaceAndQualifiers}; @@ -9,7 +11,7 @@ use crate::types::attribute_write::{ ProtocolMemberWriteRequirement, attribute_write_requirement, property_setter_returns_never, }; use crate::types::call::{Bindings, CallArguments, CallDiagnosticOverride, CallError}; -use crate::types::class::FrozenDataclassDispatch; +use crate::types::class::{DynamicNamedTupleLiteral, FrozenDataclassDispatch}; use crate::types::dedicated::pydantic; use crate::types::diagnostic::{ DEPRECATED, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, MISSING_SLOT, UNRESOLVED_ATTRIBUTE, @@ -19,7 +21,7 @@ use crate::types::diagnostic::{ use crate::types::safe_variance::private_member_view; use crate::types::{ CallDunderError, DisplaySettings, MemberLookupPolicy, PropertyDeprecations, Type, TypeContext, - TypeQualifiers, + TypeQualifiers, UnionType, }; use crate::{Db, ProgramEnvironment}; @@ -104,6 +106,11 @@ enum AssignmentAttributeWriteDiagnostic<'db> { object_ty: Type<'db>, value_ty: Type<'db>, }, + /// basedpython: the elements of a union receiver would build different anonymous named + /// tuples from a plain tuple literal in the value + DifferentAnonNamedTuples { + object_ty: Type<'db>, + }, CannotAssign, CannotAssignToClassVar, TerminalSetAttr { @@ -252,6 +259,36 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { } } + /// the context the assigned value is recorded with, when the write is checked against + /// `write_type`. + /// + /// the check re-infers the value with `write_type` as its context, so in a `.py` file the + /// recorded type only ever loses precision by leaving it out. in a `.by` file what a + /// literal *is* can depend on its context — a plain tuple literal written where an + /// anonymous named tuple is declared is built as one — so the value is recorded as what + /// it was checked as, which is what the transpiler builds + fn commit_context(&self, write_type: Option>) -> TypeContext<'db> { + if self.builder.is_basedpython_file() { + TypeContext::new(write_type) + } else { + TypeContext::default() + } + } + + /// the anonymous named tuple each plain tuple literal in the value is built as, inferred + /// against `write_type` + fn constructions_against( + &mut self, + write_type: Type<'db>, + ) -> Vec<(TextRange, DynamicNamedTupleLiteral<'db>)> { + let value = self.value; + self.infer_value_ty.infer_silent_with( + self.builder, + TypeContext::new(Some(write_type)), + |speculative| speculative.anon_named_tuple_constructions(value), + ) + } + /// Infer the value again using the context that succeeded. /// /// The earlier inference was only a trial, so its result was not saved. @@ -334,8 +371,53 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { object_ty, element_tys, } => { - let value_ty = self.infer_value(TypeContext::default(), emit_diagnostics); let attribute = self.attribute; + let by = self.builder.is_basedpython_file(); + // each element's declaration, where it has one. in a `.by` file the value is + // recorded against the declaration the elements share, if they share one + let write_types: Vec>> = if by { + element_tys + .iter() + .map(|ty| { + attribute_write_requirement(db, env, *ty, attribute).write_type(db, env) + }) + .collect() + } else { + Vec::new() + }; + // one value reaches whichever element the receiver is, and a plain tuple + // literal in it is built as the anonymous named tuple its expected type + // names. each element is still checked against its own declaration, so where + // the declarations differ, no two elements may build different classes from + // one literal; the value is recorded against the declarations of those that + // build one, which then builds it. a class built where another element builds + // none is one that element accepts too: it is a subtype of the plain tuple. + // where they share a declaration, it is recorded against that one + let shared = write_types.iter().all_equal_value().ok().copied().flatten(); + let mut builds_apart = false; + let tcx = if shared.is_some() { + self.commit_context(shared) + } else { + let mut built = FxHashMap::default(); + let mut building = Vec::new(); + for write_type in write_types.iter().flatten() { + let constructions = self.constructions_against(*write_type); + if !constructions.is_empty() { + building.push(*write_type); + } + for (range, class) in constructions { + builds_apart |= built + .insert(range, class) + .is_some_and(|earlier| earlier != class); + } + } + if builds_apart || building.is_empty() { + TypeContext::default() + } else { + self.commit_context(Some(UnionType::from_elements(db, env, building))) + } + }; + let value_ty = self.infer_value(tcx, emit_diagnostics); let mut requirements = element_tys .iter() .map(|ty| attribute_write_requirement(db, env, *ty, attribute)); @@ -364,6 +446,16 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { ) }); } + if builds_apart { + if emit_diagnostics { + self.report( + AssignmentAttributeWriteDiagnostic::DifferentAnonNamedTuples { + object_ty: *object_ty, + }, + ); + } + return false; + } if valid { self.validate_composite_final_assignment(*object_ty, emit_diagnostics); } else if emit_diagnostics { @@ -545,13 +637,14 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { } else { Vec::new() }; - if let Some((range, repair)) = conversions.first() { + if let Some(((range, repair), rest)) = conversions.split_first() { if emit_diagnostics { - crate::types::conversions::report_ambiguous_conversion( - &self.builder.context, - *range, - repair, - ); + crate::types::conversions::accept_conversion(&self.builder.context, *range, repair); + } else { + crate::types::conversions::record_conversion_uses(&self.builder.context, repair); + } + for (_, repair) in rest { + crate::types::conversions::record_conversion_uses(&self.builder.context, repair); } return true; } @@ -626,7 +719,10 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { ) { self.infer_and_try_call_setattr(setattr_receiver, emit_diagnostics) } else { - let value_ty = self.infer_value(TypeContext::default(), emit_diagnostics); + let value_ty = self.infer_value( + self.commit_context(member.write_type(db, env, object_ty)), + emit_diagnostics, + ); let setattr_result = setattr_receiver.try_call_dunder_with_policy( db, env, @@ -771,7 +867,10 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { self.infer_value(TypeContext::default(), emit_diagnostics); return false; } - let value_ty = self.infer_value(TypeContext::default(), emit_diagnostics); + let value_ty = self.infer_value( + self.commit_context(member.write_type(db, env, object_ty)), + emit_diagnostics, + ); let member_valid = self.evaluate_explicit_member(object_ty, member, value_ty, emit_diagnostics); if let Some(fallback) = fallback { @@ -1071,6 +1170,25 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { )); } } + AssignmentAttributeWriteDiagnostic::DifferentAnonNamedTuples { object_ty } => { + if let Some(builder) = self + .builder + .context + .report_lint(&INVALID_ASSIGNMENT, self.target) + { + let mut diagnostic = builder.into_diagnostic(format_args!( + "The members of `{}` would build different anonymous named tuples \ + from this value for attribute `{}`", + object_ty.display(db, env), + self.attribute, + )); + diagnostic.info( + "a plain tuple literal is built as the anonymous named tuple its \ + declaration expects, and one value reaches whichever member the \ + receiver is", + ); + } + } AssignmentAttributeWriteDiagnostic::CannotAssign => { if let Some(builder) = self .builder diff --git a/crates/ty_python_semantic/src/types/infer/builder/class.rs b/crates/ty_python_semantic/src/types/infer/builder/class.rs index fb8e6c5c50..50a7355ea6 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/class.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/class.rs @@ -2,6 +2,7 @@ use crate::Db; use crate::ProgramEnvironment; use crate::reified::{UnansweredReason, reified_class_reads}; use crate::types::diagnostic::{INVALID_VARIANCE_DECLARATION, REIFIED_WITHOUT_RECEIVER}; +use crate::types::infer::TypeExpressionFlags; use crate::types::{ CallArguments, ClassLiteralFlags, DataclassFlags, DataclassParams, KnownClass, KnownInstanceType, SpecialFormType, StaticClassLiteral, SubclassOfType, Type, TypeContext, @@ -115,8 +116,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { self.report_reified_class_errors(class); if class.arguments.is_some() { - let defer_class_args = self.in_stub() || self.is_basedpython_file(); - let previous_deferred_state = self.replace_deferred_state(defer_class_args.into()); + let defer_bases = self.reads_bases_deferred(class); + let previous_deferred_state = self.replace_deferred_state(defer_bases.into()); // PEP 695 class headers are inferred in the type-parameter scope, before the completed // class type is available. Infer the bases first because `extra_items=T` is an @@ -125,8 +126,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { let mut is_typed_dict = false; for base in class.bases() { - let ty = if let Some(ty) = - self.infer_parameter_shape_class_base(base, defer_class_args) + let ty = if let Some(ty) = self.infer_parameter_shape_class_base(base, defer_bases) { ty } else if let ast::Expr::Starred(starred) = base { @@ -144,9 +144,10 @@ impl<'db> TypeInferenceBuilder<'db, '_> { }; } + self.replace_deferred_state(self.in_stub().into()); for keyword in class.keywords() { if is_typed_dict && keyword.arg.as_deref() == Some("extra_items") { - self.infer_extra_items_kwarg(&keyword.value); + self.infer_typed_dict_extra_items(&keyword.value); } else { self.infer_expression(&keyword.value, TypeContext::default()); } @@ -527,10 +528,9 @@ impl<'db> TypeInferenceBuilder<'db, '_> { // if there are type parameters, then the keywords and bases are within that scope // and we don't need to run inference here if type_params.is_none() { - // In stub files (and basedpython files, where self-refs are auto-quoted by the - // transpiler), keyword values may reference names that are defined later. - let defer_class_args = self.in_stub() || self.is_basedpython_file(); - let previous_deferred_state = self.replace_deferred_state(defer_class_args.into()); + // a keyword is an object the class is built with, read where it is written as a + // base is, except in a stub, which is never run + let previous_deferred_state = self.replace_deferred_state(self.in_stub().into()); for keyword in class_node.keywords() { if keyword.arg.as_deref() != Some("extra_items") { self.infer_expression(&keyword.value, TypeContext::default()); @@ -539,7 +539,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { self.deferred_state = previous_deferred_state; // Inference of bases deferred in stubs/basedpython, or if any are string literals. - if defer_class_args + if self.in_stub() + || self.is_basedpython_file() || class_node .bases() .iter() @@ -615,13 +616,39 @@ impl<'db> TypeInferenceBuilder<'db, '_> { Some(base_ty) } + /// a `TypedDict`'s `extra_items=`, which is a type expression. in a basedpython file it is + /// read as deferred, as an annotation is, though python evaluates it as the class is built, + /// so the transpiler writes a forward reference in it as one python reads lazily + fn infer_typed_dict_extra_items(&mut self, value: &ast::Expr) { + let previous_deferred_state = + self.replace_deferred_state((self.in_stub() || self.is_basedpython_file()).into()); + self.infer_extra_items_kwarg(value); + self.deferred_state = previous_deferred_state; + if self.is_basedpython_file() { + self.store_type_expression_flags(value, TypeExpressionFlags::READ_DEFERRED); + } + } + + /// whether a class's bases are read as deferred rather than where they are written + /// + /// a base is the object the class is built on, so outside a stub it is read where it is + /// written, and a name not yet bound there is unresolved, as python's `NameError` for it is + /// + /// a basedpython base's type arguments are deferred all the same, as every specialization + /// written as a value is — see `infer_subscript_load`. the protocols a basedpython extension + /// lists (`extension str(Show)`) are not bases: the conformance check reads them, and reports + /// one declared below the extension itself, as `invalid-conformance` + fn reads_bases_deferred(&self, class: &ast::StmtClassDef) -> bool { + self.in_stub() || class.is_extension() + } + pub(super) fn infer_class_deferred( &mut self, definition: Definition<'db>, class: &ast::StmtClassDef, ) { let previous_typevar_binding_context = self.typevar_binding_context.replace(definition); - let defer_class_args = self.in_stub() || self.is_basedpython_file(); + let defer_bases = self.reads_bases_deferred(class); // resolve bases with `IN_TYPE_EXPRESSION` set so the basedpython // `dynamic` keyword resolves to `Any` in a base list (`class C(dynamic)`), // matching the transpiler's `dynamic → Any` rewrite. only `dynamic` name @@ -632,12 +659,12 @@ impl<'db> TypeInferenceBuilder<'db, '_> { .replace(InferenceFlags::IN_TYPE_EXPRESSION, true); for base in class.bases() { if self - .infer_parameter_shape_class_base(base, defer_class_args) + .infer_parameter_shape_class_base(base, defer_bases) .is_some() { continue; } - if defer_class_args { + if defer_bases { self.infer_expression_with_state( base, TypeContext::default(), @@ -658,8 +685,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { if original_class_type(self.db(), definition) .is_some_and(|class_literal| class_literal.is_typed_dict(self.db())) { - self.infer_extra_items_kwarg(&extra_items_keyword.value); - } else if defer_class_args { + self.infer_typed_dict_extra_items(&extra_items_keyword.value); + } else if self.in_stub() { self.infer_expression_with_state( &extra_items_keyword.value, TypeContext::default(), diff --git a/crates/ty_python_semantic/src/types/infer/builder/conditions.rs b/crates/ty_python_semantic/src/types/infer/builder/conditions.rs index 640049dd0b..9339e6193c 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/conditions.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/conditions.rs @@ -220,6 +220,11 @@ fn definition_is_environment_derived_inner<'db>( return false; } let module = parsed_module(db, definition.python_file(db)).load(db); + // the declaration of `typing.TYPE_CHECKING` is the fact itself — typeshed declares it + // without a value — so a name imported from it under another name stands for it too + if crate::place::declares_typing_type_checking(db, definition) { + return true; + } // an import is followed by name resolution before it ever reaches here, so what is left is // the value a binding was written with let value: &ast::Expr = match definition.kind(db) { diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index dde10d8215..e61d4142aa 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -338,6 +338,25 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { |call| self.call_solution(call), ); + // basedpython: the guard a `raises` function is lowered behind names the declared + // exceptions as the module reaches them, so it reads the bindings it names them by + if function.raises.is_some() && self.is_basedpython_file() { + let db = self.db(); + let function_ty = + crate::types::binding_type(db, self.index.expect_single_definition(function)); + if let Some(target) = crate::types::exceptions::declared_raises_runtime_target( + db, + self.program_environment(), + self.file(), + function_ty, + ) { + self.context.record_spelling(&target.ceiling); + if let Some(resolved) = &target.resolved { + self.context.record_spelling(&resolved.expression); + } + } + } + // basedpython: a trailing-lambda block in a loop that captures a loop // variable is a late-binding trap unless its callee confines it // (`local` / `once`) — the type-aware complement to ruff's `B023` @@ -585,7 +604,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ), ) .is_some_and(|repair| { - crate::types::conversions::report_ambiguous_conversion( + crate::types::conversions::accept_conversion( &self.context, return_statement.range, &repair, diff --git a/crates/ty_python_semantic/src/types/infer/builder/subscript.rs b/crates/ty_python_semantic/src/types/infer/builder/subscript.rs index ec2f58c6bb..d0c0d371c8 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/subscript.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/subscript.rs @@ -5,7 +5,7 @@ use ruff_python_ast::{self as ast, ArgOrKeyword, ExprContext, ParameterBorrow}; use ruff_text_size::Ranged; use ty_module_resolver::file_to_module; -use super::TypeInferenceBuilder; +use super::{DeferredExpressionState, TypeInferenceBuilder}; use crate::place::{DefinedPlace, Definedness, Place}; use crate::types::call::Argument; use crate::types::call::CallErrorKind; @@ -483,13 +483,27 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // let the `None` ride out to the end of the chain let (value_ty, in_chain) = self.basedpython_chain_receiver(&subscript.value, value_ty); - // If we have an implicit type alias like `MyList = list[T]`, and if `MyList` is being - // used in another implicit type alias like `Numbers = MyList[int]`, then we infer the - // right hand side as a value expression, and need to handle the specialization here. - if value_ty.is_generic_alias() { - return Ok(self.infer_explicit_type_alias_specialization(subscript, value_ty, false)); + // basedpython: the type arguments of a specialization written as a value — `list[F]()`, + // `Alias = dict[str, F]`, `Foo[F]` among a class's bases — are read as deferred, as an + // annotation is, so they may name a class the module binds further on: the transpiler + // writes such a name as the forward reference python reads lazily + let defer = self.is_basedpython_file() + && crate::types::subscript::defers_type_arguments(self.db(), value_ty); + let previous_states = defer.then(|| { + let evaluated = self.replace_deferred_state(DeferredExpressionState::Deferred); + let previous_evaluation = self.specialization_evaluation_state.replace(evaluated); + (evaluated, previous_evaluation) + }); + let loaded = self.infer_subscript_load_specialization(subscript, value_ty, tcx); + if let Some((evaluated, previous_evaluation)) = previous_states { + self.deferred_state = evaluated; + self.specialization_evaluation_state = previous_evaluation; + self.store_type_expression_flags(&*subscript.slice, TypeExpressionFlags::READ_DEFERRED); } - let loaded = self.infer_subscript_load_impl(value_ty, subscript, tcx); + let loaded = match loaded { + Ok(loaded) => loaded, + Err(specialized) => return Ok(specialized), + }; let ty = loaded.unwrap_or_else(|recovery_ty| recovery_ty); // `m[1]` goes through `Match.__getitem__`, whose stub can only say @@ -502,6 +516,23 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { loaded.map(|_| ty).map_err(|_| ty) } + /// the subscript `value_ty[...]` loads: `Err` with the result of re-specializing an implicit + /// type alias, which is final, or `Ok` with the ordinary subscript's + fn infer_subscript_load_specialization( + &mut self, + subscript: &ast::ExprSubscript, + value_ty: Type<'db>, + tcx: TypeContext<'db>, + ) -> Result, Type<'db>>, Type<'db>> { + // If we have an implicit type alias like `MyList = list[T]`, and if `MyList` is being + // used in another implicit type alias like `Numbers = MyList[int]`, then we infer the + // right hand side as a value expression, and need to handle the specialization here. + if value_ty.is_generic_alias() { + return Err(self.infer_explicit_type_alias_specialization(subscript, value_ty, false)); + } + Ok(self.infer_subscript_load_impl(value_ty, subscript, tcx)) + } + /// The type of `m[key]` for a `re.Match` whose capture groups are known. fn refine_regex_subscript( &self, @@ -936,12 +967,26 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .replace(InferenceFlags::DISABLE_INT_FLOAT_SPECIAL_CASE, true) }); - let result = self.infer_explicit_callable_specialization( - subscript, - value_ty, - generic_context, - specialize, - ); + // a reified class keeps its type arguments as runtime values, so they are read where + // they are written even inside a specialization whose type arguments are deferred: + // `list[Box[F]]` hands `Box` the object `F`, not a forward reference to it + let result = if crate::types::subscript::defers_type_arguments(db, value_ty) { + self.infer_explicit_callable_specialization( + subscript, + value_ty, + generic_context, + specialize, + ) + } else { + self.infer_as_evaluated(|builder| { + builder.infer_explicit_callable_specialization( + subscript, + value_ty, + generic_context, + specialize, + ) + }) + }; if let Some(previously_disabled_int_float_special_case) = previously_disabled_int_float_special_case @@ -3159,9 +3204,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .context .inference_flags .replace(InferenceFlags::IN_TYPE_ALIAS, false); - for metadata_element in &arguments[1..] { - self.infer_expression(metadata_element, TypeContext::default()); - } + self.infer_as_evaluated(|builder| { + for metadata_element in &arguments[1..] { + builder.infer_expression(metadata_element, TypeContext::default()); + } + }); self.context .inference_flags .set(InferenceFlags::IN_TYPE_ALIAS, previous_in_type_alias); diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index fd456868c4..df79e40699 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -1017,40 +1017,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { .unwrap_or(class_lit); } - // basedpython parameter-shape tuple in type position. three - // sub-cases: - // * variadic (`(*args: T)`, `(int, *args: T, str)`, - // `(*: T)`): emit a real variable-length tuple type so - // assignability matches `tuple[T, ...]` semantics - // * fixed with markers (`(int, /, name: T)`): the markers - // signal parameter-spec semantics — drop names and emit - // a heterogeneous `tuple[...]`. matches the forward - // transpile lowering - // * fixed without markers but with named fields - // (`(int, name: T)`): handled upstream by - // `is_anon_named_tuple` dispatch if tuple.has_parameter_shape() { - if let Some(ty) = self.lower_parameter_shape_to_tuple_type(tuple) { - return ty; - } - let has_markers = - tuple.parameter_slash().is_some() || tuple.parameter_star().is_some(); - if has_markers { - let elt_tys: Vec> = tuple - .elts - .iter() - .map(|e| match e { - ast::Expr::Named(n) => self.infer_type_expression(&n.value), - _ => self.infer_type_expression(e), - }) - .collect(); - return Type::heterogeneous_tuple(self.db(), env, elt_tys); - } - let class_lit = self - .synthesize_anon_named_tuple_class(tuple, /* is_type_form = */ true); - return class_lit - .to_instance_approximation(self.db(), env) - .unwrap_or(class_lit); + return self.infer_parameter_shape_tuple_type_expression(tuple); } // basedpython: parenthesized tuples are valid type @@ -1913,6 +1881,59 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } } + /// basedpython: a parameter-shape tuple in type position. two sub-cases: + /// * variadic (`(*args: T)`, `(int, *args: T, str)`, `(*: T)`): a real variable-length + /// tuple type, so assignability matches `tuple[T, ...]` semantics + /// * fixed with markers (`(int, /, name: T)`): the names are dropped, giving a + /// heterogeneous `tuple[...]` + /// + /// a tuple with named fields and neither is an anonymous named tuple, which the parser flags + /// and [`Self::infer_type_expression_no_store`] handles. a keyword variadic collects keyword + /// arguments, which a tuple has no place for, so it is only valid in a parameter list + /// + /// kept out of [`Self::infer_type_expression_no_store`], which recurses once per nesting + /// level of a type expression: an unoptimized build gives every local of every arm its own + /// stack slot in that one frame + // an optimizing build would inline it back into that frame + #[inline(never)] + fn infer_parameter_shape_tuple_type_expression(&mut self, tuple: &ast::ExprTuple) -> Type<'db> { + if let Some(keyword_variadic) = tuple.elts.iter().find(|elt| is_keyword_variadic(elt)) { + if let Some(builder) = self + .context + .report_lint(&INVALID_TYPE_FORM, keyword_variadic) + { + builder.into_diagnostic( + "A keyword variadic is only valid in a callable's parameter list, not in a \ + tuple type", + ); + } + for elt in &tuple.elts { + let annotation = match elt { + ast::Expr::Named(named) => &*named.value, + ast::Expr::Starred(starred) => match &*starred.value { + ast::Expr::Starred(inner) => &*inner.value, + value => value, + }, + _ => elt, + }; + self.infer_type_expression(annotation); + } + return Type::unknown(); + } + if let Some(ty) = self.lower_parameter_shape_to_tuple_type(tuple) { + return ty; + } + let elt_tys: Vec> = tuple + .elts + .iter() + .map(|e| match e { + ast::Expr::Named(n) => self.infer_type_expression(&n.value), + _ => self.infer_type_expression(e), + }) + .collect(); + Type::heterogeneous_tuple(self.db(), self.program_environment(), elt_tys) + } + /// basedpython: the callable type of a method member of an inline protocol, /// `def f(self, x: int) -> str`. /// @@ -4558,6 +4579,15 @@ impl<'db> TypeInferenceBuilder<'db, '_> { pub(crate) fn infer_literal_parameter_type<'param>( &mut self, parameters: &'param ast::Expr, + ) -> Result, Vec<&'param ast::Expr>> { + // a literal's parameters are values python evaluates where they are written, an enum + // member's class included + self.infer_as_evaluated(|builder| builder.infer_literal_parameter_type_impl(parameters)) + } + + fn infer_literal_parameter_type_impl<'param>( + &mut self, + parameters: &'param ast::Expr, ) -> Result, Vec<&'param ast::Expr>> { let db = self.db(); let env = self.program_environment(); @@ -4565,7 +4595,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { ast::Expr::Subscript(ast::ExprSubscript { value, slice, .. }) => { let value_ty = self.infer_expression(value, TypeContext::default()); if matches!(value_ty, Type::SpecialForm(SpecialFormType::Literal)) { - let ty = self.infer_literal_parameter_type(slice)?; + let ty = self.infer_literal_parameter_type_impl(slice)?; // This branch deals with annotations such as `Literal[Literal[1]]`. // Here, we store the type for the inner `Literal[1]` expression: @@ -4582,7 +4612,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { let mut errors = vec![]; let mut builder = UnionBuilder::new(db, env); for elt in tuple { - match self.infer_literal_parameter_type(elt) { + match self.infer_literal_parameter_type_impl(elt) { Ok(ty) => { builder = builder.add(ty); } @@ -5261,3 +5291,19 @@ fn is_narrowing_predicate_place(expr: &ast::Expr) -> bool { _ => false, } } + +/// basedpython: whether `elt`, an element of a parameter-shape tuple, is a keyword variadic — +/// `**name: T`, `**: T` or a bare `**P` — rather than the bare-star `*: *`, which shares the +/// outer shape of the last two +fn is_keyword_variadic(elt: &ast::Expr) -> bool { + match elt { + ast::Expr::Named(named) => matches!( + &*named.target, + ast::Expr::Starred(starred) if starred.value.is_starred_expr() + ), + ast::Expr::Starred(starred) => { + starred.value.is_starred_expr() && !is_top_star_marker(&starred.value) + } + _ => false, + } +} diff --git a/crates/ty_python_semantic/src/types/infer/tests.rs b/crates/ty_python_semantic/src/types/infer/tests.rs index bca5a4e2ab..3169128c6f 100644 --- a/crates/ty_python_semantic/src/types/infer/tests.rs +++ b/crates/ty_python_semantic/src/types/infer/tests.rs @@ -854,6 +854,37 @@ class Form(Outer): handle.join().expect("regression test thread panicked") } +/// basedpython: a type test at a module's own level is checked against what the class it names +/// holds when the program runs, which reads every condition the module goes on to make, each +/// `assert` below the test among them. asked during the module's own inference, each of those +/// would be inferred inside the inference of the next, and the stack would grow with them +#[test] +fn module_level_type_test_before_many_asserts_does_not_overflow_stack() -> anyhow::Result<()> { + const ASSERTS: usize = 2_000; + let handle = std::thread::Builder::new() + .name("module-level-type-test-stack-test".into()) + // far less than ty's worker threads have, so a stack that grows with the asserts + // overflows at any optimization level, while one that does not fits with room to spare + .stack_size(2 * 1024 * 1024) + .spawn(|| -> anyhow::Result<()> { + let mut db = setup_db(); + let mut source = String::from( + "class Shown: ...\n\ndef make() -> object:\n return Shown()\n\n\ + value = make()\nflag = value is Shown\n", + ); + for _ in 0..ASSERTS { + source.push_str("assert make()\n"); + } + db.write_file("src/a.by", source)?; + + assert_file_diagnostics(&db, "src/a.by", &[]); + + Ok(()) + })?; + + handle.join().expect("regression test thread panicked") +} + #[test] fn implicit_attribute_preserves_terminal_narrowing_after_many_calls() -> anyhow::Result<()> { let mut db = setup_db(); diff --git a/crates/ty_python_semantic/src/types/instance.rs b/crates/ty_python_semantic/src/types/instance.rs index d9cdd1d9bd..5a225458f3 100644 --- a/crates/ty_python_semantic/src/types/instance.rs +++ b/crates/ty_python_semantic/src/types/instance.rs @@ -17,7 +17,7 @@ use super::{ MaterializationKind, SubclassOfType, Type, TypeAliasType, }; use crate::place::PlaceAndQualifiers; -use crate::types::class::{DynamicNamedTupleAnchor, GenericAlias, StaticClassLiteral}; +use crate::types::class::{GenericAlias, StaticClassLiteral}; use crate::types::constraints::{ ConstraintSet, ConstraintSetBuilder, IteratorConstraintsExtension, OwnedConstraintSet, }; @@ -1261,29 +1261,12 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { source: ClassType<'db>, target: ClassType<'db>, ) -> Option> { - let source_nt = match source.class_literal(db) { - ClassLiteral::DynamicNamedTuple(nt) - if nt.name(db).as_str().starts_with("_AnonNamedTuple_") => - { - nt - } - _ => return None, - }; - let target_nt = match target.class_literal(db) { - ClassLiteral::DynamicNamedTuple(nt) - if nt.name(db).as_str().starts_with("_AnonNamedTuple_") => - { - nt - } - _ => return None, - }; - let spec_of = |nt: crate::types::class::DynamicNamedTupleLiteral<'db>| match nt.anchor(db) { - DynamicNamedTupleAnchor::CollectionsDefinition { spec, .. } - | DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => Some(*spec), - DynamicNamedTupleAnchor::TypingDefinition(_) => None, + let anonymous_spec = |class: ClassType<'db>| match class.class_literal(db) { + ClassLiteral::DynamicNamedTuple(nt) => nt.anchor(db).anonymous_spec(), + _ => None, }; - let source_spec = spec_of(source_nt)?; - let target_spec = spec_of(target_nt)?; + let source_spec = anonymous_spec(source)?; + let target_spec = anonymous_spec(target)?; let source_fields = source_spec.fields(db); let target_fields = target_spec.fields(db); if source_fields.len() != target_fields.len() { diff --git a/crates/ty_python_semantic/src/types/known_instance.rs b/crates/ty_python_semantic/src/types/known_instance.rs index 403d63e0d3..845b470b29 100644 --- a/crates/ty_python_semantic/src/types/known_instance.rs +++ b/crates/ty_python_semantic/src/types/known_instance.rs @@ -519,10 +519,9 @@ impl<'db> KnownInstanceType<'db> { )) } KnownInstanceType::Range { .. } => match type_mapping { - TypeMapping::Promote( - PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularKeepingLiterals, - ) => self.instance_fallback(db, env), + TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular { .. }) => { + self.instance_fallback(db, env) + } _ => Type::KnownInstance(self), }, KnownInstanceType::FunctoolsPartialCall(partial) => { diff --git a/crates/ty_python_semantic/src/types/literal.rs b/crates/ty_python_semantic/src/types/literal.rs index 1df32bd228..e49ce9fee2 100644 --- a/crates/ty_python_semantic/src/types/literal.rs +++ b/crates/ty_python_semantic/src/types/literal.rs @@ -101,6 +101,27 @@ pub(crate) enum LiteralValueTypeKind<'db> { Template(TemplateLiteralType<'db>), } +impl LiteralValueTypeKind<'_> { + /// basedpython: whether a literal of this kind holds exactly one value + /// + /// two literals that each hold one value are related only when their kinds are equal (the + /// last `LiteralValue` arms of `TypeRelationChecker::check_type_pair`). the union builder + /// relies on that to group methods bound to different literals without comparing them, and + /// `one_valued_literals_are_related_only_by_kind` pins it + pub(crate) const fn holds_one_value(self) -> bool { + match self { + LiteralValueTypeKind::Int(_) + | LiteralValueTypeKind::Bool(_) + | LiteralValueTypeKind::String(_) + | LiteralValueTypeKind::Enum(_) + | LiteralValueTypeKind::Bytes(_) + | LiteralValueTypeKind::Float(_) + | LiteralValueTypeKind::Complex(_) => true, + LiteralValueTypeKind::LiteralString | LiteralValueTypeKind::Template(_) => false, + } + } +} + impl<'db> LiteralValueType<'db> { pub(crate) fn new( kind: impl Into>, diff --git a/crates/ty_python_semantic/src/types/lowering_reads.rs b/crates/ty_python_semantic/src/types/lowering_reads.rs new file mode 100644 index 0000000000..c5a568ed67 --- /dev/null +++ b/crates/ty_python_semantic/src/types/lowering_reads.rs @@ -0,0 +1,91 @@ +//! basedpython: the module-level names the transpiled program reads through what the +//! checker spells for it +//! +//! a lowering writes some types out as the checker spells them — a soundness check's target, +//! a reified call's or construction's specialization, the arms of a union parameter given a +//! reified type parameter — and a spelling names each class by the binding the module's +//! global scope reaches it through. the source spells none of those reads, so the import +//! binding a class read only there would look unused. these are the same queries the +//! lowerings ask, asked once the file is checked: each needs the inference of the scope the +//! node is in to be finished +//! +//! the transpiler asks them of the source after its pre-passes (an erased union given a type +//! parameter, a context-sensitive name qualified, an enum lowered), and these ask them of the +//! source as written, which is the one the checker has. a site only the rewritten source has +//! is not seen here + +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{self as ast, Expr, Stmt}; + +use crate::semantic_model::LoweringModel; +use crate::types::runtime_spelling::RuntimeSpelling; +use crate::types::soundness_sites::{SoundnessPositions, soundness_sites}; + +/// every module-level name a spelling the transpiled program writes for `suite` reads. every +/// soundness position counts, because which ones a build enables is the build's choice and +/// the answer has to hold for any build of the file +pub(crate) fn spelled_module_reads(model: &LoweringModel<'_>, suite: &[Stmt]) -> Vec { + let mut spellings = Spellings { + model, + reads: Vec::new(), + }; + for site in soundness_sites(model, suite, SoundnessPositions::all()) { + for plan in site.plans() { + spellings.read(&plan.spelling()); + } + } + spellings.visit_body(suite); + spellings.reads +} + +struct Spellings<'a, 'db> { + model: &'a LoweringModel<'db>, + reads: Vec, +} + +impl Spellings<'_, '_> { + fn read(&mut self, spelling: &RuntimeSpelling) { + self.reads.extend(spelling.global_reads().cloned()); + } +} + +impl<'ast> Visitor<'ast> for Spellings<'_, '_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + if let Stmt::FunctionDef(function) = stmt { + for parameter in &function.parameters { + let Some(annotation) = parameter.annotation() else { + continue; + }; + if let Some(union) = self.model.erased_union(annotation) { + self.read(&union.origin); + for arm in &union.arms { + self.read(arm); + } + for (_, fixed) in &union.fixed { + self.read(fixed); + } + } + } + } + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Call(call) = expr { + self.call(call); + } + walk_expr(self, expr); + } +} + +impl Spellings<'_, '_> { + fn call(&mut self, call: &ast::ExprCall) { + if let Some(specialization) = self.model.reified_call_specialization(call) { + self.read(&specialization); + } + if let Some(arguments) = self.model.reified_constructor_type_arguments(call) { + self.read(&arguments); + } + } +} diff --git a/crates/ty_python_semantic/src/types/narrow.rs b/crates/ty_python_semantic/src/types/narrow.rs index 170910c1b8..eda0fd730b 100644 --- a/crates/ty_python_semantic/src/types/narrow.rs +++ b/crates/ty_python_semantic/src/types/narrow.rs @@ -4667,7 +4667,12 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { expression.file(self.db), lhs_ty, rhs_ty, - Some(right), + // planned as inference plans the test, without asking what the + // names its target is written with hold where it runs. a target + // whose names do not hold the class the checker resolved is + // reported once the file is inferred, and narrows as the test it + // would be: the program does not run with that error in it + Some(crate::types::reified_infer::WrittenTarget::unchecked(right)), ) .narrows_negatively() .then_some(rhs_ty) diff --git a/crates/ty_python_semantic/src/types/receivers.rs b/crates/ty_python_semantic/src/types/receivers.rs index 833197b563..67e4d55b33 100644 --- a/crates/ty_python_semantic/src/types/receivers.rs +++ b/crates/ty_python_semantic/src/types/receivers.rs @@ -97,7 +97,7 @@ fn bind_receiver<'db>( /// annotation, and a declaration has one type wherever it is visible. A scope /// that binds the name to anything else shadows it, the same way it would shadow /// the name in an ordinary load -pub(crate) fn resolve_receiver_attribute<'db>( +fn resolve_receiver_attribute<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 64aa8f0f17..05bfe6da0c 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -26,9 +26,10 @@ use ruff_text_size::{Ranged, TextRange}; use rustc_hash::FxHashMap; use crate::Db; -use crate::place::{builtins_symbol, global_symbol}; +use crate::place::{RuntimeBinding, RuntimeRead, builtins_symbol, global_symbol, runtime_binding}; use crate::types::ProgramEnvironment; use crate::types::call::{Argument, CallArguments}; +use crate::types::check_order::InferredInCheckOrder; use crate::types::class::{ClassLiteral, ClassType, GenericAlias}; use crate::types::class_base::ClassBase; use crate::types::function::FunctionType; @@ -36,6 +37,7 @@ use crate::types::generics::{Specialization, combine_use_site_projections}; use crate::types::instance::Protocol; use crate::types::literal::LiteralValueTypeKind; use crate::types::protocol_class::ReifiedMember; +use crate::types::runtime_spelling::{RuntimeSite, RuntimeSpelling}; use crate::types::tuple::Tuple; use crate::types::typevar::TypeVarKind; use crate::types::variance::TypeVarVariance; @@ -47,6 +49,9 @@ pub(crate) enum ReifiedInferenceError<'db> { Unsolved(Name), /// solved, but the solution has no runtime spelling at this call site Unspellable(Name, Type<'db>), + /// solved and spelled, but the spelling reads a name that does not hold, where the call + /// runs, the binding the checker resolved + NotAtRuntime(Name, RuntimeBinding), /// the callable's specialization could not be derived at all (no unique /// matching overload, or the arguments do not bind) NoBinding, @@ -64,6 +69,7 @@ pub(crate) fn inferred_call_type_arguments<'db>( callee: Type<'db>, function: FunctionType<'db>, arguments: &CallArguments<'_, 'db>, + site: Option>, ) -> Result, ReifiedInferenceError<'db>> { let bindings = callee .try_call(db, env, arguments) @@ -74,7 +80,7 @@ pub(crate) fn inferred_call_type_arguments<'db>( .ok_or(ReifiedInferenceError::NoBinding)? .1 .merged_specialization(db, env); - rendered_type_arguments(db, env, file, function, specialization) + rendered_type_arguments(db, env, file, function, specialization, site) } /// [`inferred_call_type_arguments`] for callers outside the `types` module: @@ -84,6 +90,7 @@ pub(crate) fn inferred_call_type_arguments<'db>( /// keyword-variadic pack contributes fields, since a subscript takes no /// keywords. any failure collapses to `None` — the checker reports those, the /// caller just skips injection +#[expect(clippy::too_many_arguments)] pub(crate) fn injectable_call_specialization<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -92,7 +99,8 @@ pub(crate) fn injectable_call_specialization<'db>( function: FunctionType<'db>, positional: Vec>, keywords: Vec<(&str, Type<'db>)>, -) -> Option { + site: Option>, +) -> Option { let arguments: CallArguments<'_, 'db> = positional .into_iter() .map(|ty| (Argument::Positional, Some(ty))) @@ -103,7 +111,7 @@ pub(crate) fn injectable_call_specialization<'db>( ) .collect(); let rendered = - inferred_call_type_arguments(db, env, file, callee, function, &arguments).ok()?; + inferred_call_type_arguments(db, env, file, callee, function, &arguments, site).ok()?; // an empty prefix means everything defaults — the bare call is already // correct and nothing is injected if rendered.is_empty() { @@ -113,16 +121,22 @@ pub(crate) fn injectable_call_specialization<'db>( // declaration order: the wrapper binds them by name, not by slot let (fields, positional): (Vec<&TypeArgument>, Vec<&TypeArgument>) = rendered.iter().partition(|argument| argument.keyword); - let parts: Vec<&str> = positional - .into_iter() - .chain(fields.iter().copied()) - .map(|argument| argument.text.as_str()) - .collect(); - Some(if fields.is_empty() { - format!("[{}]", parts.join(", ")) + let parts = RuntimeSpelling::join( + positional + .into_iter() + .chain(fields.iter().copied()) + .map(|argument| argument.text.clone()), + ", ", + ); + let (open, close) = if fields.is_empty() { + ("[", "]") } else { - format!(".__getitem__({})", parts.join(", ")) - }) + (".__getitem__(", ")") + }; + let mut injected = RuntimeSpelling::text(open); + injected.push(&parts); + injected.push_str(close); + Some(injected) } /// The rendered runtime spellings to inject, in declaration order, from the @@ -139,6 +153,7 @@ fn rendered_type_arguments<'db>( file: File, function: FunctionType<'db>, specialization: Option>, + site: Option>, ) -> Result, ReifiedInferenceError<'db>> { let signature = function.signature(db); let generic_context = signature @@ -196,15 +211,22 @@ fn rendered_type_arguments<'db>( let ty = parameter .value .ok_or_else(|| ReifiedInferenceError::Unsolved(parameter.name.clone()))?; - let promoted = ty.promote(db, env); - parameter + let promoted = ty.promote_in(db, env, file); + let text = parameter .kind .spelling(db, env, file, promoted) - .map(|text| TypeArgument { - text, - keyword: parameter.kind == ParameterKind::KeywordPack, - }) - .ok_or_else(|| ReifiedInferenceError::Unspellable(parameter.name.clone(), promoted)) + .ok_or_else(|| { + ReifiedInferenceError::Unspellable(parameter.name.clone(), promoted) + })?; + if let Some(site) = site + && let Some((name, binding)) = text.unbound_read(db, env, site) + { + return Err(ReifiedInferenceError::NotAtRuntime(name, binding)); + } + Ok(TypeArgument { + text, + keyword: parameter.kind == ParameterKind::KeywordPack, + }) }) // a variadic or pack that absorbed nothing spells as nothing — it // occupies no slot in the injected list, exactly as the wrapper binds it @@ -214,13 +236,20 @@ fn rendered_type_arguments<'db>( /// one rendered argument of an injected specialization pub(crate) struct TypeArgument { - /// the source text of this argument — `int`, the comma-joined run of a + /// the spelling of this argument — `int`, the comma-joined run of a /// `*Ts`, or the `foo=int, bar=str` fields of a `**Kwargs` pack - text: String, + text: RuntimeSpelling, /// whether the text is keyword-spelled, and so cannot go in a subscript keyword: bool, } +impl TypeArgument { + /// every binding of the module's global scope the argument's spelling reads + pub(crate) fn global_reads(&self) -> impl Iterator { + self.text.global_reads() + } +} + /// a type parameter paired with the value the call solved it to struct ResolvedParameter<'db, 'name> { name: &'name Name, @@ -267,15 +296,15 @@ impl ParameterKind { } } - /// the source text this parameter's value spells as, or the empty string - /// when it stands for no arguments at all + /// the spelling of this parameter's value, or the empty spelling when it + /// stands for no arguments at all fn spelling<'db>( self, db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, - ) -> Option { + ) -> Option { let fields = match self { Self::Single => return runtime_spelling(db, env, file, ty), Self::Variadic => { @@ -285,20 +314,24 @@ impl ParameterKind { runtime_spelling(db, env, file, element.promote_in(db, env, file)) }) .collect::>>()?; - return Some(spellings.join(", ")); + return Some(RuntimeSpelling::join(spellings, ", ")); } Self::KeywordPack => ty.keyword_pack_fields(db)?, }; let spellings = fields .into_iter() .map(|(name, field)| { - Some(format!( - "{name}={}", - runtime_spelling(db, env, file, field.promote_in(db, env, file))? - )) + let mut spelling = RuntimeSpelling::text(format!("{name}=")); + spelling.push(&runtime_spelling( + db, + env, + file, + field.promote_in(db, env, file), + )?); + Some(spelling) }) .collect::>>()?; - Some(spellings.join(", ")) + Some(RuntimeSpelling::join(spellings, ", ")) } } @@ -341,9 +374,9 @@ fn runtime_spelling<'db>( env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, -) -> Option { +) -> Option { if ty.is_none(db) { - return Some("None".to_owned()); + return Some(RuntimeSpelling::text("None")); } match ty { Type::NominalInstance(instance) => spell_class(db, env, file, instance.class(db, env)), @@ -351,16 +384,16 @@ fn runtime_spelling<'db>( // into the enclosing function's closure, and the `generic` wrapper fills // that cell with the type argument, so the name evaluates to the type Type::TypeVar(bound_typevar) if is_reified_function_typevar(db, bound_typevar) => { - Some(bound_typevar.name(db).to_string()) + Some(RuntimeSpelling::text(bound_typevar.name(db).as_str())) } - Type::Union(union) => Some( + Type::Union(union) => Some(RuntimeSpelling::join( union .elements(db) .iter() .map(|element| runtime_spelling(db, env, file, *element)) - .collect::>>()? - .join(" | "), - ), + .collect::>>()?, + " | ", + )), _ => None, } } @@ -370,16 +403,19 @@ fn spell_class<'db>( env: &ProgramEnvironment<'db>, file: File, class: ClassType<'db>, -) -> Option { +) -> Option { match class { ClassType::NonGeneric(literal) => spell_class_literal(db, env, file, literal), ClassType::Generic(alias) => { let origin = ClassLiteral::Static(alias.origin(db)); - let base = spell_class_literal(db, env, file, origin)?; + let mut spelling = spell_class_literal(db, env, file, origin)?; let arguments = spell_specialization_arguments(db, env, file, origin, alias.specialization(db)) .ok()?; - Some(format!("{base}[{arguments}]")) + spelling.push_str("["); + spelling.push(&arguments); + spelling.push_str("]"); + Some(spelling) } } } @@ -521,7 +557,7 @@ fn spell_specialization_arguments<'db>( file: File, origin: ClassLiteral<'db>, specialization: Specialization<'db>, -) -> Result> { +) -> Result> { // the caller may be about to *invent* an `origin[…]` the source never wrote, // so it fires only on positive evidence. a wrong yes kills the program at // import; a wrong no costs the specialization nothing else can observe @@ -539,9 +575,9 @@ fn spell_specialization_arguments<'db>( .collect::>>() .ok_or(ReifiedInferenceError::NoBinding)?; return Ok(if elements.is_empty() { - "()".to_owned() + RuntimeSpelling::text("()") } else { - elements.join(", ") + RuntimeSpelling::join(elements, ", ") }); } let Some(generic_context) = origin.generic_context(db) else { @@ -557,10 +593,11 @@ fn spell_specialization_arguments<'db>( if matches!(kind, ParameterKind::KeywordPack) { return Err(ReifiedInferenceError::NoBinding); } - // an argument is promoted one by one: a covariant parameter keeps the - // literal type it was inferred from, and only a class object can be - // written at runtime - let promoted = argument.promote(db, env); + // a covariant parameter keeps the literal type it was inferred from + // (`B[Literal[1]]`), and a literal has no runtime spelling, so each argument + // is promoted on its own, outside the class's variance, to the class it is + // an instance of — under the numeric model of the module it is written into + let promoted = argument.promote_in(db, env, file); // an unsolved parameter reads as `Never`, and one solved from something // the checker cannot see reads as `Unknown`. neither is an argument // anybody wrote down, so both say the same thing: name one @@ -579,7 +616,7 @@ fn spell_specialization_arguments<'db>( arguments.push(spelling); } } - Ok(arguments.join(", ")) + Ok(RuntimeSpelling::join(arguments, ", ")) } /// The specialization a construction of `class_literal` builds, or the reason @@ -589,16 +626,22 @@ fn spell_specialization_arguments<'db>( /// instance, so the answer always matches the checker's solved specialization — /// including PEP 696 defaults and any usage-based widening. This is the single /// decision behind both directions: [`constructor_specialization_display`] is -/// the transpiler's injection and [`reified_construction_error`] the checker's -/// diagnostic, so the two cannot disagree about which constructions are legal. -fn constructor_specialization<'db>( +/// the transpiler's injection and the checker reports a reified class's +/// construction this rejects, so the two cannot disagree about which +/// constructions are legal. +/// +/// at a `site`, every name the spelling reads must hold there the binding the +/// checker resolved. inference asks without one, and the names are checked once +/// the module is inferred ([`crate::types::runtime_reads`]) +pub(crate) fn constructor_specialization<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, -) -> Result> { - let Type::NominalInstance(instance) = constructed.promote(db, env) else { + site: Option>, +) -> Result> { + let Type::NominalInstance(instance) = constructed.promote_in(db, env, file) else { return Err(ReifiedInferenceError::NoBinding); }; let ClassType::Generic(alias) = instance.class(db, env) else { @@ -608,7 +651,13 @@ fn constructor_specialization<'db>( if origin != class_literal { return Err(ReifiedInferenceError::NoBinding); } - spell_specialization_arguments(db, env, file, origin, alias.specialization(db)) + let spelling = spell_specialization_arguments(db, env, file, origin, alias.specialization(db))?; + if let Some(site) = site + && let Some((name, binding)) = spelling.unbound_read(db, env, site) + { + return Err(ReifiedInferenceError::NotAtRuntime(name, binding)); + } + Ok(spelling) } /// The bracketed type-argument spelling to inject at a bare constructor call @@ -616,65 +665,45 @@ fn constructor_specialization<'db>( /// /// `None` when the construction has no writable specialization; unlike a bare /// reified-generic call this is never an error on its own — the call simply -/// stays bare, and [`reified_construction_error`] reports it only when the class -/// reifies something. +/// stays bare, and the checker reports it only when the class reifies +/// something. pub(crate) fn constructor_specialization_display<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, -) -> Option { - constructor_specialization(db, env, file, class_literal, constructed).ok() + site: Option>, +) -> Option { + constructor_specialization(db, env, file, class_literal, constructed, site).ok() } -/// Why a construction of the reified generic class `class_literal` cannot say -/// which specialization it builds, or `None` when it can. -/// -/// A reified class records its type arguments on the specialization its -/// instances are built from, so a construction has to name one. This is -/// [`constructor_specialization_display`]'s decision with the reason kept: the -/// transpiler injects the solved specialization wherever one can be written, and -/// where it cannot the instance would have no answer for a type parameter its -/// own methods read. -pub(crate) fn reified_construction_error<'db>( - db: &'db dyn Db, - env: &ProgramEnvironment<'db>, - file: File, - class_literal: ClassLiteral<'db>, - constructed: Type<'db>, -) -> Option> { - constructor_specialization(db, env, file, class_literal, constructed).err() -} - -/// The full runtime spelling (`list[int]`, `tuple[int, str]`) with which the -/// transpiler makes a collection literal's inferred element types explicit -/// (`[1, 2]` → `list[int]([1, 2])`). -/// -/// `None` when the literal's (literal-promoted) type is not a plain -/// specialization of the expected builtin: empty or partially-`Unknown` -/// elements, a `TypedDict`-typed dict display, a shadowed builtin name, or -/// elements without a runtime spelling. /// the class's bare name, provided that name resolves — in the module's /// globals, else builtins — to this very class, so the injected expression -/// evaluates to the intended type object +/// evaluates to the intended type object. a module-level `private` class is +/// emitted under a name of the transpiler's, so its name is kept apart for the +/// transpiler to write ([`RuntimeSpelling::global`]) +/// +/// whether the name holds that class where the expression is written is the +/// writer's to ask ([`RuntimeSpelling::unbound_read`]) fn spell_class_literal<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, literal: ClassLiteral<'db>, -) -> Option { +) -> Option { let name = literal.name(db); - let resolved = global_symbol(db, db.program_file(file), name) + if let Some(global) = global_symbol(db, db.program_file(file), name) .place .ignore_possibly_undefined() - .or_else(|| { - builtins_symbol(db, env, name) - .place - .ignore_possibly_undefined() - })?; - let resolved_literal = resolved.as_class_literal()?; - (resolved_literal == literal).then(|| name.to_string()) + { + return (global.as_class_literal()? == literal) + .then(|| RuntimeSpelling::global(db, file, name)); + } + let builtin = builtins_symbol(db, env, name) + .place + .ignore_possibly_undefined()?; + (builtin.as_class_literal()? == literal).then(|| RuntimeSpelling::text(name.as_str())) } /// How a parametric type test (`x is C[args]`, keyword form) resolves. @@ -762,6 +791,10 @@ pub enum ParametricIsPlan { /// the interface's required member names, for the runtime check to look /// for on a value nothing registered members: Vec, + /// the modules declaring the conformances to the interface that are visible + /// here. the test is answered from the registry only because one is, so the + /// import making each one visible is relied on + declared_in: Vec, }, /// nothing is known about the target — an error elsewhere left `Unknown` /// behind. the test is lowered as the plain `isinstance` the source spells @@ -784,6 +817,11 @@ impl ParametricIsPlan { /// and a value that records none answers `False` even where the static /// types say it is a match — narrowing on that would remove a type the /// value really has. + /// + /// narrowing asks this of the plan inference makes, without asking what the names the + /// target is written with hold where the test runs ([`WrittenTarget::unchecked`]): a test + /// whose target the program does not bind there is reported once the file is inferred, + /// and narrows as the test it would be pub(crate) fn narrows_negatively(&self) -> bool { match self { Self::Fold(_) @@ -814,6 +852,64 @@ impl ParametricIsPlan { _ => false, } } + + /// every spelling the plan writes, in any arm + pub(crate) fn spellings(&self) -> Vec<&RuntimeSpelling> { + fn rebuilt(target: &TargetSpelling) -> Option<&RuntimeSpelling> { + match target { + TargetSpelling::Rebuilt(spelling) => Some(spelling), + TargetSpelling::Written => None, + } + } + match self { + Self::Probe { target, .. } + | Self::Isinstance(target) + | Self::Identity(target) + | Self::Subclass(target) + | Self::Conformance { target, .. } + | Self::Equality { value: target, .. } => rebuilt(target).into_iter().collect(), + Self::ProtocolStructural(checks) => checks + .iter() + .flat_map(|check| match check { + ProtocolMemberCheck::Attribute { expected, .. } => vec![expected], + ProtocolMemberCheck::Method { params, ret, .. } => params + .iter() + .chain(ret) + .map(|(spelling, _)| spelling) + .collect(), + }) + .collect(), + Self::Union(arms) => arms.iter().flat_map(Self::spellings).collect(), + Self::Fold(_) + | Self::TokenEq(_) + | Self::ErasedTarget(_) + | Self::IsCallable + | Self::IsNone + | Self::Pattern(_) + | Self::Unresolved => Vec::new(), + } + } + + /// every module whose conformance this plan answers from, in any arm + pub(crate) fn conformance_modules(&self) -> Vec { + match self { + Self::Conformance { declared_in, .. } => declared_in.clone(), + Self::Union(arms) => arms.iter().flat_map(Self::conformance_modules).collect(), + Self::Fold(_) + | Self::TokenEq(_) + | Self::Probe { .. } + | Self::ProtocolStructural(_) + | Self::ErasedTarget(_) + | Self::Isinstance(_) + | Self::IsCallable + | Self::IsNone + | Self::Equality { .. } + | Self::Identity(_) + | Self::Subclass(_) + | Self::Pattern(_) + | Self::Unresolved => Vec::new(), + } + } } /// basedpython: how a type test's target is written into the emitted python @@ -826,7 +922,7 @@ pub enum TargetSpelling { Written, /// nothing in the source spells this target on its own — one arm of a union /// the source named with a single word — so it is rebuilt - Rebuilt(String), + Rebuilt(RuntimeSpelling), } /// basedpython: one protocol member a parametric `is`-test checks structurally @@ -840,7 +936,7 @@ pub enum ProtocolMemberCheck { name: String, /// a python expression evaluating, in the checked module's scope, to the /// member's specialized type (`int`, `list[str]`) - expected: String, + expected: RuntimeSpelling, variance: ArgVariance, }, /// a method member: each declared positional parameter (contravariant) and, @@ -850,15 +946,15 @@ pub enum ProtocolMemberCheck { name: String, /// per-parameter `(specialized type spelling, variance)`, in declaration /// order after `self` — always contravariant - params: Vec<(String, ArgVariance)>, + params: Vec<(RuntimeSpelling, ArgVariance)>, /// the return `(specialized type spelling, variance)` — covariant, or /// `None` when the method declares no meaningful return to check - ret: Option<(String, ArgVariance)>, + ret: Option<(RuntimeSpelling, ArgVariance)>, }, } /// why a type test's target cannot be checked at runtime -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum ErasedTargetReason { /// the target says nothing a runtime check could look for — `Any` and /// `Unknown` admit every value, so the test has no content @@ -887,6 +983,10 @@ pub enum ErasedTargetReason { /// the target has no runtime spelling at all — a type the checker can name /// but the emitted python cannot evaluate Unspellable, + /// the target names something that does not hold, where the test runs, the binding + /// the checker resolved — a class imported only under `if TYPE_CHECKING:`, or declared + /// further on + NotAtRuntime(Name, RuntimeBinding), /// a builtin collection (`list` / `dict` / `set` / `frozenset` / `tuple`) /// erases its type arguments — its C-level instances reject /// `__orig_class__` entirely @@ -908,15 +1008,15 @@ pub enum ErasedTargetReason { /// is; the specialization has to travel with the call instead pub struct ErasedUnion { /// the shared origin's runtime spelling — `list` - pub origin: String, + pub origin: RuntimeSpelling, /// which type-argument position the arms differ in pub position: usize, /// the differing argument of each arm, in declaration order — `int`, `str` - pub arms: Vec, + pub arms: Vec, /// the arguments the arms agree on, by position, so the rewritten /// annotation can put them back (`dict[str, int] | dict[str, bool]` keeps /// `str` in position 0) - pub fixed: Vec<(usize, String)>, + pub fixed: Vec<(usize, RuntimeSpelling)>, } /// classify `ty` as an [`ErasedUnion`], or `None` when it is anything else. @@ -1117,10 +1217,10 @@ pub(crate) fn classify_parametric_is<'db>( file: File, lhs_ty: Type<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, - rhs_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { let target_origin = ClassLiteral::Static(rhs_alias.origin(db)); - let target_args_ast: Vec<&ast::Expr> = match rhs_node { + let target_args_ast: Vec<&ast::Expr> = match written.map(WrittenTarget::node) { Some(ast::Expr::Subscript(subscript)) => match subscript.slice.as_ref() { ast::Expr::Tuple(tuple) => tuple.elts.iter().collect(), single => vec![single], @@ -1134,7 +1234,7 @@ pub(crate) fn classify_parametric_is<'db>( lhs_ty.promote(db, env), rhs_alias, &target_args_ast, - rhs_node, + written, ); // the runtime probe unwinds the value's `__orig_class__` and its class's // generic bases across the mro, so a builtin-collection target is checkable @@ -1182,8 +1282,11 @@ pub(crate) fn classify_parametric_is<'db>( /// rules out a `TypedDict` (its instances are plain dicts), a callable type /// (the runtime sees only that a value is callable), and `Any`. /// -/// `target_node` is the source the target was written as, which the -/// specialization plans use to spell type arguments back out. +/// `written` is the source the target was written as, which the specialization plans use to +/// spell type arguments back out. checked where the test runs +/// ([`WrittenTarget::checked`]), every name a spelling reads must hold there the binding the +/// checker resolved ([`runtime_binding`]), and a target whose names do not is +/// [`ErasedTargetReason::NotAtRuntime`]. inference plans with an unchecked target /// /// `value_ty` is taken exactly as the value has it — a construction's `final A` /// is disjoint from an unrelated class in a way a plain `A` is not, and that is @@ -1195,17 +1298,20 @@ pub(crate) fn type_test_plan<'db>( file: File, value_ty: Type<'db>, target: Type<'db>, - target_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { - type_test_plan_seen( - db, - env, - file, - value_ty, - target, - target_node, - &mut Vec::new(), - ) + let plan = type_test_plan_seen(db, env, file, value_ty, target, written, &mut Vec::new()); + // each spelling the plan rebuilds is written where the test is: one for a union's arm + // has no node of its own to have been asked about + if let Some(site) = written.and_then(WrittenTarget::site) + && let Some((name, binding)) = plan + .spellings() + .into_iter() + .find_map(|spelling| spelling.unbound_read(db, env, site)) + { + return ParametricIsPlan::ErasedTarget(ErasedTargetReason::NotAtRuntime(name, binding)); + } + plan } /// [`type_test_plan`] carrying the aliases already opened on the way here. @@ -1219,14 +1325,14 @@ fn type_test_plan_seen<'db>( file: File, value_ty: Type<'db>, target: Type<'db>, - target_node: Option<&ast::Expr>, + written: Option>, open: &mut Vec>, ) -> ParametricIsPlan { // an `Unknown` target is not evidence of anything: every value is a subtype // of it, and folding on that would answer `True` for a test the source got // wrong somewhere else if target.is_dynamic() { - return runtime_test_plan(db, env, file, value_ty, target, target_node, open); + return runtime_test_plan(db, env, file, value_ty, target, written, open); } // a target that resolves statically needs no runtime residue at all, and // answering it here keeps every unspellable-but-decidable target working: @@ -1237,7 +1343,7 @@ fn type_test_plan_seen<'db>( if value_ty.is_disjoint_from(db, env, target) { return ParametricIsPlan::Fold(false); } - runtime_test_plan(db, env, file, value_ty, target, target_node, open) + runtime_test_plan(db, env, file, value_ty, target, written, open) } /// The runtime residue of a type test whose answer the static types do not @@ -1249,7 +1355,7 @@ fn runtime_test_plan<'db>( file: File, value_ty: Type<'db>, target: Type<'db>, - target_node: Option<&ast::Expr>, + written: Option>, open: &mut Vec>, ) -> ParametricIsPlan { if target.is_none(db) { @@ -1312,7 +1418,7 @@ fn runtime_test_plan<'db>( // a literal type holds exactly the values equal to it, which is what // the runtime compares. an enum member is a literal too, and its // equality is identity - Type::LiteralValue(literal) => literal_target_plan(db, env, file, literal, target_node), + Type::LiteralValue(literal) => literal_target_plan(db, env, file, literal, written), // `type[C]`: the runtime can check this one in full — the value must be // a class, and a subclass of `C` @@ -1329,15 +1435,15 @@ fn runtime_test_plan<'db>( }, Type::NominalInstance(_) | Type::ProtocolInstance(_) => { - class_target_plan(db, env, file, value_ty, target, target_node) + class_target_plan(db, env, file, value_ty, target, written) } // the value's type is carried by a reified type parameter on the // *target* side — `x is T` where `T` is reified spells `T` itself Type::TypeVar(bound_typevar) if is_reified_function_typevar(db, bound_typevar) => { - ParametricIsPlan::Isinstance(TargetSpelling::Rebuilt( - bound_typevar.name(db).to_string(), - )) + ParametricIsPlan::Isinstance(TargetSpelling::Rebuilt(RuntimeSpelling::text( + bound_typevar.name(db).as_str(), + ))) } // `Any` really does admit every value and is worth rejecting. `Unknown` @@ -1371,19 +1477,20 @@ fn literal_target_plan<'db>( env: &ProgramEnvironment<'db>, file: File, literal: crate::types::LiteralValueType<'db>, - target_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { let unspellable = ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); // the guard names a builtin, and the value is the source's own literal where it wrote // one - let written = target_node - .is_some_and(|node| node.is_literal_expr() || matches!(node, ast::Expr::UnaryOp(_))); + let literal_written = written.is_some_and(|written| { + written.node.is_literal_expr() || matches!(written.node, ast::Expr::UnaryOp(_)) + }); let equality = |class: &str, value: String| ParametricIsPlan::Equality { class: class.to_owned(), - value: if written { + value: if literal_written { TargetSpelling::Written } else { - TargetSpelling::Rebuilt(value) + TargetSpelling::Rebuilt(RuntimeSpelling::text(value)) }, }; match literal.kind() { @@ -1404,12 +1511,14 @@ fn literal_target_plan<'db>( // an enum member is a singleton, so identity is exact — and it is the // comparison `Enum.__eq__` performs anyway LiteralValueTypeKind::Enum(member) => { - match written_or_spelled(target_node, false, || { - spell_class_literal(db, env, file, member.enum_class(db)) - .map(|class| format!("{class}.{}", member.name(db))) + match written_or_spelled(db, env, written, false, || { + spell_class_literal(db, env, file, member.enum_class(db)).map(|mut class| { + class.push_str(&format!(".{}", member.name(db))); + class + }) }) { - Some(spelling) => ParametricIsPlan::Identity(spelling), - None => unspellable, + Ok(spelling) => ParametricIsPlan::Identity(spelling), + Err(reason) => ParametricIsPlan::ErasedTarget(reason), } } // `LiteralString` is the *property* of having been written as a literal, @@ -1424,6 +1533,43 @@ fn literal_target_plan<'db>( } } +/// a type test's target as the source wrote it, and whether what the names it is written with +/// hold where the test runs is asked +#[derive(Clone, Copy)] +pub(crate) struct WrittenTarget<'a, 'db> { + node: &'a ast::Expr, + /// present once the scopes around the test are inferred, which the question needs + inferred: Option>, +} + +impl<'a, 'db> WrittenTarget<'a, 'db> { + /// a target whose names are not asked about, as inference plans a test: what they hold + /// where it runs is asked once the file is inferred ([`crate::types::runtime_reads`]) + pub(crate) fn unchecked(node: &'a ast::Expr) -> Self { + Self { + node, + inferred: None, + } + } + + /// a target whose names must hold, where the test runs, the bindings the checker resolved + pub(crate) fn checked(node: &'a ast::Expr, inferred: InferredInCheckOrder<'db>) -> Self { + Self { + node, + inferred: Some(inferred), + } + } + + fn node(self) -> &'a ast::Expr { + self.node + } + + fn site(self) -> Option> { + self.inferred + .map(|inferred| RuntimeSite::at(self.node, inferred)) + } +} + /// how a target is written into the emitted python: the spelling rebuilt from /// the type, or the source's own text where that is what the runtime evaluates. /// @@ -1437,17 +1583,90 @@ fn literal_target_plan<'db>( /// or a class imported under another name. Only a plain dotted name qualifies, /// and only a [`Probe`](ParametricIsPlan::Probe) also accepts a subscript, /// whose value is the specialization it unwinds. -fn written_or_spelled( - target_node: Option<&ast::Expr>, +/// +/// the written node is evaluated where it stands, and a rebuilt spelling where the test is +/// written, so at a `site` each has to read names that hold there the bindings the checker +/// resolved ([`runtime_binding`]): a class imported under `if TYPE_CHECKING:` is one the +/// checker resolves and the program does not have. without a site neither is asked, and the +/// rebuilt spelling is taken wherever there is one +fn written_or_spelled<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + written: Option>, subscript_evaluates: bool, - spell: impl FnOnce() -> Option, -) -> Option { + spell: impl FnOnce() -> Option, +) -> Result { + let site = written.and_then(WrittenTarget::site); + let mut unbound = None; if let Some(spelling) = spell() { - return Some(TargetSpelling::Rebuilt(spelling)); + match site.and_then(|site| spelling.unbound_read(db, env, site)) { + None => return Ok(TargetSpelling::Rebuilt(spelling)), + Some(read) => unbound = Some(read), + } } - target_node + let Some(node) = written + .map(WrittenTarget::node) .filter(|node| evaluates_to_its_target(node, subscript_evaluates)) - .map(|_| TargetSpelling::Written) + else { + return Err( + unbound.map_or(ErasedTargetReason::Unspellable, |(name, binding)| { + ErasedTargetReason::NotAtRuntime(name, binding) + }), + ); + }; + let Some(site) = site else { + return Ok(TargetSpelling::Written); + }; + match written_unbound_read(db, env, node, site.inferred()) { + None => Ok(TargetSpelling::Written), + Some((name, binding)) => Err(ErasedTargetReason::NotAtRuntime(name, binding)), + } +} + +/// the first name the written expression `node` reads that does not hold, where `node` is +/// written, the binding the checker resolved — with what it holds instead. the same question +/// [`RuntimeSpelling::unbound_read`] asks of a spelling the checker writes, asked of one the +/// source wrote +pub(crate) fn written_unbound_read<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + node: &ast::Expr, + inferred: InferredInCheckOrder<'db>, +) -> Option<(Name, RuntimeBinding)> { + struct Names<'a, 'db> { + db: &'db dyn Db, + env: &'a ProgramEnvironment<'db>, + inferred: InferredInCheckOrder<'db>, + found: Option<(Name, RuntimeBinding)>, + } + impl<'ast> ast::visitor::Visitor<'ast> for Names<'_, '_> { + fn visit_expr(&mut self, expr: &'ast ast::Expr) { + if self.found.is_some() { + return; + } + if let ast::Expr::Name(name) = expr { + let binding = runtime_binding( + self.db, + self.env, + RuntimeRead::Written(ast::ExprRef::from(expr)), + self.inferred, + ); + if binding != RuntimeBinding::Bound { + self.found = Some((name.id.clone(), binding)); + } + return; + } + ast::visitor::walk_expr(self, expr); + } + } + let mut names = Names { + db, + env, + inferred, + found: None, + }; + ast::visitor::Visitor::visit_expr(&mut names, node); + names.found } /// whether the runtime value of `node` is the thing the type it names denotes @@ -1498,7 +1717,7 @@ fn class_target_plan<'db>( file: File, value_ty: Type<'db>, target: Type<'db>, - target_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { let Some(class) = target_class(db, env, target) else { return ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); @@ -1522,28 +1741,32 @@ fn class_target_plan<'db>( .iter() .all(Type::is_dynamic) => { - match written_or_spelled(target_node, false, || { + match written_or_spelled(db, env, written, false, || { spell_class_literal(db, env, file, ClassLiteral::Static(alias.origin(db))) }) { - Some(spelling) => ParametricIsPlan::Isinstance(spelling), - None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + Ok(spelling) => ParametricIsPlan::Isinstance(spelling), + Err(reason) => ParametricIsPlan::ErasedTarget(reason), } } // a specialization keeps the engine it always had: reified cells, // an `__orig_class__` probe, or a structural protocol check ClassType::Generic(alias) => { - classify_parametric_is(db, env, file, value_ty, alias, target_node) + classify_parametric_is(db, env, file, value_ty, alias, written) } ClassType::NonGeneric(_) => { // an interface something visibly conforms to is answered by the // registry rather than by the class hierarchy, so it is checkable // even though a conforming type is not a subclass - if let Some(members) = conformance_members(db, file, class) { - return match written_or_spelled(target_node, false, || { + if let Some((members, declared_in)) = conformance_members(db, file, class) { + return match written_or_spelled(db, env, written, false, || { spell_class(db, env, file, class) }) { - Some(target) => ParametricIsPlan::Conformance { target, members }, - None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + Ok(target) => ParametricIsPlan::Conformance { + target, + members, + declared_in, + }, + Err(reason) => ParametricIsPlan::ErasedTarget(reason), }; } if let Some(protocol) = class.into_protocol_class(db) { @@ -1565,30 +1788,39 @@ fn class_target_plan<'db>( }; } } - match written_or_spelled(target_node, false, || spell_class(db, env, file, class)) { - Some(spelling) => ParametricIsPlan::Isinstance(spelling), - None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + match written_or_spelled(db, env, written, false, || { + spell_class(db, env, file, class) + }) { + Ok(spelling) => ParametricIsPlan::Isinstance(spelling), + Err(reason) => ParametricIsPlan::ErasedTarget(reason), } } } } /// The required member names of `class` when something in `file`'s scope -/// visibly conforms to it, or `None` when it is not a conformance interface. +/// visibly conforms to it, with the modules declaring those conformances, or +/// `None` when it is not a conformance interface. fn conformance_members<'db>( db: &'db dyn Db, file: File, class: ClassType<'db>, -) -> Option> { - let conformed = crate::types::conformance::visible_conformances(db, file) +) -> Option<(Vec, Vec)> { + let mut declared_in: Vec = Vec::new(); + for (extension, declared) in crate::types::conformance::visible_conformances(db, file) { + let module = extension.file(db); + if declared.class_literal(db) == class.class_literal(db) && !declared_in.contains(&module) { + declared_in.push(module); + } + } + if declared_in.is_empty() { + return None; + } + let members = crate::types::conformance::interface_requirements(db, class) .iter() - .any(|(_, declared)| declared.class_literal(db) == class.class_literal(db)); - conformed.then(|| { - crate::types::conformance::interface_requirements(db, class) - .iter() - .map(ToString::to_string) - .collect() - }) + .map(ToString::to_string) + .collect(); + Some((members, declared_in)) } /// whether a callable signature says nothing beyond "this is callable" — the @@ -1764,7 +1996,7 @@ fn protocol_member_spelling<'db>( env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, -) -> Option { +) -> Option { if let Type::LiteralValue(literal) = ty { let value = match literal.kind() { LiteralValueTypeKind::Bool(boolean) => { @@ -1783,7 +2015,7 @@ fn protocol_member_spelling<'db>( } _ => return None, }; - return Some(format!("_by_lit({value})")); + return Some(RuntimeSpelling::text(format!("_by_lit({value})"))); } runtime_spelling(db, env, file, ty) } @@ -1795,7 +2027,7 @@ enum ReturnCheck { /// `object`) — nothing to verify Skip, /// check the value method's return annotation against this spelling - Check(String), + Check(RuntimeSpelling), /// a meaningful return with no runtime spelling — the method can't be checked Unspellable, } @@ -1827,7 +2059,7 @@ fn classify_value<'db>( value_ty: Type<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, target_args_ast: &[&ast::Expr], - rhs_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { let target_origin = ClassLiteral::Static(rhs_alias.origin(db)); // when the value's type is carried by a reified type parameter, the answer @@ -1841,7 +2073,7 @@ fn classify_value<'db>( target_origin, rhs_alias, target_args_ast, - rhs_node, + written.map(WrittenTarget::node), ) { return plan; @@ -1859,10 +2091,11 @@ fn classify_value<'db>( // runtime probe (user generic) or an erased-target error (builtin) // the source's own spelling is preferred: a name it wrote is in scope, // and the runtime probe unwraps a `TypeAliasType` for itself - let Some(target) = written_or_spelled(rhs_node, true, || { + let target = match written_or_spelled(db, env, written, true, || { spell_class(db, env, file, ClassType::Generic(rhs_alias)) - }) else { - return ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); + }) { + Ok(target) => target, + Err(reason) => return ParametricIsPlan::ErasedTarget(reason), }; ParametricIsPlan::Probe { target, @@ -1940,7 +2173,7 @@ pub(crate) fn parametric_soundness_spelling<'db>( env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, -) -> Option<(String, Box<[ArgVariance]>)> { +) -> Option<(RuntimeSpelling, Box<[ArgVariance]>)> { let Type::NominalInstance(instance) = ty else { return None; }; diff --git a/crates/ty_python_semantic/src/types/relation.rs b/crates/ty_python_semantic/src/types/relation.rs index 5179c54e5d..5f4bb9ef17 100644 --- a/crates/ty_python_semantic/src/types/relation.rs +++ b/crates/ty_python_semantic/src/types/relation.rs @@ -2522,6 +2522,11 @@ impl<'a, 'c, 'db> TypeRelationChecker<'a, 'c, 'db> { // For union simplification, we want to preserve the unpromotable form of a literal value, // and so redundancy is not symmetric. + // + // basedpython: this arm and the next one say that two literals holding one value each + // (`LiteralValueTypeKind::holds_one_value`) are related only when their kinds are equal. + // the union builder's `LiteralReceiverMethod` groups rely on that, and + // `one_valued_literals_are_related_only_by_kind` pins it (Type::LiteralValue(source), Type::LiteralValue(target)) if matches!(self.relation, TypeRelation::Redundancy { pure: false }) => { diff --git a/crates/ty_python_semantic/src/types/runtime_reads.rs b/crates/ty_python_semantic/src/types/runtime_reads.rs new file mode 100644 index 0000000000..ddf2485152 --- /dev/null +++ b/crates/ty_python_semantic/src/types/runtime_reads.rs @@ -0,0 +1,312 @@ +//! basedpython: whether each module-level name a lowering writes into code that runs holds +//! there the binding the checker resolved +//! +//! a type test, a checked cast and a bare call or construction of a reified generic are +//! lowered to code that names classes — the test's target, the specialization the call is +//! given — and the program has to find there the class the checker resolved. a name bound +//! only under `if TYPE_CHECKING:`, only on some paths, or only further on does not hold it +//! ([`runtime_binding`](crate::place::runtime_binding)). which one it is depends on every +//! binding the module makes of the name and on every condition deciding whether each is +//! made, the conditions after the site included, so it is asked once the scope holding the +//! site, and every scope around it, is inferred ([`InferredInCheckOrder`]) +//! +//! inference only records where to ask ([`RuntimeReadCheck`]). asked from inside the +//! module's own inference, the question would infer every condition below the site out of +//! order, each one inside the inference of the next + +use ruff_db::files::File; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_python_ast::name::Name; +use ruff_python_ast::{self as ast, AnyRootNodeRef, NodeIndex}; +use ruff_text_size::{Ranged, TextRange}; +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::{ProgramFile, semantic_index}; + +use crate::semantic_model::{HasType, SemanticModel}; +use crate::types::check_order::{InferredInCheckOrder, infer_scope_in_check_order}; +use crate::types::context::InferContext; +use crate::types::diagnostic::{ + ERASED_TYPE_CHECK, TypeCheckDiagnostics, report_erased_type_check, + report_unspecialized_reified_call, report_unspecialized_reified_construction, +}; +use crate::types::infer::infer_scope_types; +use crate::types::reified_infer::{ + ParametricIsPlan, ReifiedInferenceError, WrittenTarget, type_test_plan, written_unbound_read, +}; +use crate::types::runtime_spelling::{RuntimeSite, first_unbound_read}; +use crate::types::soundness::{CastCheck, cast_check_plan, cast_is_redundant}; +use crate::types::{ProgramEnvironment, TypeContext}; +use crate::{Db, FxIndexMap, FxIndexSet}; + +/// a site inference found that writes module-level names into code that runs, and whether +/// the inference that found it reports diagnostics there +#[derive(Debug, Clone, PartialEq, Eq, get_size2::GetSize)] +pub(crate) struct RuntimeReadCheck { + pub(crate) site: RuntimeReadSite, + pub(crate) reports: bool, +} + +/// what writes the names, by the nodes it is lowered from +#[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] +pub(crate) enum RuntimeReadSite { + /// `value is target`, a test inference found checkable wherever the names hold + TypeTest { value: NodeIndex, target: NodeIndex }, + /// `value cast! target` or `value cast? target` + CheckedCast { value: NodeIndex, target: NodeIndex }, + /// a bare call or construction of a reified generic, with the module-level names the + /// specialization the lowering injects reads + Reified { + call: NodeIndex, + generic: ReifiedGeneric, + reads: Box<[Name]>, + }, +} + +/// the reified generic a bare call specializes +#[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] +pub(crate) enum ReifiedGeneric { + /// the function called, by name + Function(Name), + /// the class constructed, by name + Class(Name), +} + +/// what [`runtime_read_checks`] found for one scope +#[derive(Debug, Default, PartialEq, Eq, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) struct RuntimeReadChecks { + /// a site whose names do not hold the bindings the checker resolved + pub(crate) diagnostics: TypeCheckDiagnostics, + /// the module-level names the type tests and casts are written with, which the source + /// itself need not read + pub(crate) module_reads: Box<[Name]>, +} + +/// [`runtime_read_checks`] of every scope of a checked `file` whose inference recorded a site, +/// which most scopes have none of +pub(crate) fn file_runtime_read_checks<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, +) -> impl Iterator { + semantic_index(db, file) + .scope_ids() + .filter(move |scope| { + !scope.accepts_type_context(db) + && infer_scope_types(db, *scope, TypeContext::default()) + .diagnostics() + .is_some_and(|inferred| !inferred.runtime_read_checks().is_empty()) + }) + .map(move |scope| runtime_read_checks(db, scope)) +} + +/// checks every site the inference of `scope` recorded ([`RuntimeReadCheck`]), once `scope` +/// and every scope around it are inferred +#[salsa::tracked(returns(ref), heap_size = ruff_memory_usage::heap_size)] +pub(crate) fn runtime_read_checks<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> RuntimeReadChecks { + let (inferred, in_check_order) = infer_scope_in_check_order(db, scope); + let Some(inferred) = inferred.diagnostics() else { + return RuntimeReadChecks::default(); + }; + // a site inferred more than once is checked once, and reported if any inference of it + // would have reported there + let mut sites: FxIndexMap<&RuntimeReadSite, bool> = FxIndexMap::default(); + for check in inferred.runtime_read_checks() { + *sites.entry(&check.site).or_default() |= check.reports; + } + if sites.is_empty() { + return RuntimeReadChecks::default(); + } + + let file = scope.program_file(db); + let parsed = parsed_module(db, file.python_file(db)).load(db); + let env = ProgramEnvironment::from_file(file); + let model = SemanticModel::new(db, file); + let mut checker = RuntimeReadChecker { + db, + env: &env, + file: file.file(db), + parsed: &parsed, + model: &model, + in_check_order, + contexts: FxIndexMap::default(), + module_reads: FxIndexSet::default(), + }; + for (site, reports) in sites { + checker.check(site, reports); + } + let mut diagnostics = TypeCheckDiagnostics::default(); + for context in checker.contexts.into_values() { + diagnostics.extend(&context.finish()); + } + diagnostics.shrink_to_fit(); + RuntimeReadChecks { + diagnostics, + module_reads: checker.module_reads.into_iter().collect(), + } +} + +/// checks the sites one scope's inference recorded +struct RuntimeReadChecker<'a, 'db> { + db: &'db dyn Db, + env: &'a ProgramEnvironment<'db>, + file: File, + parsed: &'a ParsedModuleRef, + model: &'a SemanticModel<'db>, + in_check_order: InferredInCheckOrder<'db>, + /// a context per scope a site is in, which decides whether a diagnostic there is reported + contexts: FxIndexMap>, + module_reads: FxIndexSet, +} + +impl<'a, 'db> RuntimeReadChecker<'a, 'db> { + fn expression(&self, index: NodeIndex) -> Option<&'a ast::Expr> { + match self.parsed.get_by_index(index) { + AnyRootNodeRef::Expr(expr) => Some(expr), + _ => None, + } + } + + fn context(&mut self, node: &ast::Expr) -> Option<&InferContext<'db, 'a>> { + let file = self.model.program_file(); + let scope = semantic_index(self.db, file).try_expression_scope_id(node)?; + let (db, env, source_file, parsed) = (self.db, self.env, self.file, self.parsed); + Some(self.contexts.entry(scope).or_insert_with(|| { + InferContext::new( + db, + env, + scope.to_scope_id(db, file), + source_file, + file, + parsed, + ) + })) + } + + fn site(&self, node: &ast::Expr) -> RuntimeSite<'db> { + RuntimeSite::at(node, self.in_check_order) + } + + fn check(&mut self, site: &RuntimeReadSite, reports: bool) { + match site { + RuntimeReadSite::TypeTest { value, target } => { + if let (Some(value), Some(target)) = + (self.expression(*value), self.expression(*target)) + { + self.type_test(value, target, reports); + } + } + RuntimeReadSite::CheckedCast { value, target } => { + if let (Some(value), Some(target)) = + (self.expression(*value), self.expression(*target)) + { + self.checked_cast(value, target, reports); + } + } + RuntimeReadSite::Reified { + call, + generic, + reads, + } => { + if reports && let Some(call) = self.expression(*call) { + self.reified(call, generic, reads); + } + } + } + } + + /// a bare reified call or construction whose specialization reads a name that does not + /// hold, where the call runs, the binding the checker resolved + fn reified(&mut self, call: &ast::Expr, generic: &ReifiedGeneric, reads: &[Name]) { + let Some((name, binding)) = first_unbound_read(self.db, self.env, reads, self.site(call)) + else { + return; + }; + let failure = ReifiedInferenceError::NotAtRuntime(name, binding); + let Some(context) = self.context(call) else { + return; + }; + match generic { + ReifiedGeneric::Function(callee) => { + report_unspecialized_reified_call(context, call.range(), callee, Some(failure)); + } + ReifiedGeneric::Class(class) => { + report_unspecialized_reified_construction(context, call.range(), class, failure); + } + } + } + + fn type_test(&mut self, value: &ast::Expr, target: &'a ast::Expr, reports: bool) { + let (Some(value_ty), Some(target_ty)) = ( + value.inferred_type(self.model), + target.inferred_type(self.model), + ) else { + return; + }; + let written = WrittenTarget::checked(target, self.in_check_order); + let plan = type_test_plan( + self.db, + self.env, + self.file, + value_ty, + target_ty, + Some(written), + ); + self.read_spellings(&plan); + if reports && let ParametricIsPlan::ErasedTarget(reason) = plan { + let source = source_text(self.db, self.file); + let primary = TextRange::new(value.start(), target.end()); + if let Some(context) = self.context(target) { + report_erased_type_check(context, primary, &source[target.range()], reason); + } + } + } + + /// a checked cast's runtime check is the type test the cast shares with `is`, falling + /// back to the shallow target the soundness checks use, so both are read as spelled + fn checked_cast(&mut self, value: &ast::Expr, target: &'a ast::Expr, reports: bool) { + let (Some(value_ty), Some(target_ty)) = ( + value.inferred_type(self.model), + target.inferred_type(self.model), + ) else { + return; + }; + let (db, env, file) = (self.db, self.env, self.file); + let written = WrittenTarget::checked(target, self.in_check_order); + let plan = type_test_plan(db, env, file, value_ty, target_ty, Some(written)); + self.read_spellings(&plan); + let check = cast_check_plan(db, env, file, target_ty, Some(self.site(target))); + if let Some(spelling) = check.as_ref().and_then(CastCheck::spelling) { + self.module_reads.extend(spelling.global_reads().cloned()); + } + // a cast with no runtime check the checker can spell tests its value against the + // target as written, so a target reading a name that does not hold there the binding + // the checker resolved — a class imported only under `if TYPE_CHECKING:` — would + // raise `NameError`, or test against another class. a cast that needs no check (a + // widening) is a plain `typing.cast`, whose target is written as a string + if !reports || check.is_some() || cast_is_redundant(db, env, value_ty, target_ty) { + return; + } + let Some((name, binding)) = written_unbound_read(db, env, target, self.in_check_order) + else { + return; + }; + let Some(builder) = self + .context(target) + .and_then(|context| context.report_lint(&ERASED_TYPE_CHECK, target)) + else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{}` cannot be checked at runtime: {}", + target_ty.display(db, env), + binding.why_not_checkable(&name) + )); + diagnostic.info(binding.describe(&name)); + } + + fn read_spellings(&mut self, plan: &ParametricIsPlan) { + for spelling in plan.spellings() { + self.module_reads.extend(spelling.global_reads().cloned()); + } + } +} diff --git a/crates/ty_python_semantic/src/types/runtime_spelling.rs b/crates/ty_python_semantic/src/types/runtime_spelling.rs new file mode 100644 index 0000000000..3829cf4660 --- /dev/null +++ b/crates/ty_python_semantic/src/types/runtime_spelling.rs @@ -0,0 +1,212 @@ +//! basedpython: the python the checker spells for the transpiled program +//! +//! a lowering that writes a type out at runtime — a soundness check's `isinstance` +//! target, the specialization a reified generic is called with, a type test's probe — +//! asks the checker how the type is spelled, and the checker names each class by the +//! binding the module's global scope reaches it through. a module-level `private` +//! symbol is not emitted under that binding's name, though: `private class Kept` is +//! emitted as `_Kept`, or under another name where the module already has that one, and +//! only the transpiler decides which. so a spelling keeps the names of those symbols +//! apart, and whoever writes the spelling out spells them +//! +//! every binding of the module's global scope a spelling reads is kept apart the same way, +//! so the checker, which spells the same thing while it checks, can record that the +//! transpiled program reads it — an import read only by a lowering is still used + +use ruff_db::files::File; +use ruff_python_ast::name::Name; +use ruff_text_size::{Ranged, TextRange}; + +use crate::Db; +use crate::place::{RuntimeBinding, RuntimeRead, runtime_binding}; +use crate::types::ProgramEnvironment; +use crate::types::check_order::InferredInCheckOrder; +use crate::types::visibility::private_symbols; + +/// where a lowering writes a spelling into code that runs, which decides what each name it +/// reads holds there. what a name holds there is asked only once the scopes around the code +/// are inferred, so a site is made only with the proof that they are +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RuntimeSite<'db> { + range: TextRange, + inferred: InferredInCheckOrder<'db>, +} + +impl<'db> RuntimeSite<'db> { + /// the code `node` is written as, in the file `inferred` is for + pub(crate) fn at(node: &impl Ranged, inferred: InferredInCheckOrder<'db>) -> Self { + Self { + range: node.range(), + inferred, + } + } + + pub(crate) fn range(self) -> TextRange { + self.range + } + + pub(crate) fn inferred(self) -> InferredInCheckOrder<'db> { + self.inferred + } +} + +/// a python expression the checker spells for the transpiled program, which evaluates in +/// the module it was spelled for to the runtime object of a type (`list[int]`) or to a +/// value (`Color.RED`) +/// +/// each module-level `private` symbol it reads is kept apart, by the name the source +/// declares it under, for [`Self::spelled`] to write as the transpiled module binds it +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct RuntimeSpelling { + /// never two [`SpellingPart::Text`] in a row and never an empty one, so two spellings + /// of the same expression compare equal however they were put together + parts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum SpellingPart { + Text(String), + /// a binding of the module's global scope, written under its own name + Global(Name), + /// a module-level `private` symbol, by the name the source declares it under + ModulePrivate(Name), +} + +impl RuntimeSpelling { + /// a spelling that reads no module-level `private` symbol + pub(crate) fn text(text: impl Into) -> Self { + let mut spelling = Self::default(); + spelling.push_str(&text.into()); + spelling + } + + /// `name` as read through `file`'s global scope, kept apart when it is one of the + /// module's `private` symbols and written as it is otherwise + /// + /// the caller has resolved `name` to a binding of `file`'s own global scope. a name + /// that reaches a builtin because the module leaves it unbound is spelled with + /// [`Self::text`] instead: the module may declare a `private` symbol of that name + /// without binding it, and the builtin is not that symbol + pub(crate) fn global(db: &dyn Db, file: File, name: &Name) -> Self { + let part = if private_symbols(db, file).contains(name) { + SpellingPart::ModulePrivate(name.clone()) + } else { + SpellingPart::Global(name.clone()) + }; + Self { parts: vec![part] } + } + + /// every binding of the module's global scope the spelling reads, by the name the + /// source binds it under + pub(crate) fn global_reads(&self) -> impl Iterator { + self.parts.iter().filter_map(|part| match part { + SpellingPart::Global(name) | SpellingPart::ModulePrivate(name) => Some(name), + SpellingPart::Text(_) => None, + }) + } + + /// every binding of the module's global scope the spelling reads, each with the spelling + /// of that read alone + pub(crate) fn global_read_spellings(&self) -> impl Iterator { + self.parts.iter().filter_map(|part| match part { + SpellingPart::Global(name) | SpellingPart::ModulePrivate(name) => Some(( + name, + Self { + parts: vec![part.clone()], + }, + )), + SpellingPart::Text(_) => None, + }) + } + + /// the first module-level name the spelling reads that does not hold, where `site` runs, + /// the binding the checker resolved — with what it holds instead + pub(crate) fn unbound_read<'db>( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + site: RuntimeSite<'db>, + ) -> Option<(Name, RuntimeBinding)> { + first_unbound_read(db, env, self.global_reads(), site) + } + + pub(crate) fn push_str(&mut self, text: &str) { + if text.is_empty() { + return; + } + if let Some(SpellingPart::Text(last)) = self.parts.last_mut() { + last.push_str(text); + } else { + self.parts.push(SpellingPart::Text(text.to_owned())); + } + } + + pub(crate) fn push(&mut self, other: &Self) { + for part in &other.parts { + match part { + SpellingPart::Text(text) => self.push_str(text), + SpellingPart::Global(_) | SpellingPart::ModulePrivate(_) => { + self.parts.push(part.clone()); + } + } + } + } + + /// `spellings`, each after the first preceded by `separator` + pub(crate) fn join(spellings: impl IntoIterator, separator: &str) -> Self { + let mut joined = Self::default(); + for (index, spelling) in spellings.into_iter().enumerate() { + if index > 0 { + joined.push_str(separator); + } + joined.push(&spelling); + } + joined + } + + pub(crate) fn is_empty(&self) -> bool { + self.parts.is_empty() + } + + /// the expression as the transpiled module reads it, each module-level `private` + /// symbol written under the name `module_private` gives it + pub fn spelled(&self, module_private: impl Fn(&str) -> String) -> String { + let mut spelled = String::new(); + for part in &self.parts { + match part { + SpellingPart::Text(text) => spelled.push_str(text), + SpellingPart::Global(name) => spelled.push_str(name), + SpellingPart::ModulePrivate(name) => spelled.push_str(&module_private(name)), + } + } + spelled + } + + /// the expression as the source reads it, each symbol by the name the source declares + /// it under — for a diagnostic, and for source a later transpile phase renames itself + pub fn source(&self) -> String { + self.spelled(ToOwned::to_owned) + } +} + +/// the first of the module-level names `reads` that does not hold, where `site` runs, the +/// binding the checker resolved — with what it holds instead +pub(crate) fn first_unbound_read<'a, 'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + reads: impl IntoIterator, + site: RuntimeSite<'db>, +) -> Option<(Name, RuntimeBinding)> { + reads.into_iter().find_map(|name| { + let binding = runtime_binding( + db, + env, + RuntimeRead::Global { + name, + range: site.range, + }, + site.inferred, + ); + (binding != RuntimeBinding::Bound).then(|| (name.clone(), binding)) + }) +} diff --git a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs index c976ecf25f..047ff4b7b3 100644 --- a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs +++ b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs @@ -38,19 +38,28 @@ //! reason, we group literal types in `UnionBuilder`. Since every different string literal type //! shares exactly the same possible super-types, and none of them are subtypes of each other //! (unless exactly the same literal type), we can avoid many unnecessary redundancy checks. +//! +//! basedpython: the same holds one step removed, for a method bound to a literal such as +//! `"a".lower`. looking up an attribute on a large literal union gives a union of as many bound +//! methods, one per receiver (`kind.name.lower` for an enum `kind`, whose `name` is a string +//! literal per member). two methods that differ only in their receiver are redundant with each +//! other only if their receivers are, which two different literals never are, so these methods +//! are grouped by everything but their receiver; see [`LiteralReceiverMethod`] use std::hint::cold_path; use super::RecursivelyDefined; use super::generic_gradual_intersections::{GenericIntersection, generic_gradual_intersection}; use crate::types::enums::EnumComplement; +use crate::types::function::FunctionType; use crate::types::regex; use crate::types::set_theoretic::expand_intersection_typevars_and_newtypes; use crate::types::visitor::any_over_type; use crate::types::{ - BytesLiteralType, ClassLiteral, EnumLiteralType, IntersectionType, KnownClass, - KnownInstanceType, LiteralValueType, LiteralValueTypeKind, NegativeIntersectionElements, - StringLiteralType, SubclassOfType, Type, TypePair, TypeVarBoundOrConstraints, UnionType, + BytesLiteralType, ClassLiteral, EnumLiteralType, IntersectionType, KnownBoundMethodType, + KnownClass, KnownInstanceType, LiteralValueType, LiteralValueTypeKind, + NegativeIntersectionElements, StringLiteralType, SubclassOfType, Type, TypePair, + TypeVarBoundOrConstraints, UnionType, }; use crate::{Db, FxOrderMap, FxOrderSet, ProgramEnvironment}; use rustc_hash::FxHashSet; @@ -309,6 +318,42 @@ impl<'db> Type<'db> { } } +/// basedpython: a method bound to a literal that holds exactly one value, without its receiver +/// +/// two bound methods with the same [`LiteralReceiverMethod`] are related only through their +/// receivers: a bound method is a subtype of another only if its receiver is a subtype of the +/// other's (see `check_bound_method_pair` and `check_known_bound_method_pair`). two different +/// literals that each hold one value are never redundant with each other (see +/// `LiteralValueTypeKind::holds_one_value`), so neither are two such methods with different +/// receivers, and the union builder compares them by receiver instead of by relation +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum LiteralReceiverMethod<'db> { + /// a function bound to its receiver, as in `"a".lower` + Function(FunctionType<'db>), + /// `str.startswith` bound to a string literal, as in `"a".startswith` + StrStartswith, +} + +impl<'db> LiteralReceiverMethod<'db> { + /// split `ty` into the method and the literal it is bound to, when it is a method bound to + /// a literal that holds exactly one value + fn of(db: &'db dyn Db, ty: Type<'db>) -> Option<(Self, LiteralValueTypeKind<'db>)> { + let (method, receiver) = match ty { + Type::BoundMethod(method) => { + let Type::LiteralValue(receiver) = method.self_instance(db) else { + return None; + }; + (Self::Function(method.function(db)), receiver.kind()) + } + Type::KnownBoundMethod(KnownBoundMethodType::StrStartswith(receiver)) => { + (Self::StrStartswith, LiteralValueTypeKind::String(receiver)) + } + _ => return None, + }; + receiver.holds_one_value().then_some((method, receiver)) + } +} + #[derive(Debug)] enum UnionElement<'db> { Type(Type<'db>), @@ -323,6 +368,12 @@ enum UnionElement<'db> { enum_class: ClassLiteral<'db>, literals: FxOrderMap, bool>, }, + /// basedpython: methods bound to different literals, keyed by the value of the literal each + /// is bound to + LiteralReceiverMethods { + method: LiteralReceiverMethod<'db>, + methods: FxOrderMap, Type<'db>>, + }, } impl<'db> UnionElement<'db> { @@ -333,6 +384,7 @@ impl<'db> UnionElement<'db> { UnionElement::StringLiterals(literals) => literals.len(), UnionElement::BytesLiterals(literals) => literals.len(), UnionElement::EnumLiterals { literals, .. } => literals.len(), + UnionElement::LiteralReceiverMethods { methods, .. } => methods.len(), } } @@ -368,6 +420,9 @@ impl<'db> UnionElement<'db> { .as_nominal_instance() .is_none_or(|instance| instance.class_literal(db, env) != *enum_class), ), + // recovery makes no relation-based simplification, and only an identical + // method could stand for one of these + UnionElement::LiteralReceiverMethods { .. } => ReduceResult::KeepIf(true), UnionElement::Type(_) => unreachable!("ordinary types are handled before recovery"), }; } @@ -469,6 +524,10 @@ impl<'db> UnionElement<'db> { .is_redundant_with(db, env, other_type) } } + UnionElement::LiteralReceiverMethods { methods, .. } => { + methods.retain(|_, method| should_retain_type(*method)); + !methods.is_empty() + } UnionElement::Type(_) => unreachable!("ordinary types are handled before reduction"), }; @@ -636,7 +695,7 @@ impl<'db> UnionBuilder<'db> { let (enum_literal, _) = literals.first().unwrap(); replace_with.push(enum_literal.enum_class_instance(db, &self.env)); } - UnionElement::Type(_) => {} + UnionElement::LiteralReceiverMethods { .. } | UnionElement::Type(_) => {} } } for ty in replace_with { @@ -684,7 +743,7 @@ impl<'db> UnionBuilder<'db> { UnionElement::StringLiterals(literals) => acc + literals.len(), UnionElement::BytesLiterals(literals) => acc + literals.len(), UnionElement::EnumLiterals { literals, .. } => acc + literals.len(), - UnionElement::Type(_) => acc, + UnionElement::LiteralReceiverMethods { .. } | UnionElement::Type(_) => acc, }); if should_widen(literals, self.recursively_defined) { self.widen_literal_types(seen_aliases); @@ -1070,7 +1129,57 @@ impl<'db> UnionBuilder<'db> { let mut ty_negated: Option = None; let mut to_remove = SmallVec::<[usize; 2]>::new(); + // basedpython: a method bound to a literal joins the group of methods that differ from + // it only in their receiver, which it needs no relation query against + let original_ty = ty; + let mut receiver_group = LiteralReceiverMethod::of(db, ty); + let mut group_methods = None; + for (i, element) in self.elements.iter_mut().enumerate() { + // the group's methods are handed over for the rest of the push, so the element is + // moved into the match rather than reborrowed, and the test that it is the group is + // made first: a guard would hold a borrow of it into the other arm + let joins_group = receiver_group.is_some_and(|(method, _)| { + matches!( + element, + UnionElement::LiteralReceiverMethods { method: group_method, .. } + if *group_method == method + ) + }); + let element = match (joins_group, receiver_group, element) { + ( + true, + Some((_, receiver)), + UnionElement::LiteralReceiverMethods { methods, .. }, + ) => { + // every other method in the group has a different receiver, so only one bound + // to the same receiver can be redundant with `ty`, or `ty` with it. it can + // differ from `ty`, e.g. bound to the promotable form of the literal where + // `ty` is bound to the unpromotable one, and only the relation can tell which + // of the two stands for the other + match methods.get(&receiver).copied() { + None => {} + Some(existing) if existing == ty => return, + Some(existing) + if !self.cycle_recovery && ty.is_redundant_with(db, env, existing) => + { + return; + } + // `ty` takes the place of the method redundant with it + Some(existing) + if !self.cycle_recovery && existing.is_redundant_with(db, env, ty) => {} + // neither stands for the other, so `ty` is kept as an element of its own + Some(_) => { + receiver_group = None; + continue; + } + } + group_methods = Some(methods); + continue; + } + (_, _, element) => element, + }; + let element_type = match element.try_reduce(db, &self.env, ty, self.cycle_recovery) { ReduceResult::KeepIf(keep) => { if !keep { @@ -1194,15 +1303,34 @@ impl<'db> UnionBuilder<'db> { } } + // the loop can replace `ty` with a merged type, which is not the method it was grouped as + let new_element = match (receiver_group, group_methods) { + (Some((_, receiver)), Some(methods)) if ty == original_ty => { + methods.insert(receiver, ty); + // descending, to keep the remaining indices valid after `swap_remove` + for index in to_remove.into_iter().rev() { + self.elements.swap_remove(index); + } + return; + } + (Some((method, receiver)), None) if ty == original_ty => { + UnionElement::LiteralReceiverMethods { + method, + methods: FxOrderMap::from_iter([(receiver, ty)]), + } + } + _ => UnionElement::Type(ty), + }; + let mut to_remove = to_remove.into_iter(); if let Some(first) = to_remove.next() { - self.elements[first] = UnionElement::Type(ty); + self.elements[first] = new_element; // We iterate in descending order to keep remaining indices valid after `swap_remove`. for index in to_remove.rev() { self.elements.swap_remove(index); } } else { - self.elements.push(UnionElement::Type(ty)); + self.elements.push(new_element); } } @@ -1253,6 +1381,9 @@ impl<'db> UnionBuilder<'db> { ) })); } + UnionElement::LiteralReceiverMethods { methods, .. } => { + types.extend(methods.into_values()); + } // basedpython: a literal without a group of its own — a float or a complex — // is carried as an ordinary element, so it is the only place the union's // `recursively_defined` can be recorded for it. The grouped kinds above are @@ -2169,13 +2300,19 @@ mod tests { RecursivelyDefined, Type, UnionBuilder, UnionType, }; - use crate::db::tests::{TestDb, setup_db}; + use std::fmt::Write as _; + + use crate::db::tests::{TestDb, TestDbBuilder, setup_db}; use crate::place::{global_symbol, known_module_symbol}; use crate::types::enums::enum_member_literals; use crate::types::type_alias::TypeAliasType; - use crate::types::{KnownClass, KnownInstanceType, Truthiness}; + use crate::types::{ + KnownClass, KnownInstanceType, LiteralValueType, LiteralValueTypeKind, Truthiness, + }; use ruff_db::system::DbWithWritableSystem as _; + use ruff_db::testing::assert_function_query_was_not_run_by_name; + use ruff_python_ast::PythonVersion; use ty_module_resolver::KnownModule; use ty_python_core::ProgramFile; @@ -2369,6 +2506,147 @@ mod tests { } } + /// basedpython: a module of enums whose members are literals of their own kind + fn literal_receiver_enums_db() -> TestDb { + let mut source = String::from( + "from enum import Enum, IntEnum, StrEnum\n\n\ + class Plain(Enum):\n ONE = 1\n TWO = 2\n\n\ + class Name(StrEnum):\n ONE = \"1\"\n TWO = \"2\"\n\n\ + class Code(IntEnum):\n", + ); + for index in 0..64 { + writeln!(source, " M{index} = {index}").unwrap(); + } + let mut db = TestDbBuilder::new() + .with_python_version(PythonVersion::PY312) + .build() + .expect("valid TestDb setup"); + db.write_dedented("/src/enums.py", &source).unwrap(); + db + } + + fn enum_members<'db>(db: &'db TestDb, name: &str) -> Vec> { + let file = ruff_db::files::system_path_to_file(db, "/src/enums.py").unwrap(); + let module = ProgramFile::new(db, file, db.program_environment().program(db)); + let class = global_symbol(db, module, name) + .place + .expect_type() + .expect_class_literal(); + enum_member_literals(db, class, None) + .expect("an enum") + .collect() + } + + /// basedpython: a union of methods bound to many different literals, like `kind.name.lower` + /// for a large enum, is built without a single query: in particular no + /// `is_redundant_with_impl` for each pair of methods, which is what it used to cost + #[test] + fn methods_bound_to_literals_need_no_queries() { + let db = literal_receiver_enums_db(); + // a handle on the same database and its event log, which the types below borrow + let mut events_db = db.clone(); + let db = &db; + let env = db.program_environment(); + + let strings: Vec = (0..64) + .map(|index| Type::string_literal(db, format!("m{index}").as_str())) + .collect(); + let ints: Vec = (0..64).map(Type::int_literal).collect(); + let enum_members = enum_members(db, "Code"); + assert_eq!(enum_members.len(), 64); + + for (receivers, method) in [ + (&strings, "lower"), + (&strings, "startswith"), + (&ints, "bit_length"), + (&enum_members, "bit_length"), + ] { + let methods: Vec = receivers + .iter() + .map(|receiver| receiver.member(db, &env, method).place.expect_type()) + .collect(); + events_db.take_salsa_events(); + + let union = UnionType::from_elements(db, &env, methods.iter().copied()); + + let events = events_db.take_salsa_events(); + assert_function_query_was_not_run_by_name(db, "is_redundant_with_impl", None, &events); + let queries: Vec<_> = events + .iter() + .filter(|event| matches!(event.kind, salsa::EventKind::WillExecute { .. })) + .collect(); + assert!(queries.is_empty(), "`{method}` ran {queries:#?}"); + assert_eq!(union.expect_union().elements(db), methods.as_slice()); + } + } + + /// basedpython: two literals that each hold one value are related only when their kinds are + /// equal, the promotable one being redundant with the unpromotable one. grouping methods + /// bound to literals relies on it, see `LiteralValueTypeKind::holds_one_value` + #[test] + fn one_valued_literals_are_related_only_by_kind() { + let db = literal_receiver_enums_db(); + let db = &db; + let env = db.program_environment(); + + // the same value as far as possible, in every kind holding one value + let literals = [ + Type::int_literal(1), + Type::bool_literal(true), + Type::string_literal(db, "1"), + Type::bytes_literal(db, b"1"), + Type::float_literal(1.0), + Type::complex_literal(db, 1.0, 0.0), + enum_members(db, "Plain")[0], + enum_members(db, "Name")[0], + enum_members(db, "Code")[1], + ] + .map(|ty| ty.as_literal_value().expect("a literal").kind()); + + let mut kinds_seen = [false; 7]; + for kind in literals { + assert!(kind.holds_one_value()); + let index = match kind { + LiteralValueTypeKind::Int(_) => 0, + LiteralValueTypeKind::Bool(_) => 1, + LiteralValueTypeKind::String(_) => 2, + LiteralValueTypeKind::Bytes(_) => 3, + LiteralValueTypeKind::Float(_) => 4, + LiteralValueTypeKind::Complex(_) => 5, + LiteralValueTypeKind::Enum(_) => 6, + LiteralValueTypeKind::LiteralString | LiteralValueTypeKind::Template(_) => { + panic!("`{kind:?}` holds many values") + } + }; + kinds_seen[index] = true; + } + assert_eq!( + kinds_seen, [true; 7], + "a kind holding one value is not sampled" + ); + + let forms = |kind| { + [true, false].map(|promotable| Type::from(LiteralValueType::new(kind, promotable))) + }; + for left in literals { + let [left_promotable, left_unpromotable] = forms(left); + assert!(left_promotable.is_redundant_with(db, &env, left_unpromotable)); + assert!(!left_unpromotable.is_redundant_with(db, &env, left_promotable)); + for right in literals.into_iter().filter(|right| *right != left) { + for left_form in forms(left) { + for right_form in forms(right) { + assert!( + !left_form.is_redundant_with(db, &env, right_form), + "{} is redundant with {}", + left_form.display(db, &env), + right_form.display(db, &env), + ); + } + } + } + } + } + #[test] fn union_common_literal_supertype() { let db = setup_db(); diff --git a/crates/ty_python_semantic/src/types/soundness.rs b/crates/ty_python_semantic/src/types/soundness.rs index e549f62eb1..f170fd282a 100644 --- a/crates/ty_python_semantic/src/types/soundness.rs +++ b/crates/ty_python_semantic/src/types/soundness.rs @@ -7,7 +7,7 @@ //! these queries answer the two questions that decide an insertion: //! //! - does this expression's type rest on such an assumption? (the gates: -//! [`call_result_is_typevar_derived`], [`is_specialized_generic_instance`]) +//! `call_result_is_typevar_derived`, `is_specialized_generic_instance`) //! - can the inferred type be validated with `isinstance` at runtime, and //! what second argument does that check take? (`runtime_check_target`) @@ -21,6 +21,7 @@ use crate::types::literal::LiteralValueTypeKind; use crate::types::reified_infer::{ ArgVariance, parametric_soundness_spelling, protocol_structural_members, }; +use crate::types::runtime_spelling::{RuntimeSite, RuntimeSpelling}; use crate::types::signatures::CallableSignature; use crate::types::visitor::any_over_type; use crate::types::{ClassLiteral, ClassType, FunctionType, KnownClass, Type}; @@ -28,39 +29,59 @@ use ty_module_resolver::{KnownModule, file_to_module}; /// The second argument of a shallow `isinstance` soundness check. /// -/// Structured rather than spelled, because the transpiler writes it as python source -/// while the native backend resolves each class and tests against it directly; its -/// [`Display`](std::fmt::Display) is the spelling. +/// structured rather than spelled, because the transpiler writes it as python source +/// while the native backend resolves each class and tests against it directly. +/// [`Self::spelling`] is how it is written out #[derive(Debug, Clone, PartialEq, Eq)] pub enum CheckTarget { /// a class, by the name module scope reaches it through: a binding of the /// module's own, or a builtin the module leaves unbound - Class(String), + Class(RuntimeSpelling), /// `type(None)` NoneType, /// any one of several, which `isinstance` accepts as a tuple AnyOf(Vec), } -impl std::fmt::Display for CheckTarget { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl CheckTarget { + /// the second argument of the `isinstance` call + pub fn spelling(&self) -> RuntimeSpelling { match self { - Self::Class(name) => f.write_str(name), - Self::NoneType => f.write_str("type(None)"), + Self::Class(name) => name.clone(), + Self::NoneType => RuntimeSpelling::text("type(None)"), Self::AnyOf(parts) => { - f.write_str("(")?; - for (index, part) in parts.iter().enumerate() { - if index > 0 { - f.write_str(", ")?; - } - write!(f, "{part}")?; - } - f.write_str(")") + let mut spelling = RuntimeSpelling::text("("); + spelling.push(&RuntimeSpelling::join( + parts.iter().map(Self::spelling), + ", ", + )); + spelling.push_str(")"); + spelling } } } } +impl CheckKind { + /// the spelling the check writes its target with + pub(crate) fn spelling(&self) -> RuntimeSpelling { + match self { + Self::Isinstance(target) => target.spelling(), + Self::Parametric { alias, .. } => alias.clone(), + } + } +} + +impl CastCheck { + /// the spelling the check writes its target with, when it writes one + pub(crate) fn spelling(&self) -> Option { + match self { + Self::Kind(kind) => Some(kind.spelling()), + Self::Protocol | Self::Unchecked | Self::Members(_) => None, + } + } +} + /// How a runtime soundness check validates a value against a target type. #[derive(Debug, Clone, PartialEq, Eq)] pub enum CheckKind { @@ -70,7 +91,10 @@ pub enum CheckKind { /// carries `__orig_class__`, its type arguments. `alias` is the runtime /// spelling of the specialization (`A[int]`); `variances` is one code per /// type parameter (0 invariant, 1 covariant, 2 contravariant, 3 bivariant) - Parametric { alias: String, variances: Vec }, + Parametric { + alias: RuntimeSpelling, + variances: Vec, + }, } /// How a checked `cast` / `cast?` validates its value against the target type. @@ -179,7 +203,28 @@ fn python_string_literal(value: &str) -> String { /// protocol with no faithful check degrades to [`CastCheck::Unchecked`]. This /// keeps the emitted code from ever running `isinstance` against a subscripted /// or non-`@runtime_checkable` protocol, which raises at runtime. -pub fn cast_check_plan<'db>( +/// +/// `site` is the cast's written target, where the check runs: a check whose target reads a +/// name that does not hold there the binding the checker resolved has no plan +pub(crate) fn cast_check_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + ty: Type<'db>, + site: Option>, +) -> Option { + let plan = cast_check_plan_anywhere(db, env, file, ty)?; + if let Some(site) = site + && plan + .spelling() + .is_some_and(|spelling| spelling.unbound_read(db, env, site).is_some()) + { + return None; + } + Some(plan) +} + +fn cast_check_plan_anywhere<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, @@ -203,7 +248,7 @@ pub fn cast_check_plan<'db>( /// whether a checked cast to `target` has no faithful runtime residue and must /// degrade to an unchecked `typing.cast`: a protocol target that -/// [`cast_check_plan`] cannot validate structurally. this is what stops the +/// `cast_check_plan` cannot validate structurally. this is what stops the /// transpiler emitting `isinstance(value, )`, which raises at runtime pub fn cast_target_is_unverifiable_protocol<'db>( db: &'db dyn Db, @@ -212,7 +257,7 @@ pub fn cast_target_is_unverifiable_protocol<'db>( target: Type<'db>, ) -> bool { matches!( - cast_check_plan(db, env, file, target), + cast_check_plan_anywhere(db, env, file, target), Some(CastCheck::Unchecked) ) } @@ -222,7 +267,7 @@ pub fn cast_target_is_unverifiable_protocol<'db>( /// (`def t[T]() -> T`), or the method is bound to a specialized generic /// instance (`dict[str, int].get`), where the specialization itself is an /// unverified annotation-level claim -pub fn call_result_is_typevar_derived<'db>( +pub(crate) fn call_result_is_typevar_derived<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, callee: Type<'db>, @@ -259,7 +304,7 @@ fn mentions_type_var<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Ty /// specialization — the runtime-unverifiable part of an annotation like /// `list[str]`. element projections out of such values (subscripts, method /// results, iteration) are where the specialization's claim is consumed -pub fn is_specialized_generic_instance<'db>( +pub(crate) fn is_specialized_generic_instance<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>, @@ -297,8 +342,8 @@ pub(crate) fn runtime_check_target<'db>( env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, -) -> Option { - target(db, env, file, ty, 0).map(|target| target.to_string()) +) -> Option { + target(db, env, file, ty, 0).map(|target| target.spelling()) } /// The runtime soundness check for a value whose declared type is `ty`. @@ -308,7 +353,7 @@ pub(crate) fn runtime_check_target<'db>( /// arguments are checkable at runtime); otherwise falls back to the shallow /// [`CheckKind::Isinstance`] of `runtime_check_target`. `None` when neither /// applies (no faithful runtime test). -pub fn runtime_check_plan<'db>( +pub(crate) fn runtime_check_plan<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, @@ -337,7 +382,7 @@ pub fn runtime_check_plan<'db>( /// of the bound is recovered from how the body happens to use the parameter, and a /// recovered requirement is not something to raise a `TypeError` over — it is a /// sample, and it can be withdrawn. -pub fn parameter_runtime_check_plan<'db>( +pub(crate) fn parameter_runtime_check_plan<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, @@ -470,19 +515,20 @@ fn target<'db>( /// resolve the runtime name for `literal` as seen from module scope in /// `file`: either the module binds the class's name to this exact class /// (definition or import), or the name is unbound and the class is a -/// builtin so the bare name reaches it +/// builtin so the bare name reaches it. whether the name holds that binding +/// where the check runs is the site's to ask ([`RuntimeSpelling::unbound_read`]) fn class_target<'db>( db: &'db dyn Db, file: File, literal: ClassLiteral<'db>, ) -> Option { - let name = literal.name(db).as_str(); + let name = literal.name(db); match explicit_global_symbol(db, db.program_file(file), name).place { Place::Defined(defined) if defined.ty == Type::ClassLiteral(literal) => { - Some(CheckTarget::Class(name.to_owned())) + Some(CheckTarget::Class(RuntimeSpelling::global(db, file, name))) } Place::Undefined if class_is_builtin(db, literal) => { - Some(CheckTarget::Class(name.to_owned())) + Some(CheckTarget::Class(RuntimeSpelling::text(name.as_str()))) } _ => None, } @@ -497,13 +543,13 @@ fn class_is_builtin<'db>(db: &'db dyn Db, literal: ClassLiteral<'db>) -> bool { /// does not rebind that name fn builtin_target(db: &dyn Db, file: File, name: &str) -> Option { match explicit_global_symbol(db, db.program_file(file), name).place { - Place::Undefined => Some(CheckTarget::Class(name.to_owned())), + Place::Undefined => Some(CheckTarget::Class(RuntimeSpelling::text(name))), Place::Defined(_) => None, } } /// selects which parameter of a call an argument binds to, for -/// [`parameter_check_plan`] +/// `parameter_check_plan` #[derive(Clone, Copy)] pub enum ArgSelector<'a> { /// a positional argument at the given 0-based index (after any bound @@ -520,7 +566,7 @@ pub enum ArgSelector<'a> { /// misleading), a variadic / unmatched / unannotated parameter, or a /// parameter type with no runtime test. deliberately conservative — /// a missed check is a no-op, a wrong one changes semantics -pub fn parameter_check_plan<'db>( +pub(crate) fn parameter_check_plan<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, diff --git a/crates/ty_python_semantic/src/types/soundness_sites.rs b/crates/ty_python_semantic/src/types/soundness_sites.rs new file mode 100644 index 0000000000..bf42f4f961 --- /dev/null +++ b/crates/ty_python_semantic/src/types/soundness_sites.rs @@ -0,0 +1,575 @@ +//! basedpython: where the transpiled program checks a value at runtime against the type +//! the checker gave it +//! +//! the transpiler writes a check at each site this decides on, and a native build of the +//! same module makes the same checks, so neither decides for itself. the checker asks too: +//! a check names its type's classes as the module binds them, and the import binding one is +//! used by the check although the source never spells the read + +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{Comprehension, Expr, ExprCall, Parameter, Stmt, StmtFunctionDef, UnaryOp}; +use ruff_text_size::{Ranged, TextRange}; +use ty_python_core::scope::ScopeKind; +use ty_python_core::semantic_index; + +use crate::Db; +use crate::place::{RuntimeBinding, RuntimeRead, binds_one_class_wherever_bound, runtime_binding}; +use crate::semantic_model::{HasType, LoweringModel}; +use crate::types::check_order::InferredInCheckOrder; +use crate::types::runtime_spelling::{RuntimeSite, RuntimeSpelling}; +use crate::types::soundness::{ + ArgSelector, CheckKind, CheckTarget, call_result_is_typevar_derived, + is_specialized_generic_instance, parameter_check_plan, parameter_runtime_check_plan, + runtime_check_plan, +}; +use crate::types::{DynamicType, KnownInstanceType, ProgramEnvironment, Type}; + +/// every runtime soundness check `suite` makes under `positions`, in the order the +/// checks are found +pub fn soundness_sites<'ast>( + model: &LoweringModel<'_>, + suite: &'ast [Stmt], + positions: SoundnessPositions, +) -> Vec> { + if !positions.any() { + return Vec::new(); + } + let mut walker = Soundness::new(model, positions); + walker.visit_body(suite); + walker.sites +} + +/// Per-position toggles for the runtime type-soundness checks. Each field +/// enables validation at one syntactic position where ty accepts a value on +/// an annotation-level claim it can't verify at runtime. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[expect( + clippy::struct_excessive_bools, + reason = "independent per-position toggles, not a state machine" +)] +pub struct SoundnessPositions { + /// results of calls whose type is typevar-derived — a generic function's + /// return (`t[T]() -> T`) or a method bound to a specialized generic + /// instance (`dict[str, int].get`) + pub generic_calls: bool, + /// element reads out of a specialized container (`a[0]` on `list[str]`) + pub projections: bool, + /// loop / comprehension elements drawn from a specialized iterable + pub iterations: bool, + /// explicit `Any` (or a context-solved generic result) flowing into an + /// annotated assignment target (`a: str = any_val`) + pub assignments: bool, + /// a returned value validated against the enclosing function's declared + /// return type (`def g() -> str: return any_val`) + pub returns: bool, + /// a call argument validated against its matched parameter's annotation + /// (`takes(any_val)` where `takes(s: str)`) + pub arguments: bool, + /// a function's own parameters validated against their annotations at + /// entry, defending its contract against callers the checker never saw + /// (untyped or third-party code). off in the default set — it runs on + /// every call, so it's opt-in — but included in [`Self::all`] + pub parameters: bool, +} + +impl SoundnessPositions { + /// every position enabled, including the opt-in defensive `parameters` + /// entry checks (the CLI `--soundness all`) + pub fn all() -> Self { + Self { + parameters: true, + ..Self::defaults() + } + } + + /// the positions enabled without an explicit request: every inference-gap + /// check, but *not* the defensive `parameters` entry checks (those add a + /// check on every call, so they stay opt-in) + pub fn defaults() -> Self { + Self { + generic_calls: true, + projections: true, + iterations: true, + assignments: true, + returns: true, + arguments: true, + parameters: false, + } + } + + /// no position enabled — the pass is a no-op + pub fn none() -> Self { + Self { + generic_calls: false, + projections: false, + iterations: false, + assignments: false, + returns: false, + arguments: false, + parameters: false, + } + } + + /// whether any position is enabled + pub fn any(self) -> bool { + self.generic_calls + || self.projections + || self.iterations + || self.assignments + || self.returns + || self.arguments + || self.parameters + } +} + +/// which parameter an argument binds to, for the `arguments` gate +enum ArgSlot<'a> { + Positional(usize), + Keyword(&'a str), +} + +/// one runtime soundness check the transpiled program makes, at the syntax node it is +/// made at +#[derive(Debug, Clone)] +pub enum SoundnessSite<'ast> { + /// the value `expr` produces is checked as it is produced + Value { expr: &'ast Expr, check: SiteCheck }, + /// each element drawn from `iterable` is checked as it is drawn + Elements { + iterable: &'ast Expr, + is_async: bool, + check: SiteCheck, + }, + /// `function`'s own parameters are checked where its body begins, each in the order + /// it is written + Parameters { + function: &'ast StmtFunctionDef, + guards: Vec<(&'ast Parameter, SiteCheck)>, + }, +} + +impl SoundnessSite<'_> { + /// every check the site makes + pub(crate) fn plans(&self) -> Vec<&CheckKind> { + match self { + Self::Value { check, .. } | Self::Elements { check, .. } => vec![&check.plan], + Self::Parameters { guards, .. } => { + guards.iter().map(|(_, check)| &check.plan).collect() + } + } + } +} + +/// a check a site makes, and the module-level names it reads that the program may not have +/// bound yet when it runs +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SiteCheck { + pub plan: CheckKind, + /// each name the check's target reads that is bound only further down the module, or + /// only on some paths, as a spelling of its own. every binding the module makes of one + /// binds the class the checker resolved, so once all of them are bound the target is the + /// one the checker named; the check is made only then, and the value passes unchecked + /// before. empty for a check whose names are bound wherever it runs + pub once_bound: Vec, +} + +/// where the checks go, decided once over a syntax tree, and what is not a place +/// for one +struct Soundness<'a, 'db, 'ast> { + model: &'a LoweringModel<'db>, + /// what a name the checks are written with holds is asked once the file is inferred + inferred: InferredInCheckOrder<'db>, + positions: SoundnessPositions, + sites: Vec>, + /// gated expressions already covered by a wrap on an enclosing `!` + /// force-unwrap — wrapping them directly would splice the check's second + /// argument into `_force_unwrap`'s call parens + consumed: Vec, + /// declared-return-type check plans of the enclosing functions (innermost + /// last); `None` when a function has no annotation or an uncheckable one + return_targets: Vec>, +} + +impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { + fn new(model: &'a LoweringModel<'db>, positions: SoundnessPositions) -> Self { + Self { + model, + inferred: model.inferred_in_check_order(), + positions, + sites: Vec::new(), + consumed: Vec::new(), + return_targets: Vec::new(), + } + } + + fn db(&self) -> &'db dyn Db { + self.model.db() + } + + fn env(&self) -> ProgramEnvironment<'db> { + self.model.program_environment() + } + + fn call_result_is_typevar_derived(&self, callee: &Expr) -> bool { + callee + .inferred_type(self.model) + .is_some_and(|ty| call_result_is_typevar_derived(self.db(), &self.env(), ty)) + } + + fn is_specialized_generic_instance(&self, expr: &Expr) -> bool { + expr.inferred_type(self.model) + .is_some_and(|ty| is_specialized_generic_instance(self.db(), &self.env(), ty)) + } + + fn is_any(&self, expr: &Expr) -> bool { + expr.inferred_type(self.model) + .is_some_and(|ty| matches!(ty, Type::Dynamic(DynamicType::Any))) + } + + fn is_field_specifier(&self, expr: &Expr) -> bool { + matches!( + expr.inferred_type(self.model), + Some(Type::KnownInstance(KnownInstanceType::Field(_))) + ) + } + + /// the check for the parameter `selector` binds to in a call through `callee` + fn parameter_check_plan_for( + &self, + callee: &Expr, + selector: ArgSelector<'_>, + ) -> Option { + let ty = callee.inferred_type(self.model)?; + parameter_check_plan(self.db(), &self.env(), self.model.file(), ty, selector) + } + + /// whether `expr` is a value this pass classifies as resting on an + /// unverified claim: a call whose result type came from a typevar + /// solution, or an element read out of a specialized container. this is + /// the position-agnostic *classification*; the direct-wrap sites also + /// require the corresponding position to be enabled ([`Self::gated_enabled`]) + fn gated(&self, expr: &Expr) -> bool { + match expr { + Expr::Call(call) => self.call_result_is_typevar_derived(&call.func), + Expr::Subscript(subscript) => { + subscript.ctx.is_load() && self.is_specialized_generic_instance(&subscript.value) + } + _ => false, + } + } + + /// [`Self::gated`] restricted to the enabled positions: a `Call` needs + /// `generic_calls`, a `Subscript` needs `projections` + fn gated_enabled(&self, expr: &Expr) -> bool { + match expr { + Expr::Call(_) => self.positions.generic_calls && self.gated(expr), + Expr::Subscript(_) => self.positions.projections && self.gated(expr), + _ => false, + } + } + + /// whether `value` is "unhelpful" — its own inferred type doesn't back a + /// standalone check, so a surrounding annotation (assignment / return / + /// parameter) must supply the target: a plain `Any`, or a gated value + /// whose type is unresolved (a typevar solved only by that context) + fn value_needs_context_target(&self, value: &Expr) -> bool { + self.is_any(value) || (self.gated(value) && self.check_plan(value).is_none()) + } + + /// the check plan for `expr`'s inferred type, with the pure-`None` + /// isinstance noise case dropped (validating a `None` result guards nothing) + fn check_plan(&self, expr: &Expr) -> Option { + let ty = expr.inferred_type(self.model)?; + runtime_check_plan(self.db(), &self.env(), self.model.file(), ty) + .filter(|plan| !matches!(plan, CheckKind::Isinstance(CheckTarget::NoneType))) + } + + /// [`Self::check_plan`] for a parameter, read off the parameter rather than off + /// its annotation — which is the only way to see a type the source stated with a + /// default instead of with an annotation + fn parameter_plan(&self, parameter: &Parameter) -> Option { + let ty = parameter.inferred_type(self.model)?; + parameter_runtime_check_plan(self.db(), &self.env(), self.model.file(), ty) + .filter(|plan| !matches!(plan, CheckKind::Isinstance(CheckTarget::NoneType))) + } + + /// the check `plan` makes at `site`, if it can make one there: every name its target + /// reads must hold, where `site` runs, the binding the checker resolved. a check that would + /// read one that does not — a class imported only under `if TYPE_CHECKING:`, or declared + /// further on — is left out, as a check with no faithful target is. + /// + /// code in a function runs whenever the function is called, which can be before or after + /// the module binds a class declared further down. there a name the module binds only to + /// one class, and never to anything else, is looked up as the check runs instead + /// ([`SiteCheck::once_bound`]), and the check is made once it is bound + fn site_check( + &self, + plan: CheckKind, + site: RuntimeSite<'db>, + runs_later: bool, + ) -> Option { + let db = self.db(); + let file = self.model.file(); + let program_file = db.program_file(file); + let mut once_bound = Vec::new(); + for (name, read) in plan.spelling().global_read_spellings() { + match runtime_binding( + db, + &self.env(), + RuntimeRead::Global { + name, + range: site.range(), + }, + site.inferred(), + ) { + RuntimeBinding::Bound => {} + RuntimeBinding::Unbound | RuntimeBinding::NotTheCheckers + if runs_later && binds_one_class_wherever_bound(db, program_file, name) => + { + once_bound.push(read); + } + RuntimeBinding::Unbound | RuntimeBinding::NotTheCheckers => return None, + } + } + Some(SiteCheck { plan, once_bound }) + } + + /// whether `expr` runs only once a function or a lambda holding it is called, rather than + /// as the module or a class body reaches it + fn runs_later(&self, expr: &Expr) -> bool { + let index = semantic_index(self.db(), self.db().program_file(self.model.file())); + index.try_expression_scope_id(expr).is_some_and(|scope| { + index + .ancestor_scopes(scope) + .any(|(_, scope)| matches!(scope.kind(), ScopeKind::Function | ScopeKind::Lambda)) + }) + } + + /// check the value `expr` produces against `plan` + fn check_value(&mut self, expr: &'ast Expr, plan: CheckKind) { + let Some(check) = self.site_check( + plan, + RuntimeSite::at(expr, self.inferred), + self.runs_later(expr), + ) else { + return; + }; + self.sites.push(SoundnessSite::Value { expr, check }); + } + + /// check each element drawn from the iterable of a `for` / comprehension + /// clause when the iterable carries a generic specialization and the element + /// (loop target) type is checkable + fn check_iteration(&mut self, iterable: &'ast Expr, target_expr: &Expr, is_async: bool) { + if !self.is_specialized_generic_instance(iterable) { + return; + } + let Some(plan) = self.check_plan(target_expr) else { + return; + }; + let Some(check) = self.site_check( + plan, + RuntimeSite::at(iterable, self.inferred), + self.runs_later(iterable), + ) else { + return; + }; + self.sites.push(SoundnessSite::Elements { + iterable, + is_async, + check, + }); + } + + /// check each argument of `call` whose own type is unhelpful against its + /// matched parameter's annotation. positional mapping stops at the first + /// starred spread (positions past it are unknown); `**kwargs` spreads are + /// skipped + fn check_call_arguments(&mut self, call: &'ast ExprCall) { + let callee = call.func.as_ref(); + for (index, arg) in call.arguments.args.iter().enumerate() { + if arg.is_starred_expr() { + break; + } + self.maybe_check_argument(callee, arg, &ArgSlot::Positional(index)); + } + for keyword in &call.arguments.keywords { + if let Some(name) = &keyword.arg { + self.maybe_check_argument(callee, &keyword.value, &ArgSlot::Keyword(name.as_str())); + } + } + } + + fn maybe_check_argument(&mut self, callee: &Expr, arg: &'ast Expr, slot: &ArgSlot<'_>) { + if self.consumed.contains(&arg.range()) || !self.value_needs_context_target(arg) { + return; + } + let plan = match *slot { + ArgSlot::Positional(index) => { + self.parameter_check_plan_for(callee, ArgSelector::Positional(index)) + } + ArgSlot::Keyword(name) => { + self.parameter_check_plan_for(callee, ArgSelector::Keyword(name)) + } + }; + if let Some(plan) = plan { + self.check_value(arg, plan); + } + } + + /// the plan to validate a returned `value` against — the enclosing + /// function's declared return plan — but only when `value`'s own type + /// is unhelpful (else `generic_calls`/`projections` already covers it) + fn return_plan(&self, value: &Expr) -> Option { + let plan = self.return_targets.last()?.clone()?; + self.value_needs_context_target(value).then_some(plan) + } + + /// check each checkable parameter of `func` where its body begins — the + /// `parameters` position, defending the contract against callers the checker + /// never saw. variadic (`*args` / `**kwargs`) parameters are skipped, and so is + /// any parameter whose source states no type: an unannotated one with no + /// default states nothing, and `x=None` says the argument may be left out rather + /// than that `None` belongs there + /// + /// the plan is read off the *parameter*, not off its annotation, because a + /// default is a written type too. `def f(safe='/')` says `safe` is a `str` + /// — the native backend lays the parameter out at that bound and checks it + /// at the boundary, and asking the annotation node left the interpreted twin + /// silently more permissive than its own compiled form + fn check_parameters(&mut self, func: &'ast StmtFunctionDef) { + let params = &func.parameters; + let mut guards = Vec::new(); + for pwd in params + .posonlyargs + .iter() + .chain(¶ms.args) + .chain(¶ms.kwonlyargs) + { + let parameter = &pwd.parameter; + // the guards run where the function's body begins, when it is called + if let Some(plan) = self.parameter_plan(parameter) + && let Some(check) = + self.site_check(plan, RuntimeSite::at(func, self.inferred), true) + { + guards.push((parameter, check)); + } + } + if !guards.is_empty() { + self.sites.push(SoundnessSite::Parameters { + function: func, + guards, + }); + } + } +} + +impl<'ast> Visitor<'ast> for Soundness<'_, '_, 'ast> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + match stmt { + Stmt::FunctionDef(func) => { + if self.positions.parameters { + self.check_parameters(func); + } + // track the declared return plan so nested `return`s validate + // against the right function's annotation + let plan = func + .returns + .as_deref() + .and_then(|returns| self.check_plan(returns)); + self.return_targets.push(plan); + walk_stmt(self, stmt); + self.return_targets.pop(); + return; + } + Stmt::For(for_stmt) => { + if self.positions.iterations { + self.check_iteration(&for_stmt.iter, &for_stmt.target, for_stmt.is_async); + } + } + Stmt::AnnAssign(ann) => { + // the annotation node's stored type is the declared type; + // it backs two checks the value's own type can't: + // - an explicit `Any` flowing into the declared binding + // - a gated value whose typevar solution came from this very + // declaration (`a: str = t()` stores `Unknown` for `t()` — + // the context, not the expression, carries the claim) + if self.positions.assignments + && let Some(value) = &ann.value + && self.value_needs_context_target(value) + // a field-specifier RHS (`a: int = Field()`) is modelled as + // the field type but is a descriptor object at runtime, so a + // soundness check against the annotation would always fail + && !self.is_field_specifier(value) + && let Some(plan) = self.check_plan(&ann.annotation) + { + self.check_value(value, plan); + } + } + Stmt::Return(ret) => { + if self.positions.returns + && let Some(value) = &ret.value + && let Some(plan) = self.return_plan(value) + { + self.check_value(value, plan); + } + } + // a type-alias value is a type expression; nothing in it executes + Stmt::TypeAlias(_) => return, + _ => {} + } + walk_stmt(self, stmt); + } + + // annotations are type positions — never wrap inside them + fn visit_annotation(&mut self, _expr: &'ast Expr) {} + + fn visit_comprehension(&mut self, comprehension: &'ast Comprehension) { + if self.positions.iterations { + self.check_iteration( + &comprehension.iter, + &comprehension.target, + comprehension.is_async, + ); + } + ruff_python_ast::visitor::walk_comprehension(self, comprehension); + } + + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::UnaryOp(unary) = expr + && unary.op == UnaryOp::Force + { + // the force lowering inserts `_force_unwrap(` at the operand's + // first byte; a check wrapped around the bare operand would land + // its second argument inside that call. wrap the whole `expr!` + // instead (the force edits materialize inside the passthrough) + // and consume the operand — and any inner `!` layers — so they + // aren't wrapped again + let already_consumed = self.consumed.contains(&expr.range()); + let mut operand = unary.operand.as_ref(); + while let Expr::UnaryOp(inner) = operand + && inner.op == UnaryOp::Force + { + self.consumed.push(operand.range()); + operand = &inner.operand; + } + if self.gated_enabled(operand) { + self.consumed.push(operand.range()); + if !already_consumed && let Some(plan) = self.check_plan(expr) { + self.check_value(expr, plan); + } + } + } else if self.gated_enabled(expr) + && !self.consumed.contains(&expr.range()) + && let Some(plan) = self.check_plan(expr) + { + self.check_value(expr, plan); + } + if self.positions.arguments + && let Expr::Call(call) = expr + { + self.check_call_arguments(call); + } + walk_expr(self, expr); + } +} diff --git a/crates/ty_python_semantic/src/types/subscript.rs b/crates/ty_python_semantic/src/types/subscript.rs index 2193a37afc..90cb4b4c55 100644 --- a/crates/ty_python_semantic/src/types/subscript.rs +++ b/crates/ty_python_semantic/src/types/subscript.rs @@ -27,6 +27,43 @@ use super::{ TypeVarBoundOrConstraints, TypedDictType, UnionBuilder, UnsafeUnionType, todo_type, }; +/// basedpython: whether subscripting a value of type `ty` specializes it, so that its slice +/// holds type arguments, which inference reads as type expressions: a generic class, `tuple`, +/// `type`, a generic alias or a special form. a class that is not generic is subscripted +/// through its metaclass's `__getitem__` or its `__class_getitem__` — `Colour["Red"]` looks an +/// enum member up — and its slice is a value +pub(crate) fn specializes_on_subscript<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { + match ty { + Type::ClassLiteral(class) => { + class.generic_context(db).is_some() + || class.is_tuple(db) + || class.is_known(db, KnownClass::Type) + } + Type::GenericAlias(_) | Type::SpecialForm(_) | Type::KnownInstance(_) => true, + _ => false, + } +} + +/// basedpython: whether the type arguments subscripting a value of type `ty` are read as +/// deferred in a `.by` file, as an annotation is: a name in them may be one the module binds +/// only further on, and the transpiler writes it as the forward reference python reads lazily +/// +/// a reified class keeps its type arguments as runtime values, which a forward reference +/// cannot stand in for, so its are read where they are written +pub(crate) fn defers_type_arguments<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { + if !specializes_on_subscript(db, ty) { + return false; + } + match ty { + Type::ClassLiteral(class) => !class.generic_context(db).is_some_and(|context| { + context + .variables(db) + .any(|typevar| typevar.is_reified_class_typevar(db)) + }), + _ => true, + } +} + /// basedpython: whether subscripting a value of type `ty` is a runtime /// `__getitem__` call rather than a type specialization. /// diff --git a/crates/ty_python_semantic/src/types/unused_imports.rs b/crates/ty_python_semantic/src/types/unused_imports.rs new file mode 100644 index 0000000000..42f80fa04b --- /dev/null +++ b/crates/ty_python_semantic/src/types/unused_imports.rs @@ -0,0 +1,939 @@ +//! basedpython: `unused-import`, the checker's half of pyflakes' unused-import rule for +//! `.by` files +//! +//! whether an import in a `.by` file is used is a question about types as well as names. +//! `import colours` may be written only for the `extension Enum:` block `colours` declares, +//! and then nothing reads the name `colours` — the import is used because a member access +//! resolved through an extension it makes applicable. the linter answers per file and has no +//! types, so the rule lives here, where the file's inference has recorded every extension +//! and conformance it relied on, and buff's `F401` leaves `.by` files alone +//! +//! everything else is `F401`'s rule, read in the checker's model of the file: +//! +//! - a read of a binding counts for every later binding of the name in the scope. an import +//! nothing reads before another import rebinds the name is reported too, which is the +//! redefinition `F811` reports, so that one round of fixes removes all of them +//! - a `global` or `nonlocal` name binds in another scope, and is never reported +//! - `import a as a`, `from a import b as b` and `from a export b` re-export on purpose +//! - `from __future__ import …` is a compiler directive rather than a binding +//! - a module-scope name listed in `__all__` is exported, which is a use +//! - in a package's `__init__.by` an unused import may be an intended re-export, so it is +//! reported without a fix, and a first-party one says how to make the re-export explicit +//! - an import inside a `try` that handles `ImportError` probes for the module, so it is +//! reported without a fix, and says how to probe without importing + +use ruff_db::files::File; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_diagnostics::{Edit, Fix}; +use ruff_python_ast::identifier::Identifier; +use ruff_python_ast::name::Name; +use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; +use ruff_python_ast::{self as ast}; +use ruff_python_index::Indexer; +use ruff_python_trivia::{PythonWhitespace, is_python_whitespace}; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::{FxHashMap, FxHashSet}; +use ty_module_resolver::{ImportingFile, ModuleName, resolve_module}; +use ty_python_core::definition::{Definition, DefinitionCategory, DefinitionKind}; +use ty_python_core::place::ScopedPlaceId; +use ty_python_core::scope::{FileScopeId, ScopeKind}; +use ty_python_core::symbol::ScopedSymbolId; +use ty_python_core::{ProgramFile, ScopedDefinitionId, SemanticIndex, semantic_index}; + +use crate::dunder_all::dunder_all_names; +use crate::lint::LintId; +use crate::place::builtins_symbol; +use crate::reachability::is_reachable; +use crate::types::context::InferContext; +use crate::types::definition_resolution::user_visible_definitions; +use crate::types::diagnostic::{TypeCheckDiagnostics, UNUSED_IMPORT}; +use crate::types::extensions::{extension_granting_imports, extensions_in_module}; +use crate::types::lowering_reads::spelled_module_reads; +use crate::types::runtime_reads::file_runtime_read_checks; +use crate::types::{ProgramEnvironment, Type}; +use crate::{Db, FxIndexMap}; + +/// report every unused import in `file`. `inferred` holds the diagnostics of the whole file's +/// inference, whose record of the extensions it relied on decides the imports no name is +/// read through +pub(crate) fn report_unused_imports( + db: &dyn Db, + file: ProgramFile<'_>, + inferred: &TypeCheckDiagnostics, + diagnostics: &mut TypeCheckDiagnostics, +) { + let source_file = file.file(db); + if !source_file.source_type(db).is_basedpython() { + return; + } + if db + .rule_selection(source_file) + .get(LintId::of(&UNUSED_IMPORT)) + .is_none() + { + return; + } + let env = ProgramEnvironment::from_file(file); + let parsed = parsed_module(db, file.python_file(db)).load(db); + // a file that does not parse may have lost a read to the parser's recovery, and `F401` + // does not check one either + if parsed.has_syntax_errors() { + return; + } + + let index = semantic_index(db, file); + let checked_scopes = || { + index.scope_ids().filter(|scope| { + matches!( + index.scope(scope.file_scope_id(db)).kind(), + ScopeKind::Module | ScopeKind::Function + ) + }) + }; + // most files read every import they make, which the semantic index alone can tell: an + // import is only ever unused when the index records no read of it + let unread_import = checked_scopes().any(|scope| { + index + .use_def_map(scope.file_scope_id(db)) + .definitions_with_usage() + .any(|(_, definition, is_used)| !is_used && definition.kind(db).is_import()) + }); + if !unread_import { + return; + } + + let uses = Uses::new(db, file, index, &parsed, inferred); + let unused_by_scope: Vec<_> = checked_scopes() + .filter_map(|scope| { + let file_scope = scope.file_scope_id(db); + let unused = unused_in_scope(db, index, file_scope, &parsed, &uses, source_file); + (!unused.is_empty()).then_some((scope, unused)) + }) + .collect(); + if unused_by_scope.is_empty() { + return; + } + + let context = ImportContext::new(db, file, &env, &parsed); + let scopes: Vec<_> = unused_by_scope + .into_iter() + .map(|(scope, unused)| { + let at_module = scope.file_scope_id(db).is_global(); + let infer_context = InferContext::new(db, &env, scope, source_file, file, &parsed); + (infer_context, unused, at_module) + }) + .collect(); + + // an import the lint is not reporting — a suppressed one, or one in unreachable code — is + // not removed, so the fix for its statement leaves it in place + let removed = scopes.iter().flat_map(|(infer_context, unused, _)| { + unused.iter().filter(|import| { + context.fixable(import) + && infer_context + .report_lint(&UNUSED_IMPORT, import.binding) + .is_some() + }) + }); + let source = source_text(db, source_file); + let removals = Removals::new(&parsed, source.as_str(), removed); + + for (infer_context, unused, at_module) in scopes { + for import in &unused { + import.report(&infer_context, &context, at_module, &removals); + } + diagnostics.extend(&infer_context.finish()); + } +} + +/// every read of a binding the file makes, from both the semantic index and inference +struct Uses<'db> { + /// the definitions a name expression, a nested scope or a loop reads + definitions: FxHashSet>, + /// the imports whose module supplied an extension or a conformance inference relied on + granting: FxHashSet>, + /// names inference read as every binding their scope makes, rather than at a position + /// the semantic index records: a receiver callable read as a method, and a name in an + /// annotation, which is evaluated lazily + scope_reads: FxHashSet<(FileScopeId, ScopedSymbolId)>, + /// module-scope names read as a whole rather than at a position: an `__all__` entry, + /// and the type an `extension` header names + module_names: FxHashSet, +} + +impl<'db> Uses<'db> { + fn new( + db: &'db dyn Db, + file: ProgramFile<'db>, + index: &'db SemanticIndex<'db>, + parsed: &ParsedModuleRef, + inferred: &TypeCheckDiagnostics, + ) -> Self { + let source_file = file.file(db); + + let read = index.scope_ids().flat_map(|scope| { + index + .use_def_map(scope.file_scope_id(db)) + .definitions_with_usage() + .filter_map(|(_, definition, is_used)| is_used.then_some(definition)) + }); + // a read of a synthetic definition reads the definitions it stands for + let mut definitions: FxHashSet> = + user_visible_definitions(db, read).into_iter().collect(); + // a read of a loop's header reads every binding live at the top of the loop, which + // is how a use earlier in the body reaches a binding later in it + for scope in index.scope_ids() { + let use_def = index.use_def_map(scope.file_scope_id(db)); + for (_, definition, is_used) in use_def.definitions_with_usage() { + if !is_used { + continue; + } + let DefinitionKind::LoopHeader(header) = definition.kind(db) else { + continue; + }; + let loop_header = use_def.loop_header(header.loop_header_id()); + for live in loop_header.bindings_for_place(header.place()) { + if is_reachable(db, use_def, live.reachability_constraint()) + && let Some(binding) = use_def.definition(live.binding()).definition() + { + definitions.insert(binding); + } + } + } + } + + let granting = extension_granting_imports(db, source_file) + .iter() + .filter(|grant| inferred.is_extension_module_used(grant.module)) + .map(|grant| grant.import) + .collect(); + + let mut module_names: FxHashSet = dunder_all_names(db, file) + .into_iter() + .flatten() + .cloned() + .collect(); + let mut literal_all = DunderAllLiterals::default(); + literal_all.visit_body(parsed.suite()); + module_names.extend(literal_all.names); + // an extension resolves the type it names in its module's globals, as a whole — + // see `extended_class` + module_names.extend( + extensions_in_module(db, source_file) + .iter() + .map(|extension| extension.name(db).clone()), + ); + // a type the transpiled program writes out names its classes as the module binds them + let model = crate::LoweringModel::new(db, file); + module_names.extend(spelled_module_reads(&model, parsed.suite())); + for checks in file_runtime_read_checks(db, file) { + module_names.extend(checks.module_reads.iter().cloned()); + } + + Self { + definitions, + granting, + scope_reads: inferred.scope_reads().collect(), + module_names, + } + } + + fn reads(&self, definition: Definition<'db>) -> bool { + self.definitions.contains(&definition) || self.granting.contains(&definition) + } +} + +/// an import binding that nothing reads +struct UnusedImport { + /// the statement the binding is one alias of + statement: TextRange, + /// which of the statement's aliases makes the binding + alias_index: usize, + /// the name the import makes available, as F401 spells it: `a.b`, `a.b.c` for + /// `from a.b import c`, `.c` for `from . import c` + qualified_name: String, + /// the module the import names, for deciding whether it is first party + module: Option, + /// the bound name, the range the diagnostic is reported on + binding: TextRange, +} + +fn unused_in_scope<'db>( + db: &'db dyn Db, + index: &'db SemanticIndex<'db>, + scope: FileScopeId, + parsed: &ParsedModuleRef, + uses: &Uses<'db>, + file: File, +) -> Vec { + let is_stub = file.is_stub(db); + let place_table = index.place_table(scope); + let use_def = index.use_def_map(scope); + + // what happens to each name over the scope, in the order the scope does it: a binding, + // or a `del` (or the end of an `except … as` clause) that unbinds it + let mut history: FxIndexMap< + ScopedSymbolId, + Vec<(ScopedDefinitionId, Option>)>, + > = FxIndexMap::default(); + for (id, definition, _) in use_def.definitions_with_usage() { + let ScopedPlaceId::Symbol(symbol) = definition.place(db) else { + continue; + }; + let kind = definition.kind(db); + // a definition the source does not spell — a loop header, the submodule binding + // `from .x import y` makes in a package — stands for something written elsewhere + if !kind.is_user_visible() || matches!(kind, DefinitionKind::ImportFromSubmodule(_)) { + continue; + } + // a bare annotation declares a type and binds nothing + if matches!( + kind.category(is_stub, parsed), + DefinitionCategory::Declaration + ) { + continue; + } + history + .entry(symbol) + .or_default() + .push((id, Some(definition))); + } + for (id, place) in use_def.deletions() { + if let ScopedPlaceId::Symbol(symbol) = place + && let Some(events) = history.get_mut(&symbol) + { + events.push((id, None)); + } + } + + let mut unused = Vec::new(); + for (symbol_id, mut events) in history { + let symbol = place_table.symbol(symbol_id); + if symbol.is_global() || symbol.is_nonlocal() { + continue; + } + // a name read as a whole reads whichever binding it is left holding, and every + // binding before it carries that read forward + if uses.scope_reads.contains(&(scope, symbol_id)) + || (scope.is_global() && uses.module_names.contains(symbol.name())) + { + continue; + } + events.sort_unstable_by_key(|(id, _)| *id); + // the bindings since the name was last unbound, in order. a read of one counts for + // every binding after it, as `F401` carries a shadowed binding's reads to the one that + // shadows it, so the bindings that count as unread are a prefix of these + let since_unbound: Vec> = + match events.iter().rposition(|(_, event)| event.is_none()) { + Some(deletion) => &events[deletion + 1..], + None => &events[..], + } + .iter() + .filter_map(|(_, definition)| *definition) + .collect(); + for (position, &definition) in since_unbound.iter().enumerate() { + if uses.reads(definition) { + break; + } + // a binding the name goes on to be rebound by is reported only when the import + // rebinding it redefines it — the unused import `F811` reports as redefined. + // reporting every one at once is what lets the fix remove them all in one round + if let Some(&next) = since_unbound.get(position + 1) + && !redefines(db, parsed, next, definition) + { + continue; + } + if let Some(import) = reportable_import(db, definition, parsed) { + unused.push(import); + } + } + } + unused.sort_unstable_by_key(|import| import.binding.start()); + unused +} + +/// what an import binds, for deciding whether a later import redefines it: the name it +/// imports, as `F401` spells it, and whether it is a submodule import (`import a.b`, which +/// binds `a` but makes `a.b` reachable through it) +fn imported_name<'db>( + db: &'db dyn Db, + definition: Definition<'db>, + parsed: &ParsedModuleRef, +) -> Option<(String, bool)> { + match definition.kind(db) { + DefinitionKind::Import(import) => { + let alias = import.alias(parsed); + let submodule = alias.asname.is_none() && alias.name.contains('.'); + Some((alias.name.to_string(), submodule)) + } + DefinitionKind::ImportFrom(import) => { + let statement = import.import(parsed); + let mut name = ".".repeat(statement.level as usize); + if let Some(module) = &statement.module { + name.push_str(module); + name.push('.'); + } + name.push_str(&import.alias(parsed).name); + Some((name, false)) + } + _ => None, + } +} + +/// whether the import `later` redefines the import `earlier`, by ruff's rule: a submodule +/// import redefines, and is redefined by, only an import of the same submodule — `import a.b` +/// then `import a.c` binds `a` twice, and the second leaves `a.b` reachable through it +fn redefines<'db>( + db: &'db dyn Db, + parsed: &ParsedModuleRef, + later: Definition<'db>, + earlier: Definition<'db>, +) -> bool { + let (Some((later_name, later_submodule)), Some((earlier_name, earlier_submodule))) = ( + imported_name(db, later, parsed), + imported_name(db, earlier, parsed), + ) else { + return false; + }; + if later_submodule || earlier_submodule { + later_name == earlier_name + } else { + true + } +} + +/// `definition` as an import F401 reports when nothing reads it: not a `__future__` +/// directive, and not an explicit re-export +fn reportable_import<'db>( + db: &'db dyn Db, + definition: Definition<'db>, + parsed: &ParsedModuleRef, +) -> Option { + match definition.kind(db) { + DefinitionKind::Import(import) => { + let statement = import.import(parsed); + let alias = import.alias(parsed); + if alias + .asname + .as_ref() + .is_some_and(|asname| asname.id == alias.name.id) + { + return None; + } + Some(UnusedImport { + statement: statement.range(), + alias_index: import.alias_index(), + qualified_name: alias.name.to_string(), + module: (!alias.is_resource) + .then(|| ModuleName::new(&alias.name)) + .flatten(), + binding: alias.identifier(), + }) + } + DefinitionKind::ImportFrom(import) => { + let statement = import.import(parsed); + let alias = import.alias(parsed); + if statement.module.as_deref() == Some("__future__") + && statement.level == 0 + && !statement.is_lazy + { + return None; + } + if statement.is_export + || alias + .asname + .as_ref() + .is_some_and(|asname| asname.id == alias.name.id) + { + return None; + } + let mut qualified_name = ".".repeat(statement.level as usize); + if let Some(module) = &statement.module { + qualified_name.push_str(module); + qualified_name.push('.'); + } + qualified_name.push_str(&alias.name); + Some(UnusedImport { + statement: statement.range(), + alias_index: import.alias_index(), + qualified_name, + module: statement + .module + .as_deref() + .and_then(ModuleName::new) + .filter(|_| statement.level == 0), + binding: alias.identifier(), + }) + } + _ => None, + } +} + +/// what the diagnostics of one file share: whether it is a package's `__init__`, and which +/// import statements probe for their module +struct ImportContext<'db> { + db: &'db dyn Db, + file: ProgramFile<'db>, + in_init: bool, + /// the start of every import statement written in the body of a `try` that handles + /// `ImportError` or `ModuleNotFoundError` + probing: FxHashSet, +} + +impl<'db> ImportContext<'db> { + fn new( + db: &'db dyn Db, + file: ProgramFile<'db>, + env: &ProgramEnvironment<'db>, + parsed: &ParsedModuleRef, + ) -> Self { + let source_file = file.file(db); + // a package's `__init__.by` is the module importing the package runs; its stub + // re-exports only what it spells as one, as any stub does + let in_init = !source_file.is_stub(db) + && source_file + .path(db) + .as_system_path() + .and_then(|path| path.file_stem()) + .is_some_and(|stem| stem == "__init__"); + let import_errors: Vec> = ["ImportError", "ModuleNotFoundError"] + .into_iter() + .filter_map(|name| { + builtins_symbol(db, env, name) + .place + .ignore_possibly_undefined() + }) + .collect(); + let model = crate::SemanticModel::new(db, file); + let mut probes = ImportProbes { + model: &model, + import_errors: &import_errors, + depth: 0, + probing: FxHashSet::default(), + }; + probes.visit_body(parsed.suite()); + Self { + db, + file, + in_init, + probing: probes.probing, + } + } + + /// whether the import is removed by a fix: not one that may be a re-export from a + /// package's `__init__`, and not one that tests for its module's availability + fn fixable(&self, import: &UnusedImport) -> bool { + !self.in_init && !self.probing.contains(&import.statement.start()) + } + + /// whether `module` is the project's own code + fn is_first_party(&self, import: &UnusedImport) -> bool { + let db = self.db; + let Some(module) = &import.module else { + // a relative import names the package it is written in + return import.qualified_name.starts_with('.'); + }; + let file = self.file.file(db); + resolve_module( + db, + ImportingFile::File(file, self.file.resolver_environment(db)), + module, + ) + .and_then(|module| module.search_path(db)) + .is_some_and(ty_module_resolver::SearchPath::is_first_party) + } +} + +impl UnusedImport { + fn report( + &self, + context: &InferContext<'_, '_>, + imports: &ImportContext<'_>, + at_module: bool, + removals: &Removals<'_>, + ) { + let Some(builder) = context.report_lint(&UNUSED_IMPORT, self.binding) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{}` imported but unused", + self.qualified_name + )); + if let Some(annotation) = diagnostic.primary_annotation_mut() { + annotation.push_tag(ruff_db::diagnostic::DiagnosticTag::Unnecessary); + } + if imports.probing.contains(&self.statement.start()) { + diagnostic.help( + "To test whether a module is available, use `importlib.util.find_spec` \ + instead of importing it", + ); + } else if imports.in_init && at_module && imports.is_first_party(self) { + diagnostic.help( + "If this is a re-export, write it as `from … export …`, as a redundant alias, \ + or list it in `__all__`", + ); + } + if imports.fixable(self) + && let Some(edits) = removals.edits(self.statement) + { + let mut edits = edits.into_iter(); + if let Some(first) = edits.next() { + diagnostic.set_fix(Fix::safe_edits(first, edits)); + diagnostic.help(format_args!( + "Remove the unused import `{}`", + self.qualified_name + )); + } + } + } +} + +/// the edits that remove the unused imports the lint reports. every diagnostic in a statement +/// carries the same fix, so applying any of them removes all of the statement's unused +/// aliases at once, as `F401`'s fix does. a statement that loses every alias is deleted, and +/// the statements deleted next to one another in a suite share one fix too: the deletion of +/// one ends where the next begins, and two fixes that touch are applied in separate rounds +struct Removals<'a> { + source: &'a str, + indexer: Indexer, + /// each statement with an alias to remove, by its start, with the aliases to remove + statements: FxHashMap)>, + /// each deleted statement, by its start, with the run of adjacent deleted statements in + /// its suite it belongs to + deleted: FxHashMap, + runs: Vec>, +} + +/// statements next to one another in a suite, all deleted +struct Run<'a> { + statements: Vec<&'a ast::Stmt>, + /// whether the run is the whole of a suite that may not be left empty, so its last + /// statement is written as `pass`. deleting each one before it takes the separator after + /// it too, so a suite written on one line (`import os; import sys`) is left as `pass` + leaves_pass: bool, +} + +impl<'a> Removals<'a> { + fn new<'i>( + parsed: &'a ParsedModuleRef, + source: &'a str, + removed: impl Iterator, + ) -> Self { + let mut wanted: FxHashMap> = FxHashMap::default(); + for import in removed { + wanted + .entry(import.statement.start()) + .or_default() + .push(import.alias_index); + } + + let mut suites = Suites { + wanted: &wanted, + statements: FxHashMap::default(), + deleted: FxHashMap::default(), + runs: Vec::new(), + in_module: true, + }; + suites.visit_body(parsed.suite()); + + Self { + source, + indexer: Indexer::from_tokens(parsed.tokens(), source), + statements: suites.statements, + deleted: suites.deleted, + runs: suites.runs, + } + } + + /// the edits removing the unused aliases of the statement starting at `statement` + fn edits(&self, statement: TextRange) -> Option> { + let start = statement.start(); + if let Some(&run) = self.deleted.get(&start) { + let run = &self.runs[run]; + let last = run.statements.len() - 1; + return Some( + run.statements + .iter() + .enumerate() + .map(|(position, stmt)| { + if run.leaves_pass && position == last { + Edit::range_replacement("pass".to_owned(), stmt.range()) + } else { + ruff_python_edits::delete_stmt(stmt, None, self.source, &self.indexer) + } + }) + .collect(), + ); + } + let (stmt, aliases) = self.statements.get(&start)?; + let names = import_aliases(stmt)?; + Some(remove_aliases(self.source, stmt, names, aliases)) + } +} + +/// the aliases of an import statement +fn import_aliases(stmt: &ast::Stmt) -> Option<&[ast::Alias]> { + match stmt { + ast::Stmt::Import(import) => Some(&import.names), + ast::Stmt::ImportFrom(import) => Some(&import.names), + _ => None, + } +} + +/// finds each import statement with an alias to remove, and groups the ones that lose every +/// alias, and with it the whole statement, into runs of neighbours +struct Suites<'a, 'w> { + wanted: &'w FxHashMap>, + statements: FxHashMap)>, + deleted: FxHashMap, + runs: Vec>, + /// whether the suite being visited is the module's own, which may be left empty + in_module: bool, +} + +impl<'a> StatementVisitor<'a> for Suites<'a, '_> { + fn visit_body(&mut self, body: &'a [ast::Stmt]) { + let in_module = std::mem::replace(&mut self.in_module, false); + let mut run: Vec<&'a ast::Stmt> = Vec::new(); + let mut runs: Vec> = Vec::new(); + for stmt in body { + let start = stmt.start(); + let mut deleted = false; + if let (Some(aliases), Some(names)) = (self.wanted.get(&start), import_aliases(stmt)) { + let mut aliases = aliases.clone(); + aliases.sort_unstable(); + aliases.dedup(); + deleted = aliases.len() == names.len(); + if !deleted { + self.statements.insert(start, (stmt, aliases)); + } + } + if deleted { + run.push(stmt); + } else if !run.is_empty() { + runs.push(std::mem::take(&mut run)); + } + self.visit_stmt(stmt); + } + if !run.is_empty() { + runs.push(run); + } + let whole_suite = !in_module && runs.len() == 1 && runs[0].len() == body.len(); + for statements in runs { + let id = self.runs.len(); + for stmt in &statements { + self.deleted.insert(stmt.start(), id); + } + self.runs.push(Run { + statements, + leaves_pass: whole_suite, + }); + } + self.in_module = in_module; + } +} + +/// the edits removing the aliases at `remove` (sorted, and not every one) from an import +/// statement whose aliases are `aliases`. +/// +/// a run of aliases removed together goes with the separator after it, up to the next alias +/// kept, or — at the end of the statement — with the separator before it. a run that has +/// lines of its own inside parentheses goes with those lines, so the comments the formatter +/// leaves on an alias's line go with it and no other line changes +fn remove_aliases( + source: &str, + statement: &ast::Stmt, + aliases: &[ast::Alias], + remove: &[usize], +) -> Vec { + let mut edits = Vec::new(); + let mut runs: Vec<(usize, usize)> = Vec::new(); + for &index in remove { + match runs.last_mut() { + Some((_, last)) if *last + 1 == index => *last = index, + _ => runs.push((index, index)), + } + } + for (first, last) in runs { + let start = aliases[first].start(); + let end = aliases[last].end(); + if let Some(lines) = own_lines(source, statement, aliases, first, last) { + edits.push(Edit::range_deletion(lines)); + } else if let Some(next) = aliases.get(last + 1) { + edits.push(Edit::deletion(start, next.start())); + } else { + // the run ends the statement, and something before it is kept + edits.push(Edit::deletion(aliases[first - 1].end(), end)); + } + } + edits +} + +/// the whole lines the aliases `first..=last` occupy, when they occupy lines of their own +/// inside the statement's parentheses: nothing but indentation before the first, and nothing +/// but its comma and a comment after the last +fn own_lines( + source: &str, + statement: &ast::Stmt, + aliases: &[ast::Alias], + first: usize, + last: usize, +) -> Option { + // only a `from` import is parenthesized, and nothing before its first alias but the + // parenthesis can be one + let parenthesized = matches!(statement, ast::Stmt::ImportFrom(_)) + && source[TextRange::new(statement.start(), aliases.first()?.start())].contains('('); + if !parenthesized { + return None; + } + let start = aliases[first].start(); + let end = aliases[last].end(); + let line_start = source.line_start(start); + if !source[TextRange::new(line_start, start)] + .chars() + .all(is_python_whitespace) + { + return None; + } + let line_end = source.line_end(end); + let rest = source[TextRange::new(end, line_end)].trim_whitespace_start(); + let rest = rest + .strip_prefix(',') + .unwrap_or(rest) + .trim_whitespace_start(); + if !(rest.is_empty() || rest.starts_with('#')) { + return None; + } + Some(TextRange::new(line_start, source.full_line_end(end))) +} + +/// finds the import statements written in the body of a `try` that handles `ImportError` or +/// `ModuleNotFoundError` — an import made to find out whether the module is there +struct ImportProbes<'a, 'db> { + model: &'a crate::SemanticModel<'db>, + import_errors: &'a [Type<'db>], + /// how many enclosing `try` bodies handle an import error + depth: usize, + probing: FxHashSet, +} + +impl ImportProbes<'_, '_> { + fn handles_import_error(&self, handlers: &[ast::ExceptHandler]) -> bool { + handlers.iter().any(|handler| { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + let Some(caught) = handler.type_.as_deref() else { + return false; + }; + let caught: &[ast::Expr] = match caught { + ast::Expr::Tuple(tuple) => &tuple.elts, + single => std::slice::from_ref(single), + }; + caught.iter().any(|element| { + crate::HasType::inferred_type(element, self.model) + .is_some_and(|ty| self.import_errors.contains(&ty)) + }) + }) + } +} + +impl<'ast> StatementVisitor<'ast> for ImportProbes<'_, '_> { + fn visit_stmt(&mut self, stmt: &'ast ast::Stmt) { + match stmt { + ast::Stmt::Import(_) | ast::Stmt::ImportFrom(_) => { + if self.depth > 0 { + self.probing.insert(stmt.start()); + } + } + // a function's body runs when it is called, not inside the `try` it is defined + // in, and the `try` protects none of its imports — `F401` reads it the same way + ast::Stmt::FunctionDef(function) => { + let depth = std::mem::take(&mut self.depth); + self.visit_body(&function.body); + self.depth = depth; + } + ast::Stmt::Try(try_stmt) => { + let probes = self.handles_import_error(&try_stmt.handlers); + self.depth += usize::from(probes); + self.visit_body(&try_stmt.body); + self.depth -= usize::from(probes); + for handler in &try_stmt.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + self.visit_body(&handler.body); + } + self.visit_body(&try_stmt.orelse); + self.visit_body(&try_stmt.finalbody); + } + _ => walk_stmt(self, stmt), + } + } +} + +/// the string literals listed in the module's `__all__`, wherever its definition is too +/// dynamic for the checker's own reading of it to give a set of names +#[derive(Default)] +struct DunderAllLiterals { + names: Vec, +} + +impl DunderAllLiterals { + fn extend_from(&mut self, value: &ast::Expr) { + match value { + ast::Expr::List(ast::ExprList { elts, .. }) + | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { + for element in elts { + if let ast::Expr::StringLiteral(literal) = element { + self.names.push(Name::new(literal.value.to_str())); + } + } + } + ast::Expr::BinOp(ast::ExprBinOp { left, right, .. }) => { + self.extend_from(left); + self.extend_from(right); + } + _ => {} + } + } +} + +fn is_dunder_all(target: &ast::Expr) -> bool { + matches!(target, ast::Expr::Name(name) if name.id == "__all__") +} + +impl<'ast> StatementVisitor<'ast> for DunderAllLiterals { + fn visit_stmt(&mut self, stmt: &'ast ast::Stmt) { + match stmt { + ast::Stmt::Assign(assign) if assign.targets.iter().any(is_dunder_all) => { + self.extend_from(&assign.value); + } + ast::Stmt::AnnAssign(assign) if is_dunder_all(&assign.target) => { + if let Some(value) = &assign.value { + self.extend_from(value); + } + } + ast::Stmt::AugAssign(assign) if is_dunder_all(&assign.target) => { + self.extend_from(&assign.value); + } + ast::Stmt::Expr(ast::StmtExpr { value, .. }) => { + if let ast::Expr::Call(call) = value.as_ref() + && let ast::Expr::Attribute(attribute) = call.func.as_ref() + && is_dunder_all(&attribute.value) + && let [argument] = &*call.arguments.args + { + match attribute.attr.as_str() { + "extend" => self.extend_from(argument), + "append" => { + if let ast::Expr::StringLiteral(literal) = argument { + self.names.push(Name::new(literal.value.to_str())); + } + } + _ => {} + } + } + } + // a nested scope's `__all__` is not the module's + ast::Stmt::FunctionDef(_) | ast::Stmt::ClassDef(_) => {} + _ => walk_stmt(self, stmt), + } + } +} diff --git a/crates/ty_python_semantic/src/types/visibility.rs b/crates/ty_python_semantic/src/types/visibility.rs index ac5e5ee2c7..9029779088 100644 --- a/crates/ty_python_semantic/src/types/visibility.rs +++ b/crates/ty_python_semantic/src/types/visibility.rs @@ -386,6 +386,35 @@ pub(crate) fn resolves_to_module_scope<'db>( Some(false) } +/// basedpython: whether the name `reference` reads is read in a class body's own +/// scope and may find a binding the class body made — which the same read, moved +/// into a scope nested in the class body, would no longer see. a read the body has +/// not bound on any path resolves outward, as python resolves it, so it answers +/// `false`. `None` when ty did not index the reference +pub(crate) fn reads_class_scope_binding<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + reference: &ast::ExprName, +) -> Option { + let index = crate::semantic_index(db, file); + let reference_scope = index.try_expression_scope_id(&ast::ExprRef::from(reference))?; + if index.scope(reference_scope).kind() != ScopeKind::Class || !reference.ctx.is_load() { + return Some(false); + } + let scope_id = reference_scope.to_scope_id(db, file); + let table = ty_python_core::place_table(db, scope_id); + let Some(symbol) = table.symbol_by_name(&reference.id) else { + return Some(false); + }; + if symbol.is_global() || symbol.is_nonlocal() || !(symbol.is_bound() || symbol.is_declared()) { + return Some(false); + } + Some(match class_scope_binding(db, scope_id, reference) { + ClassBinding::Unbound => false, + ClassBinding::Bound | ClassBinding::Maybe => true, + }) +} + /// basedpython: how a member reached through a receiver is restricted on one /// class the receiver can be #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/ty_server/src/server/api/requests/detached_transpile.rs b/crates/ty_server/src/server/api/requests/detached_transpile.rs index 3bcb5643a7..fe72a9d09d 100644 --- a/crates/ty_server/src/server/api/requests/detached_transpile.rs +++ b/crates/ty_server/src/server/api/requests/detached_transpile.rs @@ -15,8 +15,11 @@ //! This is a workaround and worth naming as one. The nesting belongs to the //! transpiler, not to any request, so the fix that would remove this is for a //! type-aware pass to be handed a database rather than construct one. Until then -//! every caller that already holds one attached has to arrange not to be — the same -//! reason `by_stage::emit`'s transpile loop cannot be parallelised. +//! every caller that already holds one attached has to arrange not to be. A +//! re-stage also reads its own clone of the project database on the thread it is +//! called from, which inside a handler would be a second database attached over +//! the session's. `by_stage::emit` arranges the same thing for its parallel +//! transpile, by not attaching a job's database for the whole job. use std::panic::AssertUnwindSafe; diff --git a/crates/ty_test/src/db.rs b/crates/ty_test/src/db.rs index 93326a54ae..57ed39baf0 100644 --- a/crates/ty_test/src/db.rs +++ b/crates/ty_test/src/db.rs @@ -536,6 +536,11 @@ fn mdtest_rule_selection( // `.by` test to write `let` or `var` on assignments whose subject is // something else entirely. "implicit-declaration", + // `unused-import` too: a `.by` test imports what its subject needs, and often + // more than one block reads, so reporting every unused import would bury the + // tests that are about something else. `basedpython_unused_import.md` turns it + // back on for the tests that are about it. + "unused-import", ]; let registry = default_lint_registry(); diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index fe592550a2..60db177a3f 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -136,7 +136,11 @@ artifacts and exit status answer different questions. everything that could be emitted is emitted, sourcemap and package markers included; the exit status says whether anything was *reported*. so a build that prints an error exits 1 while still leaving a usable `build/`, and `by build && pytest build/tests` runs the tests -only against a tree the checker had nothing to say about +only against a tree the checker had nothing to say about. when a reported +diagnostic is fatal the exit status is 101 instead, as it is for `by check`. the +fatal one is `panic`, reported when `by` itself panicked while checking or +transpiling a file, so a script can tell a crash in `by` from a problem in the +code. `by run`, `by compile` and `by transpile ` answer the same way writes the transpiled python to the project root's `build/` — wherever in the project it is run, as `by compile` does too; `--out DIR` names another directory, diff --git a/docs/basedpython/development/compilation/plan.md b/docs/basedpython/development/compilation/plan.md index 2173b72bc3..6c509e37ab 100644 --- a/docs/basedpython/development/compilation/plan.md +++ b/docs/basedpython/development/compilation/plan.md @@ -299,7 +299,13 @@ M0, M1 and M2 are done. what exists: class-level constant, both come from the *interpreted definition* that already evaluated them at definition time rather than being computed a second time. that is what makes a mutable default shared by every call that omits it, and it - keeps the object identical between the two halves + keeps the object identical between the two halves. a call that hands the + boundary the very object the interpreted definition holds as a default is handed + to that definition too: in `.by` that object is the + [marker](../../features/mutable-defaults.md#what-inspectsignature-shows) a call + gets from `inspect.signature`, which only the definition's guard reads as the + argument not given — evaluating the default per call, or raising for a required + parameter after a defaulted one - **plain classes** — a hand-written `__init__` is lowered as a method and `tp_init` binds against *its* signature; the layout is the attributes that constructor diff --git a/docs/basedpython/development/compilation/technology.md b/docs/basedpython/development/compilation/technology.md index 9333b8663c..2cc55c134e 100644 --- a/docs/basedpython/development/compilation/technology.md +++ b/docs/basedpython/development/compilation/technology.md @@ -338,7 +338,9 @@ needs whatever that python needs at import time. that turns out to be nothing extra, but only because the transpiler is configured the same way `by build` configures it: the [lazy-import pass](../../features/lazy-imports.md) binds `JustFloat = float` locally rather than emitting a `from ty_extensions import …` -that has no module behind it +that has no module behind it. the one setting that differs is the deferral of `from` +imports, which is off: below 3.15 a deferred `from` import binds nothing where it +is written, and the compiled functions read the module's globals by name this is a sharp edge worth naming. transpiling for the fallback with a *different* config than the interpreted build uses produces an extension that diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index 327a23f2ed..3f880c4d93 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -35,9 +35,13 @@ source (.by) │ when X is not yet stdlib at the configured min Python version │ ├─ phase 2b anon-named-tuple cleanup - │ └─ re-run `anon_named_tuple` on the post-lowering output to catch anon-NT - │ spans copied verbatim by other transforms (e.g. the PEP-695 polyfill). - │ bounded to a few iterations + │ └─ re-run `anon_named_tuple` and the inline-protocol lowering on the + │ post-lowering output to catch spans copied verbatim by other transforms + │ (e.g. the PEP-695 polyfill). bounded to a few iterations. an output that + │ spells no anonymous named tuple and no inline protocol is passed over + │ after a parse, without a db or any type inference: the checker makes an + │ anonymous named tuple type only from such a form, so no plain tuple in + │ it can need constructing as one │ ├─ phase 2c lazy-import marking │ └─ lower imports to the `lazy` keyword (3.15+) or a runtime polyfill @@ -51,11 +55,9 @@ source (.by) │ └─ phase 3 syntax verification ├─ parse the final output as `.py` — any parse error aborts with a - │ source-annotated diagnostic (the span is mapped back to `.by`) - ├─ scan the AST for leftover basedpython-only flags - │ (`is_anon_named_tuple`, `is_anon_named_tuple_value`, `is_typeof`). - │ a leftover flag means a transform failed to lower its construct; the - │ pipeline aborts rather than emit syntactically-valid-but-wrong Python + │ source-annotated diagnostic (the span is mapped back to `.by`). a + │ python parse rejects every basedpython-only form, so this is also + │ what catches one a transform failed to lower ├─ reject a call to one of the transpiler's own runtime helpers that the │ module never got. a transform emits the call and records the need in │ two different places, and forgetting the second half produces python @@ -71,7 +73,11 @@ source (.by) holds a `.by` file to its target for every other kind, so a checker reports what this check would refuse. a basedpython form written as syntax of its own is a kind in that list too: a destructuring - pattern is written as a `match`, and is held to where one runs + pattern is written as a `match`, and is held to where one runs. + then run python's compile-time checks over it, on every version + from the floor on: python parses an assignment expression in a + comprehension's iterable, a parameter named twice, a capture that + makes the remaining `case`s unreachable, and refuses to compile them ``` entry points in `crates/by_transforms/src/lib.rs`: @@ -113,8 +119,15 @@ body, so a helper brings the rest of what it needs along all three paste the definitions in both renderings are slices of the same text. the lazy-import pass leaves the -runtime import eager, since a helper reached through its proxy would be a proxy -call on every use +runtime import eager: the polyfill runs on it, and a deferred helper would be an +attribute read on every use + +below 3.15 the lazy-import pass asks ty which `from` imports it can defer +(`SemanticModel::deferrable_from_imports`, run over the output itself, so it sees +every name an earlier lowering wrote). an import qualifies when it is the only +binding its name has anywhere in the module, and the answer carries every read +that reaches it; the pass writes each of those reads through the module's +`_LazyFrom`, whose `__dict__` is the module's namespace `_by_runtime.py` is excluded from this repository's own ruff configuration. reformatting it rewrites transpiled output, and the isort rule would put a diff --git a/docs/basedpython/features/anonymous-named-tuple.md b/docs/basedpython/features/anonymous-named-tuple.md index 783be2d1cf..e5f1c317bc 100644 --- a/docs/basedpython/features/anonymous-named-tuple.md +++ b/docs/basedpython/features/anonymous-named-tuple.md @@ -55,52 +55,121 @@ duplicate named field (`(name=1, name=2)`) within a single tuple, all named fields must use the same separator; you cannot mix `:` and `=` in the same anonymous named tuple -## plain-tuple coercion +## plain tuples where one is expected -basedpython auto-wraps plain tuple literals as constructor calls when the -surrounding annotation says they should be anonymous named tuples. coercion -fires in three positions: +a plain tuple literal written where an anonymous named tuple is expected *is* one: it is +constructed as that class, so its fields are there at runtime -**return statements** inside a function whose return annotation is an -anonymous named tuple: +```by +def take(p: (name: str, age: int)) -> str: + return p.name + +take(("ada", 36)) +``` + +transpiles to: + +```python +def take(p: _AnonNamedTuple_xxx) -> str: + return p.name + +take(_AnonNamedTuple_xxx("ada", 36)) +``` + +the expectation can come from anywhere the type checker gives a literal its expected type, so +every one of these is constructed: + +```by +type P = (name: str, age: int) + +def make() -> P: + return ("ada", 36) # a return, through an alias or not + +def greet(p: P = ("bob", 1)): ... # a default +def each(*ps: P): ... + +greet(p=("cy", 2)) # a keyword argument +each(("dee", 3), ("eve", 4)) # each element of `*args` + +people: list[P] = [("fay", 5)] # an element of a collection literal +people.append(("gus", 6)) # an argument of a method call +by_name: dict[str, list[P]] = {"h": [("hal", 7)]} # at any depth + +class Team: + var lead: P = ("ivy", 8) # an attribute's default + +team = Team() +team.lead = ("jo", 9) # a write to an attribute, or to a property + # whose setter takes a `P` +``` + +a `yield` in a generator typed `Iterator[P]` is constructed the same way, and a field that is itself +an anonymous named tuple constructs its own literal: `((1, 2), "a")` passed as +`(point: (x: int, y: int), label: str)` builds both classes + +only a literal is constructed. a value that already has a type is checked as it is: a variable +holding a plain tuple is not an anonymous named tuple, and a named tuple passed where a plain +`tuple` is expected stays the named tuple it is. a literal whose length doesn't match the expected +shape is left alone, so the type checker reports the mismatch + +### several expected shapes + +when the expected type is a union, the literal is constructed as the one anonymous named tuple that +accepts it: one with as many fields as the literal has elements, whose field types accept them ```by -def f() -> (age: int, name: str): - return (1, "a") # transpiled to: return _AnonNamedTuple_xxx(1, "a") +a: (age: int) | None = (36,) # constructed as `(age: int)` +b: (age: int) | (name: str) = ("ada",) # constructed as `(name: str)` +``` -f().name # works at runtime +a literal that more than one of them accepts is an error (`ambiguous-conversion`): the class decides +which field names the value has, and a plain tuple doesn't say which one was meant. name the fields +with the value form instead + +```by +c: (age: int) | (year: int) = (36,) # error: [ambiguous-conversion] +d: (age: int) | (year: int) = (age=36) # fine ``` -**annotated assignments** whose annotation is an anonymous named tuple, with or -without a [declaration modifier](modifiers.md): +an anonymous named tuple whose fields don't accept the literal is passed over, and the literal is +checked against the rest of the expected type as a plain tuple: ```by -a: (name: str, age: int) = ("asdf", 1) -let b: (name: str, age: int) = ("asdf", 1) -# transpiled to: a: _AnonNamedTuple_xxx = _AnonNamedTuple_xxx("asdf", 1) +e: (a: list[int]) | tuple[list[object]] = ([1, "s"],) # a plain tuple ``` -**list/set literals** whose annotation is `list[anon-NT]`, `set[anon-NT]`, -or `frozenset[anon-NT]` — every plain tuple element is wrapped: +an anonymous named tuple is a subtype of the plain tuple of its field types, so a union with that +tuple *is* that tuple, and a literal there stays a plain tuple: ```by -a: list[(age: int, name: str)] = [(1, "a"), (2, "b")] -# transpiled to: -# a: list[_AnonNamedTuple_xxx] = [_AnonNamedTuple_xxx(1, "a"), _AnonNamedTuple_xxx(2, "b")] +f: (age: int) | tuple[int] = (36,) # the expected type is `tuple[int]`: a plain tuple ``` -if a tuple literal's arity doesn't match the annotation it's left alone, so -ty diagnoses the mismatch rather than the transpiler silently constructing -the wrong shape +for the same reason, writing a literal to an attribute of a union whose members would build +different anonymous named tuples from it is an error (`invalid-assignment`): no single value is +both. members that would each build the same one — `P` and `P | None` alike — build it, and a +value with no plain tuple literal in it is checked against each member's own declaration as usual -coercion reads what the annotation *means*, not how it is spelled, so all three -positions fire through an alias declared above them — both `P = (name: str)` and -`type P = (name: str)`: +### `.py` files + +a `.py` file is python, where a plain tuple is never a named tuple. passing one where a `.by` +module expects an anonymous named tuple is an error there (`invalid-argument-type`). construct the +value through a name the `.by` module binds the type to: ```by -P = (name: str, age: int) +# shapes.by +Point = (x: int, y: int) + +def norm(p: (x: int, y: int)) -> int: + return p.x * p.x + p.y * p.y +``` + +```python +# main.py +from shapes import Point, norm -a: P = ("asdf", 1) # transpiled to: a: P = _AnonNamedTuple_xxx("asdf", 1) +norm((3, 4)) # error: [invalid-argument-type] +norm(Point(3, 4)) # fine ``` ## semantics @@ -113,12 +182,8 @@ class _Anon(NamedTuple): age: int ``` -the type checker treats `(name: str, age: int)` as the equivalent -heterogeneous `tuple[str, int]` for assignability and inference. plain tuple -literals can therefore be assigned to or returned from positions annotated -with an anonymous named tuple — at runtime they remain plain tuples, so -field-name access (`x.name`) requires constructing through the synthesized -class explicitly +it is a subclass of the heterogeneous `tuple[str, int]`, so it can be used anywhere that tuple +type is expected ## structural identity @@ -140,7 +205,3 @@ ordered list of types - field defaults aren't yet supported in the surface syntax. if you need defaults, declare a `NamedTuple` class explicitly -- auto-coercion of plain tuple literals only fires on `return`, - annotated assignment, and `list/set/frozenset[anon-NT]` literal sites. - nested positions like `dict[K, anon-NT]` values or call arguments need - the explicit value form `(name=…)` instead diff --git a/docs/basedpython/features/checked-cast.md b/docs/basedpython/features/checked-cast.md index 3d9d509be5..2204fffa51 100644 --- a/docs/basedpython/features/checked-cast.md +++ b/docs/basedpython/features/checked-cast.md @@ -76,6 +76,13 @@ both forms: - skip the check entirely when it is provably redundant. if the value is already the target, the probe would always pass, so the cast degrades to the same plain `typing.cast` the unsuffixed `cast` emits +- refuse a target naming a class the program does not have where the cast runs + (`erased-type-check`), as a [type test](parametric-type-tests.md#the-erased-type-check-error) + does: one imported only under `if TYPE_CHECKING:` (or past any other condition + that reads the constant), imported only on some paths, + or bound in the `else:` of a `TYPE_CHECKING` block, where the name need not be + the class the checker resolved. an unchecked `cast` tests nothing, and writes + such a type as a string - evaluate the value exactly once, even when it has side effects (`g() cast! int` calls `g()` once) - are basedpython-only: a `.py` file using either produces a parse error. they diff --git a/docs/basedpython/features/differences-from-python.md b/docs/basedpython/features/differences-from-python.md index 8cc2af8d6f..e5add2c339 100644 --- a/docs/basedpython/features/differences-from-python.md +++ b/docs/basedpython/features/differences-from-python.md @@ -73,7 +73,9 @@ print(os) # this is what loads it an import with a side effect — registering a plugin, patching something at module scope — no longer happens just because the importing module was loaded. `from __future__ import ...`, `from x import *`, and an unaliased `import a.b` -stay eager. see [lazy imports](lazy-imports.md) +stay eager. below python 3.15 a name imported with `from` is read off its module +on first use, and is in the importing module's namespace only from then on. see +[lazy imports](lazy-imports.md) ### an annotation read at runtime is its python spelling @@ -105,9 +107,10 @@ x: "Foo" ``` python reads that as a deferred reference to the name `Foo`; basedpython reads -it as the string-literal type. there is no manual forward-reference syntax -because none is needed — the transpiler -[quotes a self-reference for you](forward-references.md) when the +it as the string-literal type, in an annotation or a type argument +(`list["a"]()`) alike. there is no manual forward-reference syntax because none +is needed — the transpiler +[quotes a forward reference for you](forward-references.md) when the runtime requires it ### `float` means `float` diff --git a/docs/basedpython/features/extensions.md b/docs/basedpython/features/extensions.md index 60fddbfad8..6924408864 100644 --- a/docs/basedpython/features/extensions.md +++ b/docs/basedpython/features/extensions.md @@ -309,9 +309,13 @@ greeting.dedented() naming what a module declares is the usual way a file depends on it — a conformance is written against an interface imported by name — so -`from mod import Show` carries `mod`'s extensions too. requiring a separate -`import mod` whose symbols were never used would leave an import that reads as -removable to anyone tidying the file, silently withdrawing conformance +`from mod import Show` carries `mod`'s extensions too. an import of `mod` +anywhere in the file is enough, inside a function included + +a bare `import mod` whose name is never read is not unused: `by check` counts an +import as used when an extension or conformance it makes applicable supplies +something the file relies on, so [`unused-import`](unused-imports.md) does not +report it and `by check --fix` keeps it nothing is registered globally and nothing is monkeypatched, so two dependencies cannot fight over the same pair: their conformances are simply not visible to @@ -514,5 +518,3 @@ header already carries is not reported yet, and the last module imported wins at runtime - a requirement cannot be reached through an optional chain (`value?.show()`) or assigned through the interface; both are reported rather than lowered -- an unused `import` that carries a conformance is still reported by `F401`, - whose autofix would remove it and silently withdraw the conformance diff --git a/docs/basedpython/features/forward-references.md b/docs/basedpython/features/forward-references.md index 42f7995221..cf6b7fc9e7 100644 --- a/docs/basedpython/features/forward-references.md +++ b/docs/basedpython/features/forward-references.md @@ -46,13 +46,60 @@ in its lowered form (`"Tree | None"`, `"Callable[[Tree], None]"`) a name that is already bound is left alone, as is a local variable's annotation, which python never evaluates -a class's *subscript bases* evaluate while the class is being built, and so do -value-position subscripts in its body (`list[Tree]()`). a self-reference there -is quoted where it stands: `class Tree(list["Tree"])`. a direct base — -`class A(A)` — is *not* quoted. that is always a runtime error, and quoting it -would only mask the bug. method *bodies* are not rewritten either: by the time -a body runs, the class is bound, and quoting would produce a string instead of -a value +method *bodies* are not rewritten: by the time a body runs, the class is bound, +and quoting would produce a string instead of a value + +## outside an annotation + +a type written as a value can name a later class too, wherever python only +records it as a type: the type arguments of a generic (`list[Tree]()`, +`Alias = dict[str, Tree]`, a base `Node[Tree]`), and the types a typing +construct assigned to a name takes as arguments (`TypeVar`, `NewType`, the +functional `NamedTuple` and `TypedDict`). a type argument that names one is +quoted whole, as an annotation is, with whatever it lowers to inside the quote: + +```by +from typing import Callable, TypeVar + +registry = dict[str, Plugin]() +Handler = Callable[[Plugin], None] +P = TypeVar("P", bound=Plugin) + + +class Plugin: ... +``` + +python evaluates these where they stand, on every version, so the reference is +quoted where it is written: + +```python +from typing import Callable, TypeVar + +registry = dict[str, "Plugin"]() +Handler = Callable[["Plugin"], None] +P = TypeVar("P", bound="Plugin") + + +class Plugin: ... +``` + +a string there is a [literal type](literal-types.md), as it is in an +annotation, so you write the forward reference as the bare name and never as a +string + +some positions python reads as an object rather than as a type, and a quote +cannot stand in for one. a name not bound by then is an `unresolved-reference` +there: + +- a base itself: `class A(A)`, or `class Bar(Plugin)` above `Plugin`. its type + arguments are quoted as above (`class Tree(list[Tree])`) +- a keyword the class is built with, such as `metaclass=Meta`. a `TypedDict`'s + `extra_items=` is a type, and is quoted as above +- the type arguments of a [reified](reified-class-generics.md) class, which keeps + them as runtime values, wherever it stands: `list[Box[Plugin]]` too +- the type an `is`, `cast!` or `cast?` checks for, and the type `cast` reads +- a value inside a type: the arguments of `Literal[…]` (an enum member) and + the metadata of `Annotated[…]` ## why automatic @@ -66,9 +113,17 @@ transpiler quotes exactly the references that need it in python a string annotation is a forward reference. in basedpython a string in an annotation is a [literal type](literal-types.md), so `by transpile --reverse` writes each one as the expression it spells: -`-> "Tree"` becomes `-> Tree`, and `"Tree | None"` becomes `Tree?`. the -arguments of `Literal[…]` and the metadata of `Annotated[…]` are values, and -stay strings +`-> "Tree"` becomes `-> Tree`, and `"Tree | None"` becomes `Tree?`. the same +goes for a string outside an annotation that names a type: `List["Tree"]` +becomes `List[Tree]`, and `TypeVar("T", bound="Tree")` becomes +`TypeVar("T", bound=Tree)`. the arguments of `Literal[…]` and the metadata of +`Annotated[…]` are values, and stay strings, as does a subscript that is a +lookup rather than a generic (`Colour["Red"]` on an enum) + +the type `cast` reads is read where it is written, so it has no spelling for a +class defined further down: `cast("Later", o)` above `class Later` comes back +as `cast(Later, o)`, which the checker reports as an `unresolved-reference`. +move the cast below the class ## when quoting is skipped @@ -82,5 +137,5 @@ evaluated eagerly. it is skipped when: opted into the blanket injection, or target a version below 3.10, which always gets it -a subscript base evaluates eagerly on every target, so its self-reference is -quoted either way +a type outside an annotation evaluates where it stands on every target, so a +forward reference in one is quoted either way diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index e25d7a63c8..7dab9b8871 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -45,6 +45,8 @@ features that apply to a project rather than to a file language is written inside it - [linting](linter.md) — the `BY` rules, and how ruff's own rules read `.by` source +- [unused imports](unused-imports.md) — an import nothing uses, counting the extensions it brings + into scope - [the project server](project-server.md) — `by check` answered out of the language server's warm state diff --git a/docs/basedpython/features/init-method.md b/docs/basedpython/features/init-method.md index 11b3a3b254..e1c2052ca7 100644 --- a/docs/basedpython/features/init-method.md +++ b/docs/basedpython/features/init-method.md @@ -60,10 +60,10 @@ class Fight: ```python class Fight: - def __init__(self, seen: set[int] = _MISSING, last: int = _MISSING): + def __init__(self, seen: set[int] = _MISSING, last: int = _MISSING2): if seen is _MISSING: seen = set() - if last is _MISSING: + if last is _MISSING2: raise TypeError("__init__() missing required argument: 'last'") self.seen: set[int] = seen self.last: int = last diff --git a/docs/basedpython/features/lazy-imports.md b/docs/basedpython/features/lazy-imports.md index facea98208..7360cc81a9 100644 --- a/docs/basedpython/features/lazy-imports.md +++ b/docs/basedpython/features/lazy-imports.md @@ -57,59 +57,146 @@ set the target with `--min-version 3.15` on `by transpile`/`by build`/`by run` ## the polyfill (python < 3.15) without the `lazy` keyword there is no language-level way to defer a binding, -so the polyfill emits two different shapes: - -- `import os` → `os = _lazy_module("os")`, which wraps the loader in - `importlib.util.LazyLoader` and hands back a **real module object**. nothing - is proxied, so this form behaves exactly like an eager import apart from when - the body runs -- `from os import path` → `path = _lazy_attr("os", "path")`, a `_LazyAttr` - **proxy**. a proxy is unavoidable here: `_lazy_module` already defers the - module body, but reading `path` off the module would force it to run - immediately, which is the very thing being deferred - -the proxy is transparent — it forwards the operators (`==`, `+`, `len`, `in`, -`str`, `hash`, comparisons, …) to the value behind it, so it behaves like the -imported object. this matters more than it sounds: python looks special methods -up on the *type* and never routes them through `__getattr__`, so any dunder the -proxy fails to forward silently falls back to `object`'s version rather than -raising — an unforwarded `__eq__` would make `a == b` compare proxy identity -and quietly answer `False` for equal values - -`isinstance` works in both directions: `isinstance(x, C)` for a proxied `x` -(via `__class__`), and `isinstance(x, C)` where `C` itself was lazily imported -(via `__instancecheck__`) - -an **exception class is never proxied**. `except` is the one place cpython -refuses a stand-in: it checks that what it catches is a real class inheriting -`BaseException`, and never consults `__instancecheck__`. so -`from errors import ParseError` stays an ordinary import, and the rest of the -statement's names still don't: +so the polyfill lowers the two forms differently -```py -from errors import ParseError -decode = _lazy_attr("errors", "decode") +`import os` binds a real module object whose body runs on first attribute access: + +```python +os = _lazy_module("os") +``` + +nothing stands in for the module, so this form behaves exactly like an eager +import apart from when the body runs + +`from json import dumps` binds nothing where it is written. every read of `dumps` +is written as a read through one object the module keeps for its deferred names, +which resolves the name on first use: + +```by +from json import dumps + +def show(value: int) -> str: + return dumps(value) ``` -nor is a **special form** — `ClassVar`, `Final`, `Literal`, `Annotated`, `Protocol` and -the rest of what a type checker gives a meaning of its own. what reads one tells it -apart by identity: `dataclasses` decides a bare `ClassVar` annotation is not a field -by asking whether it *is* `typing.ClassVar`, and a checker reading the emitted python -rejects `ClassVar[int]` once `ClassVar` names a variable. that holds for the imports the -transpiler writes itself, such as the `ClassVar` an `enum class` lowers through: +transpiles to: + +```python +_by_lazy = _LazyFrom(globals(), ("dumps",)) +__getattr__ = _by_lazy.__by_getattr__ +__dir__ = _by_lazy.__by_dir__ +_lazy_from(_by_lazy, "json", {"dumps": "dumps"}) + +def show(value: int) -> str: + return _by_lazy.dumps(value) +``` + +(the runtime the polyfill calls is left out here) + +the first read of `dumps` imports `json`, reads `json.dumps` and stores it in the +module's namespace under `dumps`, and every read after that finds it there. so +the name evaluates to the object itself — `type(dumps)`, `dumps === json.dumps`, +handing it to `json.dumps` or `pytest.approx`, all answer as they would for an +ordinary import + +### what other modules see + +the module serves the name to other modules as an ordinary import would: +`from app import dumps`, `app.dumps`, `hasattr(app, "dumps")` and `dir(app)` all +find it, through the module's `__getattr__` and `__dir__` + +a patch of the name where it is used — `mock.patch("app.dumps")`, +`monkeypatch.setattr(app, "dumps", fake)` — writes the module's namespace, which +is where every read in the module looks first, so the module's own code sees the +patch, and sees the original again once it is undone + +### the name is resolved once + +a name is read off its module at its first use and kept, as PEP 810 does it. an +ordinary `from` import reads it when the import runs instead, so a module that +rebinds `json.dumps` between the import and the first use is seen by the polyfill +and not by an ordinary import. a rebinding after the first use is seen by neither + +### until the first use, the name is not in the namespace + +nothing is bound where the import is written, so until something reads the +name, it is not in the module's namespace: + +| expression | before the first use | +| ------------------------------------------ | ----------------------- | +| `globals()["dumps"]`, `vars(app)["dumps"]` | `KeyError` | +| `from app import *`, with no `__all__` | does not import `dumps` | +| `eval("dumps")` in the module | `NameError` | +| `typing.ForwardRef("dumps")`, evaluated | `NameError` | + +anything that walks a module's `__dict__` rather than asking for a name misses it +too: pytest does not collect a test class a test module imports and never uses + +reading the name once puts it there, and a module with an `__all__` that lists it +passes it on to `from app import *` whether it was read or not + +a read made before the import statement has run — or where it never ran, as in +`if PY2: from itertools import izip as zip` — finds what python's lookup finds: the +builtin of that name if there is one, and a `NameError` if not + +a reload runs the module again, and each of its `from` imports reads its name off the +module again on its next use, as an ordinary import reads it again when it runs + +### what stays an ordinary import + +the polyfill defers a name only where every read of it can be rewritten, and +where nothing reads it by identity before it is used. a name it does not defer +keeps an ordinary import, and the other names of the statement are still +deferred: ```py -from typing import ClassVar -cast = _lazy_attr("typing", "cast") +from json import JSONDecodeError; _lazy_from(_by_lazy, "json", {"dumps": "dumps"}) ``` -nor is a name an **annotation** reads. whatever reads an annotation back is handed what -the name is bound to, and a proxy is not the object it stands for: `dataclasses` decides -that `_: KW_ONLY` is not a field by asking whether the annotation *is* -`dataclasses.KW_ONLY`, and `typing.get_type_hints` answers with the proxy rather than the -class. so a `from` import whose name any annotation reads stays an ordinary import — a -parameter's or a return's annotation, a variable's in a module or a class body, or one -that is a string, as every annotation is under `from __future__ import annotations`: +- **an exception class.** a deferred name is resolved where it is first read, and + for an exception class that is a handler: an import failing there would + surface while another exception is being handled, and replace it +- **a special form** — `ClassVar`, `Final`, `Literal`, `Annotated`, `Protocol` and + the rest of what a type checker gives a meaning of its own. what reads one + tells it apart by identity: `dataclasses` decides a bare `ClassVar` annotation + is not a field by asking whether it *is* `typing.ClassVar`, and a checker + reading the emitted python rejects `ClassVar[int]` once `ClassVar` is anything + else. that holds for the imports the transpiler writes itself, such as the + `ClassVar` an `enum class` lowers through +- **a name an annotation reads.** whatever reads an annotation back reads its names + from the module's namespace: `typing.get_type_hints` evaluates a string + annotation against it, and `dataclasses` decides that `_: KW_ONLY` is not a + field by asking whether the annotation *is* `dataclasses.KW_ONLY`. that counts a + parameter's or a return's annotation, a variable's in a module or a class + body, and one that is a string, as every annotation is under + `from __future__ import annotations`. an annotation on a variable inside a + function body is never evaluated, so it does not count +- **a name a forward reference the transpiler writes reads** — a type parameter's + bound written ahead of the class it reads a name of, as + `TypeVar("T", bound="dict[Thing, C.Inner]")`: whatever evaluates that string + looks the name up in the module's namespace. a string in a `.by` type expression + is a [literal type](literal-types.md) and is written as `Literal["Thing"]`, so it + names nothing; a string handed to `typing.ForwardRef` by hand or to `eval` is not + seen, and is the last two rows of the table above +- **a name a decorator reads.** a decorator says what the definition under it + *is*, and a checker reading the emitted python recognizes `@overload` or + `@dataclass` only by name. deferring one buys nothing either: a decorator is + applied while the module is executing +- **a name bound again anywhere in the module** — assigned, deleted, a `def` or a + `class` of that name, a `for`, `with`, `except` or walrus target, a second + import, or a `global` declaration that binds it from a function. a read could + find either binding, and only the one the import makes is deferred +- **a name a class body reads where the body may have bound it already**: the read + finds the class's binding on some paths and the import on others +- **a name read where python keeps the text of the read**: `f"{dumps=}"` prints + the expression as written, and a t-string's `Interpolation` carries it +- **a name beginning with two underscores**, which a class body mangles +- **every name of a module that has a `from … import *`**, which can bind any + name, **or its own `__getattr__` or `__dir__`**, which the deferred names are + served through +- **every name of a module compiled with [`by compile`](../cli-reference.md#by-compile)**, + whose compiled functions read the module's globals by name. its module imports + are still deferred ```by from fractions import Fraction @@ -121,45 +208,10 @@ def show(value: Fraction) -> str: ```py from fractions import Fraction -dumps = _lazy_attr("json", "dumps") +_lazy_from(_by_lazy, "json", {"dumps": "dumps"}) ``` this is an edge of the polyfill: **an import used only in annotations is not deferred on python before 3.15**, and the module it names runs where the import is written — so one -this python does not have fails there, as an ordinary import does. an -annotation on a variable inside a function body is never evaluated, so it does not count. -on 3.15 and later the `lazy` keyword binds the real object, and such an import is deferred -like any other - -the functional forms build annotations out of values instead: `NamedTuple("P", [("x", T)])`, -`TypedDict("TD", {"a": T})` and `dataclasses.make_dataclass(...)` are handed their types as -arguments, when the call runs. a name imported with `from` and read only there is read as a -value, so under the polyfill it is still a proxy, and whatever reads those annotations back is -handed the proxy: `typing.get_type_hints(P)["x"]` is the proxy rather than the class, and -`make_dataclass` makes a field of the `_` in this `Row`, and leaves `extra` positional: - -```by -from dataclasses import KW_ONLY, make_dataclass - -Row = make_dataclass("Row", [("name", str), ("_", KW_ONLY), ("extra", int)]) -``` - -reach such a name through its module — `import dataclasses` and `dataclasses.KW_ONLY` — which -binds the real module rather than a proxy, or target `--min-version 3.15` - -two things a proxy cannot emulate, and which are therefore limitations of the -polyfill only: - -| expression | result | -| ---------------------------- | ----------------------------------------------- | -| `type(x)` | `_LazyAttr`, not the value's real type | -| `x === y` (identity) | compares proxy identity, not value identity | -| `x === None` on imported `x` | always `False`, even when the value *is* `None` | - -the `=== None` case is the sharp one, because identity-against-a-singleton is a -common idiom: `from cfg import SENTINEL` then `SENTINEL === None` is `False` -even when `SENTINEL` is `None`, because `===` (python `is`) sees the proxy, not -the value it stands for. test imported values for emptiness by *value* — `x`'s -truthiness, `x == None`, `isinstance(x, T)` — all of which the proxy forwards. -or target `--min-version 3.15`, where the `lazy` keyword binds the real object -and none of these limitations exist +this python does not have fails there, as an ordinary import does. on 3.15 and later the +`lazy` keyword binds the real object, and such an import is deferred like any other diff --git a/docs/basedpython/features/linter.md b/docs/basedpython/features/linter.md index 3e8550f015..8e17f34eb5 100644 --- a/docs/basedpython/features/linter.md +++ b/docs/basedpython/features/linter.md @@ -66,6 +66,14 @@ here, and why `t[0]` is not reported as a [tuple member access](tuple-index.md) — whether `t` is a tuple is a type question +`F401` is the one upstream rule that follows from that split. whether an +import is used can be a type question in a `.by` file — an import may be there +only for the [extensions](extensions.md) its module declares — so `F401` does +not check `.by` or `.byi` files, and `by check` reports them as +[`unused-import`](unused-imports.md), with `F401`'s rules. a `# noqa: F401` in a +`.by` file suppresses nothing any more, and `RUF100` reports it where it is +enabled; write `# ty: ignore[unused-import]` instead + ## upstream rules on `.by` source ruff's own rules are written for python source, and two things follow from that. diff --git a/docs/basedpython/features/literal-types.md b/docs/basedpython/features/literal-types.md index f3a4608f7b..e659eabe78 100644 --- a/docs/basedpython/features/literal-types.md +++ b/docs/basedpython/features/literal-types.md @@ -68,9 +68,17 @@ the output reports the argument as invalid ## scope promotion fires only in syntactic type contexts: annotations, return types, -type aliases, and generic subscript slices whose target is a type. value -expressions are untouched. `Annotated[T, metadata]` does not treat its -metadata slice as a type context, so literal metadata is preserved +type aliases, and the type arguments of a generic, written as a value or not. +`list["a"]()` is a list of the string `"a"`, transpiled to +`list[Literal["a"]]()`, where python would read `"a"` as a forward reference +to a name `a`. a forward reference is written as the bare name, which the +transpiler quotes where python needs it (see +[forward references](forward-references.md#outside-an-annotation)) + +value expressions are untouched, and so is a subscript that looks a value up +rather than naming a generic: `Colour["Red"]` on an enum keeps its string. +`Annotated[T, metadata]` does not treat its metadata slice as a type context, +so literal metadata is preserved ## polyfill diff --git a/docs/basedpython/features/mutable-defaults.md b/docs/basedpython/features/mutable-defaults.md index fef170d0bb..ee82f9800e 100644 --- a/docs/basedpython/features/mutable-defaults.md +++ b/docs/basedpython/features/mutable-defaults.md @@ -20,6 +20,78 @@ everything else is rewritten so the default expression runs at call time. a useful side effect: `def g(a, b=a + 1)` becomes valid late-bound default syntax, with `b` computed fresh from the actual `a` at each call +a default that names a value is read when the call runs, too, not where the +`def` stands: + +```by +LIMIT = 2 + +def cap(n=LIMIT): + return n + +LIMIT = 3 +cap() # 3, where python's own default would still be 2 +``` + +## what `inspect.signature` shows + +a default re-evaluated per call has no value until a call evaluates it, so the +signature holds a marker in its place, and the marker's `repr` is the default as +basedpython reads it: + +```by +import inspect + +LIMIT = 2 + +def f(a: int, b: list[int] = [], c: int = LIMIT, d: int = 0): + ... + +print(inspect.signature(f)) +# (a: int, b: list[int] = , c: int = , d: int = 0) +``` + +transpiles to: + +```python +_MISSING: Any = _by_default("[]", __name__, "_MISSING") +_MISSING2: Any = _by_default("LIMIT", __name__, "_MISSING2") + +def f(a: int, b: list[int] = _MISSING, c: int = _MISSING2, d: int = 0): + if b is _MISSING: + b = [] + if c is _MISSING2: + c = LIMIT + ... +``` + +- a scalar default is python's own, so the signature holds its value: `d` above + is `0` +- a marker is not the value. `parameters["c"].default == 2` is `False`, and no + value could be honest there: `LIMIT` is whatever it is when the call runs, + `[]` is a new list each time, and a default reading an earlier parameter, + such as `b=a + 1`, has none without an `a` +- the text is the default after the rewrites basedpython makes ahead of this one, + on one line: `= Red` naming an enum member shows as ``, a + `??` stays `??`, the comments written inside a default are dropped, and a line + break between two of its tokens becomes a space, or nothing just inside a + bracket. a string literal keeps its own text, line breaks and all +- each default has a marker of its own. handed back as an argument, a marker is + the argument not given: calling `f` with the arguments + `BoundArguments.apply_defaults()` filled in evaluates each default as leaving + the parameter out would, compiled with [`by compile`](../cli-reference.md#by-compile) or + not. handed to any other parameter, it is a value like any other. a `.by` call + passing one gets the [runtime check](soundness.md) any other value would, and + a marker is not an `int` +- a copy of a marker is the marker, and a pickled one comes back as the marker it + was, read from the name the module binds it under +- a required parameter after a defaulted one, which python cannot declare where + it stands, shows as ``. handed back, it raises the `TypeError` + leaving it out does +- a parameter with no annotation and such a default records no type at runtime, + so a [structural protocol test](parametric-type-tests.md#a-default-evaluated-per-call-records-no-type) + against its method raises rather than answer + ## a class body is not a default the same reading does not extend to a class-body declaration, because a class diff --git a/docs/basedpython/features/none-coalesce.md b/docs/basedpython/features/none-coalesce.md index 28ed632f6e..ee0aad819f 100644 --- a/docs/basedpython/features/none-coalesce.md +++ b/docs/basedpython/features/none-coalesce.md @@ -16,6 +16,22 @@ a compound left operand is bound to a temp by the walrus so it is evaluated exactly once. a bare name needs no temp and is repeated directly — `a ?? b` is `a if a is not None else b` +python refuses the walrus in some places: a comprehension's iterable, a +comprehension in a class body, an annotation, and a generic definition's type +parameters, bases and `type` value. there the temp is a comprehension variable +instead, which still evaluates the left operand once and the right one only when +it is needed: + +```by +names = [n for n in user.nicknames ?? []] +``` + +transpiles to: + +```python +names = [n for n in ([__by_t_0__ for __by_t_0__ in (user.nicknames,) if __by_t_0__ is not None] or [[]])[0]] +``` + ## semantics `??` tests `is not None` (identity) — not falsiness. an empty string, zero, @@ -23,9 +39,17 @@ or an empty list is *not* coalesced. only `None` triggers the fallback ## precedence -`??` has the same precedence as a conditional expression — it binds looser -than `or` / `and` / arithmetic and tighter than assignment. parenthesize when -mixing with boolean operators +`??` has the same precedence as `or` and groups left to right with it. it binds +looser than `and`, `not`, comparisons and arithmetic, and tighter than a +conditional expression: + +```by +a ?? b or c # (a ?? b) or c +a or b ?? c # (a or b) ?? c +a ?? b and c # a ?? (b and c) +``` + +parenthesize when mixing it with boolean operators ## interaction with `?.` diff --git a/docs/basedpython/features/optional-chaining.md b/docs/basedpython/features/optional-chaining.md index 81d12ceec1..35f54661a6 100644 --- a/docs/basedpython/features/optional-chaining.md +++ b/docs/basedpython/features/optional-chaining.md @@ -43,6 +43,26 @@ zip = user?.address.zip `a.b?.c` works in the obvious way: only the part after `a.b` is short-circuited through a temp variable +where python refuses the walrus — a comprehension's iterable, a comprehension in +a class body, an annotation, a generic definition's type parameters — the temp is +a comprehension variable instead, one clause per guard: + +```by +class Report: + let streets: list[list[str]?] = [user.address?.streets for _ in range(2)] +``` + +transpiles to: + +```python +class Report: + streets: list[list[str] | None] = [(([__by_t_0__.streets for __by_t_0__ in (user.address,) if __by_t_0__ is not None] or [None])[0]) for _ in range(2)] +``` + +from a comprehension's first iterable in a class body, that moves the chain's +calls and subscripts out of the class body's scope, so a chain whose call reads a +name the class body binds there is refused rather than lowered + ## the chain runs to the end of the trailers a `?.` opens a chain that runs out through every trailer that follows it — diff --git a/docs/basedpython/features/parametric-type-tests.md b/docs/basedpython/features/parametric-type-tests.md index 8308412118..6f82cb8d1e 100644 --- a/docs/basedpython/features/parametric-type-tests.md +++ b/docs/basedpython/features/parametric-type-tests.md @@ -182,6 +182,40 @@ def g(x): return x is array.array # ok — an ordinary isinstance ``` +and a target naming a class the program does not have where the test runs, since the test reads +it there: one imported only under `if TYPE_CHECKING:` (or past any other condition that reads the +constant, such as `match TYPE_CHECKING:` or `if TYPE_CHECKING and ready:`), or imported only on some +paths + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +def f(x) -> bool: + return x is Decimal # error: `Decimal` is not bound when it runs +``` + +a module can import a class for the checker and bind the name some other way as it runs, in the +`else:` of the same block. the checker reads the block as always taken, so the class it resolves is +the `TYPE_CHECKING` one, and the name the test would read need not be that class — so that is +refused too: + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal +else: + Decimal = float + +def f(x) -> bool: + return x is Decimal # error: the `Decimal` bound when it runs need not be the class the checker resolved +``` + +import the class as the program runs to test against it + ## `===` is unaffected the `===` / `!==` identity operators keep python semantics; only the `is` / @@ -241,6 +275,29 @@ InitLevel() is HasA[int] # TypeError: its type is declared inside a method declare the member in the class body and the check answers. this is the same line the erased builtin target takes: refuse, never guess. +### a default evaluated per call records no type + +a method parameter with no annotation is checked by the type of its default. +a default that is not a scalar has no value until a call +[evaluates it](mutable-defaults.md#what-inspectsignature-shows), so nothing at +runtime records the type the checker reads off it, and the test raises: + +```by +class Foo: ... + +class Feeds[T](Protocol): + def m(self, x: T) -> int: ... + +class TakesFoo: + def m(self, x = Foo()) -> int: + return 1 + +def g(v: object): + v is Feeds[Foo] # TypeError for a `TakesFoo`: annotate `x` +``` + +annotate the parameter and the check answers + ### the cast forms usually never get here a [checked cast](checked-cast.md) shares this engine, but only for a value the diff --git a/docs/basedpython/features/reified-class-generics.md b/docs/basedpython/features/reified-class-generics.md index 0d474a0fe6..bd388e7976 100644 --- a/docs/basedpython/features/reified-class-generics.md +++ b/docs/basedpython/features/reified-class-generics.md @@ -127,11 +127,24 @@ class Box[T]: b: Box[int] = Box() # from the declaration Box[str]() # written out + +class Holder: + box: Box[int] + + def __init__(self): + self.box = Box() # from the attribute's declaration ``` where nothing solves it, the bare construction is an error (`unspecialized-reified-generic`) rather than an instance that would fail the -first time a method asked it a question +first time a method asked it a question. that includes a type argument the +running program does not have where the construction runs: a class imported only +under `if TYPE_CHECKING:` (or past any other condition that reads the constant), +imported only on some paths, unbound by a `del`, or declared further down the +module than the statement the construction is in — +for one in a function or a method, the `def` or `class` statement at the top of +the module that holds it, which can be called as soon as that runs. write the +specialization out there (`Box[Later]()`) where you know it is bound ## a reified parameter is invariant diff --git a/docs/basedpython/features/soundness.md b/docs/basedpython/features/soundness.md index 5b2490b667..088946f8c4 100644 --- a/docs/basedpython/features/soundness.md +++ b/docs/basedpython/features/soundness.md @@ -106,8 +106,47 @@ no check is emitted when there is no faithful shallow runtime test — protocols, callables, unsolved type parameters, and `object` (which every value satisfies) are all skipped, as is a type whose name doesn't resolve at module scope (a class shadowed by a local binding, or one not in scope where -the check would run). a pure-`None` result is skipped too: validating a value -that carries no data guards nothing +the check would run). a class the program does not have where the check runs +is skipped the same way: one bound only past a condition that reads +`TYPE_CHECKING` (how an import cycle is broken) — an `if`, `while` or `match` +on the constant under any name or inside any expression, or a value a +conditional expression on it chooses — one imported only on some paths, +unbound by a `del`, rebound from a function through `global`, or declared +further down the module than the check, and one bound in the `else:` of a +`TYPE_CHECKING` block, where the name the check would read need not be the +class the checker resolved. a pure-`None` result is skipped too: validating a +value that carries no data guards nothing + +a check in a function or a method runs whenever it is called, which can be +before or after the module reaches a class declared further down. where every +binding the module makes of the name binds that one class — none binds anything +else, none is made only for the checker, nothing unbinds it, and no function +rebinds it through `global` — the check looks the name up as it runs, and is +made once the name is bound; until then the value passes unchecked. where that +does not hold, the class has to be bound above the `def` or `class` statement +at the top of the module that holds the function, which can be called as soon +as that runs: + +```by +class Modifier: + cuts: list[Cut] + + def total(self) -> int: + total = 0 + for cut in self.cuts: + total += cut.size + return total + +class Cut: + size: int +``` + +→ `for cut in _soundness_once_bound(("Cut",), self.cuts, _soundness_iter, lambda: Cut):` + +the lookup is by the name the class is emitted under, so a `private class Cut` +is looked up as `"_Cut"`. a check at the module's own level runs in order, so +it is made only below the class it names. `by compile` leaves a function with +such a check to its interpreted definition ## deep checks for user generics diff --git a/docs/basedpython/features/trailing-lambdas.md b/docs/basedpython/features/trailing-lambdas.md index 5c5d68b2e9..e08a946bef 100644 --- a/docs/basedpython/features/trailing-lambdas.md +++ b/docs/basedpython/features/trailing-lambdas.md @@ -332,7 +332,7 @@ inspected leaves the block unconstrained so a trailing block can bind the last parameter while earlier parameters keep defaults, a `def` (not a `lambda`) may declare a required parameter after a defaulted one. python rejects that shape at runtime, so the required parameter -is lowered to a `_MISSING` sentinel default plus a guard that raises — +is lowered to a `_MISSING` marker default plus a guard that raises — mirroring python's own error: ```python @@ -343,7 +343,8 @@ def f(x=1, a=_MISSING): ``` the checker still treats the parameter as required. see -[mutable defaults](mutable-defaults.md) for the sentinel machinery this shares +[mutable defaults](mutable-defaults.md) for the marker this shares, and for what +`inspect.signature` shows for it ## inlay hints diff --git a/docs/basedpython/features/unused-imports.md b/docs/basedpython/features/unused-imports.md new file mode 100644 index 0000000000..f522a26fc5 --- /dev/null +++ b/docs/basedpython/features/unused-imports.md @@ -0,0 +1,110 @@ +# unused imports + +`by check` reports an import in a `.by` file that nothing uses, as `unused-import`, and +`by check --fix` removes it + +```by +import json # warning: `json` imported but unused +from enum import Enum + +class Colour(Enum): + RED = 1 +``` + +an import is used when a name it binds is read — and also when the module it names declares an +[extension](extensions.md) or a conformance that supplies something the file relies on. an +extension's members are only applicable in the files that import its module, so an import can be +the reason `colour.lower_name` resolves without the module's name ever being written again + +```by +# colours.by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +```by +# main.by +import colours # ok: the extension it declares is used below +from enum import Enum + +class Colour(Enum): + RED = 1 + +def describe(colour: Colour) -> str: + return colour.lower_name +``` + +each of these counts as a use of the module that declares the extension or conformance: + +- an extension member, read off a value or as a bare name in a + [receiver block](implicit-receivers.md) +- an [operator](extensions.md#operators) or a [conversion](extensions.md#conversions) an extension + supplies +- a conformance that makes a value acceptable where its protocol is asked for, or carries a + protocol extension's members to it +- a [type test](extensions.md#the-type-test) answered from the conformance registry + +a name the emitted python reads although the source never spells it is a use too, of the import +that binds it: the enum a bare `Red` or `case Red:` is lowered to qualify (`Color.Red`), and the +classes a type is written out with — the runtime check of a +[type test](parametric-type-tests.md), a [checked cast](checked-cast.md), a +[`raises`](exceptions.md) guard or a [soundness check](soundness.md), the specialization a +[reified](reified-generics.md) call or construction is given, and the bound of the type parameter +a union parameter is given. `x is AB`, with `type AB = A | B`, checks `A` and `B`. a soundness +check counts at every position, because which ones run is the build's choice + +any import of the module makes its extensions applicable — `import colours`, `import colours.extra` +(for `colours` and `colours.extra`), and `from colours import something` — so each of them is used +when one of the module's extensions is, whether or not anything reads the name it binds. an import +makes them applicable anywhere in the file, so one written inside a function counts too + +## the rules + +apart from extensions, the rule is ruff's `F401`, which in turn is pyflakes': + +- a read of a binding counts for every binding of that name after it in the scope, and a `del` + starts the name afresh. an import nothing reads before the name is rebound is reported when + another import rebinds it — what `F811` reports as a redefinition — so one round of fixes + removes every one; an import a submodule import follows, or the other way round, is only + redefined by an import of the same module. an import rebound by anything else is left to that + rebinding +- an import in a class body is never reported, nor one that binds a `global` or `nonlocal` name +- `import a as a`, `from a import b as b` and [`from a export b`](export-imports.md) are re-exports, + a name listed in `__all__` is exported, and `from __future__ import …` is a directive — none is + reported +- a read inside a `TYPE_CHECKING` block counts, and an annotation may read an import written after + it. a string in a type position is a [literal type](literal-types.md), so it reads nothing +- a file with a syntax error is not checked: the parser's recovery may have dropped a read + +two kinds of unused import are reported without a fix: + +- one in a package's `__init__.by`, which may be how the package exports the name. for one of the + project's own modules the report says how to make the re-export explicit +- one in the body of a `try` that handles `ImportError`, which is usually a test for whether the + module is installed. the report suggests `importlib.util.find_spec` + +## buff + +buff's `F401` does not check `.by` or `.byi` files. a linter reads one file at a time and has no +types, so it cannot see that an extension supplied something, and its fix would remove the import +that makes the extension applicable. `unused-import` is the rule there instead + +that changes how a suppression is written. `# noqa: F401` in a `.by` file no longer suppresses +anything, and `RUF100`, where it is enabled, reports it as unused and removes it. an import that +needs to stay although nothing uses it is suppressed for `by check` instead: + +```by +import side_effects # ty: ignore[unused-import] +``` + +`by check --add-ignore` writes that comment on every unused import still reported. +`lint.pyflakes.allowed-unused-imports` is buff's setting and does not apply to `by check`; turn the +rule off for a set of files with a [per-path override](../configuration.md#per-path-overrides) instead + +## related + +- [extensions](extensions.md) — what an extension is, and how its import makes it applicable +- [linting](linter.md) — how ruff's other rules read `.by` source diff --git a/ty.schema.json b/ty.schema.json index da4e523aef..8687669929 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -723,7 +723,7 @@ }, "ambiguous-conversion": { "title": "detects conversion sites served by more than one conversion", - "description": "## What it does\nChecks for conversion sites where more than one conversion applies — two\ndunders, a dunder and an in-scope conformance, or two applicable\n`implementation`s of the same interface and type.\n\n## Why is this bad?\n`__from__` and `__into__` are hand-written bodies that can disagree, so\nwhich one runs must not depend on arbitrary ordering. Remove one of them,\nor write the conversion you want explicitly.\n\n## Example\n\n```by\nclass Celsius:\n def __into__(self) -> Fahrenheit: ...\n\nclass Fahrenheit:\n @classmethod\n def __from__(cls, value: Celsius) -> Self: ...\n\nreport(Celsius()) # error: two conversions apply\n```", + "description": "## What it does\nChecks for conversion sites where more than one conversion applies — two\ndunders, a dunder and an in-scope conformance, or two applicable\n`implementation`s of the same interface and type. A plain tuple literal\nthat more than one expected anonymous named tuple accepts is one too:\n`(1,)` where `(a: int) | (b: int)` is expected.\n\n## Why is this bad?\n`__from__` and `__into__` are hand-written bodies that can disagree, so\nwhich one runs must not depend on arbitrary ordering. Remove one of them,\nor write the conversion you want explicitly. A plain tuple literal is\nconstructed as the anonymous named tuple it stands for, which decides the\nfield names the value has, and nothing in the literal says which one was\nmeant: name the fields, as in `(a=1)`.\n\n## Example\n\n```by\nclass Celsius:\n def __into__(self) -> Fahrenheit: ...\n\nclass Fahrenheit:\n @classmethod\n def __from__(cls, value: Celsius) -> Self: ...\n\nreport(Celsius()) # error: two conversions apply\n```", "default": "error", "oneOf": [ { @@ -2911,6 +2911,16 @@ } ] }, + "unused-import": { + "title": "detects an import that nothing uses", + "description": "## What it does\nChecks a basedpython file for an import that nothing uses: no name it binds\nis read, and no `extension` or conformance it brings into scope supplies\nanything the file relies on.\n\n## Why is this bad?\nAn unused import costs time when the module is imported, can create an\nimport cycle, and tells a reader the module depends on something it does\nnot.\n\nThe extension members a module declares are only applicable in the files\nthat import it, so an import can be used without its name ever being read.\nThis rule counts that as a use, which a linter working one file at a time\ncannot — so in a `.by` file it takes the place of buff's `F401`, which\nleaves `.by` files alone.\n\nOtherwise the rule is `F401`'s:\n\n- a read of a binding counts for every later binding of that name in the\n scope. An import nothing reads before another import rebinds the name\n is reported too, as `F811` reports the redefinition, so one round of\n fixes removes every one. An import inside a class body is never\n reported\n- `import a as a`, `from a import b as b` and `from a export b` are\n re-exports, a name listed in `__all__` is exported, and\n `from __future__ import …` is a directive — none of them is reported\n- in a package's `__init__.by`, and inside a `try` that handles\n `ImportError`, an unused import is reported without a fix: it may be a\n re-export, or a test for whether the module is installed\n\n## Examples\n```by\n# colours.by\nfrom enum import Enum\n\nextension Enum:\n let lower_name: str\n get() = self.name.lower()\n```\n\n```by\n# main.by\nimport json # error: `json` imported but unused\nimport colours # ok — the extension it declares is used below\nfrom enum import Enum\n\nclass Colour(Enum):\n RED = 1\n\ndef describe(colour: Colour) -> str:\n return colour.lower_name\n```\n\nUse instead:\n\n```by\nimport colours\nfrom enum import Enum\n```\n\n## Fix safety\nThe fix removes the import, and is safe: nothing in the file reads it.\nRemoving an import also stops the imported module from being run, which\nmatters only if the module's import has a side effect the program relies\non.", + "default": "warn", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "unused-return-value": { "title": "detects a call whose result is thrown away", "description": "## What it does\nChecks for a call, written as a statement of its own, whose result is\nthrown away.\n\n## Why is this bad?\nA call on a line of its own keeps whatever the call did on the way to its\nanswer, and discards the answer. For most functions that is a mistake:\n`text.strip()` on its own line reads as if it strips `text`, and strips\nnothing — `str` is immutable, and the stripped string was the result.\n\nA function whose result really is optional says so with\n`@ignorable_return_value`, and a member of a class that says so says\notherwise with `@must_use_return_value`. A function returning `None` has\nno result to discard and is never reported.\n\n## Examples\n```python\ndef parse(text: str) -> int: ...\ndef log(message: str) -> None: ...\n\nparse(\"1\") # warning: the parsed number goes nowhere\nlog(\"started\") # ok — nothing was returned\n```\n\n```by\n@ignorable_return_value\ndef prime_cache(key: str) -> bytes: ...\n\nprime_cache(\"k\") # ok — the call was for the caching\n```", diff --git a/zensical.toml b/zensical.toml index 2db307f10f..d852eaee25 100644 --- a/zensical.toml +++ b/zensical.toml @@ -93,6 +93,7 @@ features = [ { "editor support" = "features/editor.md" }, { "language injection" = "features/language-injection.md" }, { "linting" = "features/linter.md" }, + { "unused imports" = "features/unused-imports.md" }, { "the project server" = "features/project-server.md" }, ] }, { "standard library" = [