From 2849f78a7e3282b86ef23436dbfba3b0b15c69a2 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:30:16 +1000 Subject: [PATCH 01/47] =?UTF-8?q?write=20a=20module-level=20`private`=20cl?= =?UTF-8?q?ass=20under=20the=20name=20it=20is=20emitted=20as=20wherever=20?= =?UTF-8?q?a=20lowering=20writes=20a=20type=20out=20from=20the=20checker's?= =?UTF-8?q?=20spelling=20of=20it=20=E2=80=94=20a=20soundness=20check's=20t?= =?UTF-8?q?arget,=20an=20`is`=20test,=20a=20checked=20cast,=20a=20reified?= =?UTF-8?q?=20specialization,=20a=20`raises`=20guard=20and=20an=20inferred?= =?UTF-8?q?=20annotation=20=E2=80=94=20instead=20of=20the=20name=20the=20s?= =?UTF-8?q?ource=20declares,=20which=20the=20emitted=20module=20does=20not?= =?UTF-8?q?=20bind,=20so=20`for=20x=20in=20kept`=20over=20a=20`list[Kept]`?= =?UTF-8?q?=20no=20longer=20raises=20`NameError`=20and=20`x=20is=20Kept`?= =?UTF-8?q?=20/=20`x=20cast!=20Kept`=20no=20longer=20fail=20to=20transpile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/by_irbuild/src/lib.rs | 49 +-- .../by_transforms/src/runtime/_by_runtime.py | 11 +- .../src/transforms/ast_driver.rs | 21 +- .../src/transforms/checked_cast.rs | 13 +- .../src/transforms/erased_union.rs | 19 +- .../src/transforms/init_method.rs | 60 +++- .../src/transforms/parametric_is.rs | 40 ++- .../src/transforms/raises_clause.rs | 26 +- .../src/transforms/reified_generic.rs | 17 +- .../src/transforms/repeated_underscore.rs | 7 + .../by_transforms/src/transforms/soundness.rs | 51 +++- .../src/transforms/type_reification.rs | 19 +- .../src/transforms/visibility_rename.rs | 44 ++- crates/by_transforms/src/type_info.rs | 52 +++- .../by_transforms/tests/visibility_runtime.rs | 279 ++++++++++++++++++ crates/ty_python_semantic/src/lib.rs | 1 + .../ty_python_semantic/src/semantic_model.rs | 17 +- crates/ty_python_semantic/src/types.rs | 1 + .../ty_python_semantic/src/types/display.rs | 22 ++ .../src/types/exceptions.rs | 18 +- .../src/types/infer/builder.rs | 3 +- .../src/types/reified_infer.rs | 157 +++++----- .../src/types/runtime_spelling.rs | 117 ++++++++ .../ty_python_semantic/src/types/soundness.rs | 50 ++-- 24 files changed, 877 insertions(+), 217 deletions(-) create mode 100644 crates/ty_python_semantic/src/types/runtime_spelling.rs diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index f1de48a4ea..80a1b65dfa 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -15662,26 +15662,29 @@ impl Lowering<'_, '_> { match target { CheckTarget::AnyOf(parts) => parts.iter().any(|part| self.proves(ty, part)), CheckTarget::NoneType => *ty == RType::NONE, - CheckTarget::Class(name) => match ty { - RType::Primitive(primitive) => { - let builtin = match primitive { - Primitive::Int | Primitive::Fixed(_) => "int", - Primitive::Float => "float", - Primitive::Bool | Primitive::Bit => "bool", - Primitive::Str => "str", - Primitive::List => "list", - Primitive::None - | Primitive::Object - | Primitive::Bytes - | Primitive::Dict - | Primitive::Tuple => return false, - }; - name == builtin + CheckTarget::Class(name) => { + let name = name.source(); + match ty { + RType::Primitive(primitive) => { + let builtin = match primitive { + Primitive::Int | Primitive::Fixed(_) => "int", + Primitive::Float => "float", + Primitive::Bool | Primitive::Bit => "bool", + Primitive::Str => "str", + Primitive::List => "list", + Primitive::None + | Primitive::Object + | Primitive::Bytes + | Primitive::Dict + | Primitive::Tuple => return false, + }; + name == builtin + } + RType::Instance { class, .. } => self.extends(class, &name), + RType::Tuple(_) => name == "tuple", + RType::Array(_) => false, } - RType::Instance { class, .. } => self.extends(class, name), - RType::Tuple(_) => name == "tuple", - RType::Array(_) => false, - }, + } } } @@ -15690,13 +15693,11 @@ impl Lowering<'_, '_> { fn resolve_sound_target(&mut self, target: &CheckTarget) -> Lowered { Ok(match target { CheckTarget::Class(name) => { - let class = match self.place(name) { + let name = name.source(); + let class = match self.place(&name) { Some(Place::Global { .. }) | None => { let dest = self.builder.temp(RType::OBJECT); - self.builder.push(Op::LoadGlobal { - dest, - name: name.clone(), - }); + self.builder.push(Op::LoadGlobal { dest, name }); Value::Register(dest) } Some(place) => { diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py index f671a46f46..ad663b078b 100644 --- a/crates/by_transforms/src/runtime/_by_runtime.py +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -122,11 +122,16 @@ def _target(_generic, _args): _instance = None if _generic is None else _generic.instance _owner = _args[0] if _instance is None else _instance _values.extend(_type_argument(_owner, _param) for _param in _receiver) - return _resolve(*_values) - except Exception: + except (TypeError, KeyError, IndexError): # a parameter nothing answered for: what the declaration says without - # it is the ceiling + # it is the ceiling. binding the type arguments and reading one off a + # receiver raise `TypeError`, a parameter the specialization left + # unbound `KeyError`, and a method called with no receiver `IndexError` return _allowed + # the lambda only reads the values and the names the module binds, so + # anything it raises is a lowering that names something wrong, and is not + # mistaken for a value nothing answered for + return _resolve(*_values) def _check(_exc, _generic, _args): if not _by_isinstance(_exc, _target(_generic, _args), _allowed): diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 535a9a9fad..4b4368f59d 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -1371,7 +1371,7 @@ pub(crate) fn run_against_source<'a>( return_value_use::collect(parsed_handle.suite(), &semantic_model), ); let raises_guard_pass = - raises_clause::RaisesGuardPass::new(source_ref, config.runtime_raises_checks); + raises_clause::RaisesGuardPass::new(source_ref, written, config.runtime_raises_checks); let type_fn_pass = type_fn::TypeFnPass::new(source_ref); let match_type_pass = match_type::MatchTypePass::new(source_ref, written); let modifiers_pass = modifiers::ModifiersPass::new(source_ref, written, config); @@ -1407,9 +1407,10 @@ pub(crate) fn run_against_source<'a>( kw_subscript::KwSubscriptPass::new(source_ref, written, config.min_version); let generic_call_pass = generic_call::GenericCallStripPass::new(source_ref); let reified_generic_pass = - reified_generic::ReifiedGenericPass::new(source_ref, config.min_version); + reified_generic::ReifiedGenericPass::new(source_ref, written, config.min_version); let reified_class_pass = reified_class::ReifiedClassPass::new(source_ref, config.min_version); - let type_reification_pass = type_reification::TypeReificationPass::new(config.min_version); + let type_reification_pass = + type_reification::TypeReificationPass::new(written, config.min_version); let visibility_rename_pass = visibility_rename::VisibilityRenamePass { written }; let parametric_is_pass = parametric_is::ParametricIsPass::new(source_ref, written); let implicit_typing_pass = implicit_typing::ImplicitTypingPass::new(); @@ -2832,6 +2833,20 @@ mod subsumed_lowerings { &[], &["?.", "??"], ), + case( + "ParametricIsPass", + &[Lowering::VisibilityRename], + "private class Kept:\n pass\n\ndef f(x: object) -> bool:\n return x is Kept\n", + &["isinstance(x, _Kept)"], + &["isinstance(x, Kept)"], + ), + case( + "CheckedCastPass", + &[Lowering::VisibilityRename], + "private class Kept:\n pass\n\ndef f(x: object) -> object:\n return x cast! Kept\n", + &["_checked_cast(x, _Kept)"], + &["_checked_cast(x, Kept)"], + ), ] } diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 3d10027ac8..436b2e3390 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -170,9 +170,10 @@ impl<'a> CastLower<'a> { // no parametric claim (or none that can be checked): the shallow // `isinstance` target, or the written type when ty offers no faithful one match self.types.cast_check_plan(type_arg) { - Some(CastCheck::Kind(SoundnessCheck::Isinstance(target))) => { - (helper, vec![Fragment::Lit(target.to_string())]) - } + Some(CastCheck::Kind(SoundnessCheck::Isinstance(target))) => ( + helper, + vec![Fragment::Lit(self.written.spelled(&target.spelling()))], + ), // a `Literal` is a set of values, not a class: `isinstance` against // one raises, and testing the values' class would accept every other // value of it. the check is membership @@ -251,6 +252,12 @@ pub(crate) struct CheckedCastPass<'src> { } impl TypeAwarePass for CheckedCastPass<'_> { + /// a checked cast prints the type it checks against as the runtime check it builds, + /// the emitted name of a `private` class included + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::VisibilityRename] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = CastLower::new(types, self.written); for stmt in stmts { diff --git a/crates/by_transforms/src/transforms/erased_union.rs b/crates/by_transforms/src/transforms/erased_union.rs index 8d2ce380aa..32692069f7 100644 --- a/crates/by_transforms/src/transforms/erased_union.rs +++ b/crates/by_transforms/src/transforms/erased_union.rs @@ -56,6 +56,8 @@ use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Expr, PythonVersion, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange}; +use ty_python_semantic::RuntimeSpelling; + use crate::type_info::TypeInfo; /// the stem of a synthesized type parameter's name. dunder-ish and namespaced @@ -87,6 +89,9 @@ fn plan( if taken.contains(&name) { return None; } + // the rewrite is source the phases after this one read, so a module-level + // `private` class is spelled as the source declares it and renamed with every + // other reference to it let width = union.fixed.len() + 1; let arguments: Vec = (0..width) .map(|position| { @@ -97,18 +102,26 @@ fn plan( .fixed .iter() .find(|(at, _)| *at == position) - .map(|(_, text)| text.clone()) + .map(|(_, spelling)| spelling.source()) .unwrap_or_default() } }) .collect(); Some(Rewrite { annotation: annotation.range(), - replacement: format!("{}[{}]", union.origin, arguments.join(", ")), + replacement: format!("{}[{}]", union.origin.source(), arguments.join(", ")), // spelled as a *type mapping*, which is what a union of arms is: the // parameter ranges over exactly these types and no others. a bare // `: (int, str)` would be a tuple *bound* instead, which no arm satisfies - declaration: format!("reified {name} in ({})", union.arms.join(", ")), + declaration: format!( + "reified {name} in ({})", + union + .arms + .iter() + .map(RuntimeSpelling::source) + .collect::>() + .join(", ") + ), }) } diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index f5c6f6a24f..b2980b357c 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -21,9 +21,10 @@ //! `T?`, a bare `float`). use std::cell::RefCell; +use std::collections::HashSet; use ruff_python_ast::helpers::MemberVisibility; -use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Parameter, Stmt, StmtFunctionDef}; use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -34,6 +35,7 @@ use super::mutable_defaults::{parameter_guards, undeclarable_error}; use super::repeated_underscore::WrittenNames; use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; use super::type_expr_walker::RootKind; +use super::visibility_rename::renamed_reference; use crate::config::Config; use crate::type_info::TypeInfo; @@ -60,6 +62,7 @@ impl TypeAwarePass for InitMethod<'_> { written: self.written, types, symbolic_substitutions: ctx.symbolic_substitutions.clone(), + module_private: types.private_module_symbols().into_iter().collect(), config: &self.config, edits: RefCell::new(Vec::new()), templates: RefCell::new(Vec::new()), @@ -134,6 +137,8 @@ struct State<'src> { /// line is fresh output, so a fold inside a parameter annotation is dropped /// unless it is spliced in here symbolic_substitutions: Vec<(TextRange, String)>, + /// the module-level names the module declares `private` + module_private: HashSet, config: &'src Config, edits: RefCell>, templates: RefCell)>>, @@ -184,14 +189,39 @@ impl State<'_> { /// verbatim, or the invalid basedpython surface leaks into the `.py` output. /// the required imports and any hoisted `Protocol` class come from the /// sibling passes' visit of the same parameter annotation, so this only - /// needs to reproduce the lowered text. falls back to the source verbatim - /// when nothing lowers + /// needs to reproduce the lowered text, a name a visibility keyword renamed + /// included. falls back to the source verbatim when nothing lowers fn lower_annotation(&self, ann: &Expr) -> String { + struct Renames<'a, 'src> { + state: &'a State<'src>, + renames: Vec<(TextRange, String)>, + } + impl<'ast> Visitor<'ast> for Renames<'_, '_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && let Some(renamed) = renamed_reference( + self.state.types, + self.state.written, + &self.state.module_private, + name, + ) + { + self.renames.push((name.range, renamed)); + } + walk_expr(self, expr); + } + } + let mut renames = Renames { + state: self, + renames: Vec::new(), + }; + renames.visit_expr(ann); let substitutions: Vec<(TextRange, String)> = self .symbolic_substitutions .iter() .filter(|(range, _)| ann.range().contains_range(*range)) .cloned() + .chain(renames.renames) .collect(); lower_type_expr_full( self.source, @@ -462,6 +492,30 @@ mod tests { ); } + /// the attribute's annotation names a `private` class the way the parameter's does: by + /// the name the class is emitted under, inside a type form the line lowers too + #[test] + fn a_declared_attribute_names_a_private_class_by_its_emitted_name() { + check( + indoc! {" + private class Kept: + pass + + class Holder: + init(let k: Kept, let ks: list[Kept]?) + "}, + indoc! {" + class _Kept: + pass + + class Holder: + def __init__(self, k: _Kept, ks: list[_Kept] | None): + self.k: _Kept = k + self.ks: list[_Kept] | None = ks + "}, + ); + } + /// a real modifier still applies on the parameters either side of a variadic #[test] fn modifier_survives_alongside_a_variadic() { diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index 847bb19895..8008427d43 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -56,7 +56,9 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{self as ast, CmpOp, Expr, PySourceType, Stmt}; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange}; -use ty_python_semantic::{ArgVariance, ParametricIsPlan, ProtocolMemberCheck, TargetSpelling}; +use ty_python_semantic::{ + ArgVariance, ParametricIsPlan, ProtocolMemberCheck, RuntimeSpelling, TargetSpelling, +}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::repeated_underscore::WrittenNames; @@ -101,9 +103,13 @@ fn variance_code(variance: ArgVariance) -> u8 { /// validates the same structural claim. each entry is a kind-tagged tuple: /// `(\"attr\", name, type, variance)` or `(\"method\", name, [(type, variance), /// …], return_or_None)` -fn protocol_members_literal(checks: &[ProtocolMemberCheck]) -> String { - let type_variance = |(expected, variance): &(String, ArgVariance)| { - format!("({expected}, {})", variance_code(*variance)) +fn protocol_members_literal(written: WrittenNames, checks: &[ProtocolMemberCheck]) -> String { + let type_variance = |(expected, variance): &(RuntimeSpelling, ArgVariance)| { + format!( + "({}, {})", + written.spelled(expected), + variance_code(*variance) + ) }; let entries = checks .iter() @@ -113,7 +119,8 @@ fn protocol_members_literal(checks: &[ProtocolMemberCheck]) -> String { expected, variance, } => format!( - "(\"attr\", {name:?}, {expected}, {})", + "(\"attr\", {name:?}, {}, {})", + written.spelled(expected), variance_code(*variance) ), ProtocolMemberCheck::Method { name, params, ret } => { @@ -269,7 +276,7 @@ fn fragments_for_plan( Fragment::Lit(call.to_owned()), value_ref(), Fragment::Lit(", ".to_owned()), - target_fragment(&target, arm), + target_fragment(written, &target, arm), ]; frags.push(Fragment::Lit(format!(", {})", variance_tuple(&codes)))); frags @@ -277,7 +284,7 @@ fn fragments_for_plan( ParametricIsPlan::ProtocolStructural(checks) => { needs.protocol_runtime = true; needs.references_value = true; - let members = protocol_members_literal(&checks); + let members = protocol_members_literal(written, &checks); vec![ Fragment::Lit("_by_protocol_is(".to_owned()), value_ref(), @@ -303,7 +310,7 @@ fn fragments_for_plan( Fragment::Lit("_by_conforms(".to_owned()), value_ref(), Fragment::Lit(", ".to_owned()), - target_fragment(&target, arm), + target_fragment(written, &target, arm), Fragment::Lit(format!("{spelled})")), ] } @@ -337,7 +344,7 @@ fn fragments_for_plan( value_ref(), Fragment::Lit(", ".to_owned()), ]; - frags.push(target_fragment(&target, arm)); + frags.push(target_fragment(written, &target, arm)); frags.push(Fragment::Lit(")".to_owned())); frags } @@ -366,7 +373,7 @@ fn fragments_for_plan( Fragment::Lit(format!(") is {} and ", written.builtin(&class))), value_ref(), Fragment::Lit(" == ".to_owned()), - target_fragment(&value, arm), + target_fragment(written, &value, arm), Fragment::Lit(")".to_owned()), ] } @@ -375,7 +382,7 @@ fn fragments_for_plan( vec![ value_ref(), Fragment::Lit(" is ".to_owned()), - target_fragment(&target, arm), + target_fragment(written, &target, arm), ] } ParametricIsPlan::Subclass(target) => { @@ -391,7 +398,7 @@ fn fragments_for_plan( ))); frags.push(value_ref()); frags.push(Fragment::Lit(", ".to_owned())); - frags.push(target_fragment(&target, arm)); + frags.push(target_fragment(written, &target, arm)); frags.push(Fragment::Lit("))".to_owned())); frags } @@ -436,10 +443,10 @@ fn fragments_for_plan( /// how a plan's target is written: its own spelling, or the source the target /// was written as when the plan carries none -fn target_fragment(target: &TargetSpelling, arm: &Expr) -> Fragment { +fn target_fragment(written: WrittenNames, target: &TargetSpelling, arm: &Expr) -> Fragment { match target { TargetSpelling::Written => Fragment::Src(arm.range()), - TargetSpelling::Rebuilt(text) => Fragment::Lit(text.clone()), + TargetSpelling::Rebuilt(spelling) => Fragment::Lit(written.spelled(spelling)), } } @@ -740,12 +747,13 @@ impl TypeAwarePass for ParametricIsPass<'_> { Some(super::ast_driver::Lowering::ParametricIs) } - /// the type it tests against is printed as the runtime probe it builds, literals and - /// the `None` of an optional included + /// the type it tests against is printed as the runtime probe it builds, literals, the + /// `None` of an optional and the emitted name of a `private` class included fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { &[ super::ast_driver::Lowering::LiteralType, super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::VisibilityRename, ] } diff --git a/crates/by_transforms/src/transforms/raises_clause.rs b/crates/by_transforms/src/transforms/raises_clause.rs index 95fe886bc0..b0b1bbf4f5 100644 --- a/crates/by_transforms/src/transforms/raises_clause.rs +++ b/crates/by_transforms/src/transforms/raises_clause.rs @@ -25,6 +25,7 @@ use ruff_python_ast::{ModModule, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; /// Deletes every `raises` clause. @@ -85,12 +86,17 @@ fn clause_range(source: &str, function: &StmtFunctionDef) -> Option { /// Wraps each function with a runtime-testable `raises` clause in a guard. pub(crate) struct RaisesGuardPass<'src> { source: &'src str, + written: WrittenNames<'src>, enabled: bool, } impl<'src> RaisesGuardPass<'src> { - pub(crate) fn new(source: &'src str, enabled: bool) -> Self { - Self { source, enabled } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, enabled: bool) -> Self { + Self { + source, + written, + enabled, + } } } @@ -113,6 +119,7 @@ impl super::ast_driver::TypeAwarePass for RaisesGuardPass<'_> { for stmt in stmts { let mut collector = GuardCollector { source: self.source, + written: self.written, types, guards: &mut guards, }; @@ -135,6 +142,7 @@ impl super::ast_driver::TypeAwarePass for RaisesGuardPass<'_> { /// exactly as much in need of its contract as a top-level one. struct GuardCollector<'a> { source: &'a str, + written: WrittenNames<'a>, types: &'a dyn TypeInfo, guards: &'a mut Vec<(TextRange, String)>, } @@ -142,7 +150,7 @@ struct GuardCollector<'a> { impl<'ast> Visitor<'ast> for GuardCollector<'_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { if let Stmt::FunctionDef(function) = stmt - && let Some(guard) = guard_for(self.source, function, self.types) + && let Some(guard) = guard_for(self.source, self.written, function, self.types) { self.guards.push(guard); } @@ -161,6 +169,7 @@ impl<'ast> Visitor<'ast> for GuardCollector<'_> { /// there, so they stay above the decorator rather than between it and its `def`. fn guard_for( source: &str, + written: WrittenNames, function: &StmtFunctionDef, types: &dyn TypeInfo, ) -> Option<(TextRange, String)> { @@ -172,6 +181,7 @@ fn guard_for( let target = types.declared_raises_runtime_target(function)?; let (offset, indent) = def_keyword(source, function)?; let name = function.name.as_str(); + let ceiling = written.spelled(&target.ceiling); let arguments = match &target.resolved { Some(resolved) => { @@ -181,19 +191,19 @@ fn guard_for( .chain(&resolved.receiver) .map(String::as_str) .collect(); + let expression = written.spelled(&resolved.expression); let lambda = if parameters.is_empty() { - format!("lambda: {}", resolved.expression) + format!("lambda: {expression}") } else { - format!("lambda {}: {}", parameters.join(", "), resolved.expression) + format!("lambda {}: {expression}", parameters.join(", ")) }; format!( - "{}, \"{name}\", {lambda}, {}, {}", - target.ceiling, + "{ceiling}, \"{name}\", {lambda}, {}, {}", python_names(&resolved.own), python_names(&resolved.receiver), ) } - None => format!("{}, \"{name}\"", target.ceiling), + None => format!("{ceiling}, \"{name}\""), }; Some(( diff --git a/crates/by_transforms/src/transforms/reified_generic.rs b/crates/by_transforms/src/transforms/reified_generic.rs index ade4a6c1a3..e6fbeb6fac 100644 --- a/crates/by_transforms/src/transforms/reified_generic.rs +++ b/crates/by_transforms/src/transforms/reified_generic.rs @@ -54,6 +54,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::reified::reified_type_param_names; use super::ast_driver::{PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use super::source_util::{line_indent, line_start}; use crate::type_info::TypeInfo; @@ -147,6 +148,7 @@ impl<'ast> Visitor<'ast> for ReifiedGeneric<'_> { /// inserts the inferred `[...]` specialization at bare reified call sites struct SpecializationInjector<'ti> { types: &'ti dyn TypeInfo, + written: WrittenNames<'ti>, edits: Vec<(TextRange, String)>, } @@ -157,8 +159,10 @@ impl<'ast> Visitor<'ast> for SpecializationInjector<'_> { && !matches!(call.func.as_ref(), Expr::Subscript(_)) && let Some(specialization) = self.types.reified_call_specialization(call) { - self.edits - .push((TextRange::empty(call.func.range().end()), specialization)); + self.edits.push(( + TextRange::empty(call.func.range().end()), + self.written.spelled(&specialization), + )); } walk_expr(self, expr); } @@ -166,13 +170,19 @@ impl<'ast> Visitor<'ast> for SpecializationInjector<'_> { pub(crate) struct ReifiedGenericPass<'src> { source: &'src str, + written: WrittenNames<'src>, min_version: PythonVersion, } impl<'src> ReifiedGenericPass<'src> { - pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + min_version: PythonVersion, + ) -> Self { Self { source, + written, min_version, } } @@ -196,6 +206,7 @@ impl TypeAwarePass for ReifiedGenericPass<'_> { // call only when this injection exists, so the two stay in agreement let mut injector = SpecializationInjector { types, + written: self.written, edits: Vec::new(), }; for stmt in stmts { diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index 12d3c15a66..edda10f9d0 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -24,6 +24,7 @@ use ruff_python_ast::visitor::{ use ruff_python_ast::{Expr, ModModule, Parameter, Parameters, Stmt}; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange}; +use ty_python_semantic::RuntimeSpelling; use ty_python_semantic::types::repeated_underscore::{ self as decision, UnderscoreLowering, UnderscoreRefusal, callable_parameter_slots, lower_repeated_underscores, lowered_parameter_names, parameter_slots, @@ -612,6 +613,12 @@ impl<'src> WrittenNames<'src> { } } + /// `spelling`, which the checker spelled for this module, as the module is emitted: each + /// module-level `private` symbol it reads under the name the symbol is emitted under + pub(crate) fn spelled(self, spelling: &RuntimeSpelling) -> String { + spelling.spelled(|symbol| self.module_private(symbol)) + } + /// the name a lowering writes `name`, which `module` exports, under — `typing`, /// `typing_extensions`, `collections.abc` and the like /// diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index a8f83f7346..1b804bf43b 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -520,12 +520,14 @@ impl PrologueStatement for EntryGuard { } /// what the checks a pass decided on are written as -#[derive(Default)] #[expect( clippy::struct_excessive_bools, reason = "independent runtime-helper usage flags, not a state machine" )] -struct Rendered { +struct Rendered<'src> { + /// the module the checks are written into, whose `private` symbols a check + /// names as the module is emitted + written: WrittenNames<'src>, /// each value passed through a call: the span it is at, the text opening the call /// and the text closing it wraps: Vec<(TextRange, String, String)>, @@ -543,7 +545,26 @@ struct Rendered { used_parametric: bool, } -impl Rendered { +impl<'src> Rendered<'src> { + fn new(written: WrittenNames<'src>) -> Self { + Self { + written, + wraps: Vec::new(), + guards: Vec::new(), + unanchored: Vec::new(), + used_iter: false, + used_aiter: false, + used_iter_p: false, + used_aiter_p: false, + used_parametric: false, + } + } + + /// `isinstance`'s second argument for `target` + fn target(&self, target: &CheckTarget) -> String { + self.written.spelled(&target.spelling()) + } + /// wrap `expr` in `helper(, )`. `trailing` /// carries its own leading `, ` (e.g. `", str"` or `", A[int], (0,)"`) fn wrap_call(&mut self, expr: &Expr, helper: &str, trailing: &str) { @@ -564,10 +585,12 @@ impl Rendered { fn wrap_check(&mut self, expr: &Expr, plan: &SoundnessCheck) { match plan { SoundnessCheck::Isinstance(target) => { + let target = self.target(target); self.wrap_call(expr, "_soundness_check", &format!(", {target}")); } SoundnessCheck::Parametric { alias, variances } => { self.used_parametric = true; + let alias = self.written.spelled(alias); self.wrap_call( expr, "_soundness_parametric", @@ -589,7 +612,7 @@ impl Rendered { self.used_iter = true; "_soundness_iter" }; - (helper, format!(", {target}")) + (helper, format!(", {}", self.target(target))) } SoundnessCheck::Parametric { alias, variances } => { self.used_parametric = true; @@ -600,7 +623,14 @@ impl Rendered { self.used_iter_p = true; "_soundness_iter_p" }; - (helper, format!(", {alias}, {}", variance_tuple(variances))) + ( + helper, + format!( + ", {}, {}", + self.written.spelled(alias), + variance_tuple(variances) + ), + ) } }; self.wrap_call(iterable, helper, &trailing); @@ -617,11 +647,14 @@ impl Rendered { /// the guard statement that validates parameter `name` against `plan` fn guard_stmt(&mut self, name: &str, plan: &SoundnessCheck) -> String { match plan { - SoundnessCheck::Isinstance(target) => format!("_soundness_check({name}, {target})"), + SoundnessCheck::Isinstance(target) => { + format!("_soundness_check({name}, {})", self.target(target)) + } SoundnessCheck::Parametric { alias, variances } => { self.used_parametric = true; format!( - "_soundness_parametric({name}, {alias}, {})", + "_soundness_parametric({name}, {}, {})", + self.written.spelled(alias), variance_tuple(variances) ) } @@ -679,7 +712,7 @@ impl TypeAwarePass for SoundnessPass<'_> { if !self.positions.any() { return; } - let mut inner = Rendered::default(); + let mut inner = Rendered::new(self.written); let mut rerendered = Vec::new(); for (idx, stmt) in stmts.iter().enumerate() { let mut walker = Soundness::new(types, self.positions, self.written); @@ -687,7 +720,7 @@ impl TypeAwarePass for SoundnessPass<'_> { // an AST-mutation pass rewrote this statement, and an edit at a node it // replaced would be dropped: its checks go into the tree it is printed from if ctx.changed.contains(&idx) { - let mut rendered = Rendered::default(); + let mut rendered = Rendered::new(self.written); let mut checks = Vec::new(); for site in &walker.sites { match site { diff --git a/crates/by_transforms/src/transforms/type_reification.rs b/crates/by_transforms/src/transforms/type_reification.rs index 9e123ab8eb..15450666c6 100644 --- a/crates/by_transforms/src/transforms/type_reification.rs +++ b/crates/by_transforms/src/transforms/type_reification.rs @@ -78,6 +78,7 @@ use ruff_python_ast::{self as ast, Expr, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use crate::type_info::{TypeInfo, trailing_name}; /// dunder assignments whose values static readers (linters, type checkers, @@ -105,6 +106,7 @@ fn is_version_info(expr: &Expr) -> bool { struct Reifier<'ti> { types: &'ti dyn TypeInfo, + written: WrittenNames<'ti>, edits: Vec<(TextRange, String)>, } @@ -146,7 +148,7 @@ impl<'ast> Visitor<'ast> for Reifier<'_> { { self.edits.push(( TextRange::empty(call.func.range().end()), - format!("[{arguments}]"), + format!("[{}]", self.written.spelled(&arguments)), )); } walk_expr(self, expr); @@ -174,17 +176,21 @@ impl<'ast> Visitor<'ast> for Reifier<'_> { } } -pub(crate) struct TypeReificationPass { +pub(crate) struct TypeReificationPass<'src> { + written: WrittenNames<'src>, min_version: PythonVersion, } -impl TypeReificationPass { - pub(crate) fn new(min_version: PythonVersion) -> Self { - Self { min_version } +impl<'src> TypeReificationPass<'src> { + pub(crate) fn new(written: WrittenNames<'src>, min_version: PythonVersion) -> Self { + Self { + written, + min_version, + } } } -impl TypeAwarePass for TypeReificationPass { +impl TypeAwarePass for TypeReificationPass<'_> { // the specialization is spelled out so the value a call constructs // carries it as the program runs fn runtime_only(&self) -> bool { @@ -197,6 +203,7 @@ impl TypeAwarePass for TypeReificationPass { } let mut reifier = Reifier { types, + written: self.written, edits: Vec::new(), }; for stmt in stmts { diff --git a/crates/by_transforms/src/transforms/visibility_rename.rs b/crates/by_transforms/src/transforms/visibility_rename.rs index bcc7e7a27b..0c5bd82f5a 100644 --- a/crates/by_transforms/src/transforms/visibility_rename.rs +++ b/crates/by_transforms/src/transforms/visibility_rename.rs @@ -104,6 +104,31 @@ impl<'a> StatementVisitor<'a> for RenamedDeclarations { } } +/// the name the reference `name` is written under, when a visibility keyword renamed what +/// it reads: a module-level `private` symbol, given the module's `module_private` names, or a +/// bare name in a class body that is one of the class's own restricted members +/// +/// a reference is renamed only when it reads the renamed binding — a parameter, a local or a +/// class attribute that shares the name is a different binding, and renaming it would split +/// it from its uses — so ty is asked where each resolves +pub(crate) fn renamed_reference( + types: &dyn TypeInfo, + written: WrittenNames, + module_private: &HashSet, + name: &ast::ExprName, +) -> Option { + // a name the parser synthesized has no source to rename + if name.ctx == ExprContext::Invalid || name.range.is_empty() { + return None; + } + if module_private.contains(name.id.as_str()) + && types.resolves_to_module_scope(name) == Some(true) + { + return Some(written.module_private(name.id.as_str())); + } + types.class_body_member_name(name) +} + /// the kind of scope a statement is directly in #[derive(Clone, Copy, PartialEq, Eq)] enum LexicalScope { @@ -361,25 +386,12 @@ impl<'ast> Visitor<'ast> for Renamer<'_> { } fn visit_expr(&mut self, expr: &'ast Expr) { - // a reference to a module-level private symbol is renamed only when it is - // one — a parameter, a local or a class attribute that shares the name is a - // different binding, and renaming it would split it from its uses — and a - // bare name in a class body when it is one of the class's own restricted - // members. ty is asked where each resolves if let Expr::Name(name) = expr - && name.ctx != ExprContext::Invalid - // a name the parser synthesized has no source to rename - && !name.range.is_empty() && !self.declarations.contains(&name.range) + && let Some(renamed) = + renamed_reference(self.types, self.written, &self.module_private, name) { - if self.module_private.contains(name.id.as_str()) - && self.types.resolves_to_module_scope(name) == Some(true) - { - self.edits - .push((name.range, self.written.module_private(name.id.as_str()))); - } else if let Some(renamed) = self.types.class_body_member_name(name) { - self.edits.push((name.range, renamed)); - } + self.edits.push((name.range, renamed)); } if let Expr::Attribute(attribute) = expr && let Some(mangled) = self.types.restricted_member_name(attribute) diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 2205486c40..92afa7add7 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -12,6 +12,7 @@ use ruff_python_parser::parse_expression; use ruff_text_size::{Ranged, TextRange}; use ty_python_core::scope::ScopeKind; use ty_python_core::{global_scope, place_table, semantic_index, use_def_map}; +use ty_python_semantic::RuntimeSpelling; use ty_python_semantic::types::call_type_forms::CallTypeForm; use ty_python_semantic::types::exceptions::RaisesRuntimeTarget; use ty_python_semantic::types::ide_support::BaseTypeArgument; @@ -79,26 +80,43 @@ pub(crate) struct SynthesizedType { /// binds further down, or the class being defined, or one nested in it. evaluated there it /// raises, so it is written as a string pub(crate) forward: bool, + /// the module-level `private` classes the text names, each written bare and read + /// nowhere in the text but as that class + pub(crate) module_private: Vec, } impl SynthesizedType { /// the type as the module `written` is of writes it, each of [`Self::typing_names`] under - /// the name the lowering imports it by + /// the name the lowering imports it by, and each of [`Self::module_private`] under the + /// name it is emitted as pub(crate) fn text_in(&self, written: WrittenNames) -> String { struct Roots<'a> { typing_names: &'a [&'static str], + module_private: &'a [String], written: WrittenNames<'a>, edits: Vec<(TextRange, String)>, } + impl Roots<'_> { + /// the name the module writes `name`, a root the text reads, under + fn local(&self, name: &str) -> Option { + match name { + _ if self.typing_names.contains(&name) => { + Some(self.written.imported("typing", name)) + } + _ if self.module_private.iter().any(|class| class == name) => { + Some(self.written.module_private(name)) + } + _ => None, + } + } + } impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for Roots<'_> { fn visit_expr(&mut self, expr: &'ast Expr) { if let Expr::Name(name) = expr - && self.typing_names.contains(&name.id.as_str()) + && let Some(local) = self.local(name.id.as_str()) + && local != name.id.as_str() { - let local = self.written.imported("typing", name.id.as_str()); - if local != name.id.as_str() { - self.edits.push((name.range, local)); - } + self.edits.push((name.range, local)); } ruff_python_ast::visitor::walk_expr(self, expr); } @@ -108,6 +126,7 @@ impl SynthesizedType { }; let mut roots = Roots { typing_names: &self.typing_names, + module_private: &self.module_private, written, edits: Vec::new(), }; @@ -284,14 +303,20 @@ pub(crate) trait TypeInfo { /// call form when a keyword-variadic pack contributes fields), or `None` /// when the call needs no injection or none is possible (ty reports the /// latter) - fn reified_call_specialization(&self, call: &ruff_python_ast::ExprCall) -> Option; + fn reified_call_specialization( + &self, + call: &ruff_python_ast::ExprCall, + ) -> Option; /// the comma-joined type arguments to inject at a bare constructor call /// of a generic class (`A(1)` → `"int"`), or `None` when the callee is /// not a generic class literal or the solved specialization has no /// runtime spelling — reification of constructors is best-effort, a /// missing spelling is never an error - fn constructor_specialization(&self, call: &ruff_python_ast::ExprCall) -> Option; + fn constructor_specialization( + &self, + call: &ruff_python_ast::ExprCall, + ) -> Option; /// when the parameter annotation `annotation` denotes a union of /// specializations of one erased origin (`list[int] | list[str]`), how the @@ -932,11 +957,17 @@ impl TypeInfo for SemanticModel<'_> { .is_some_and(|function| function.is_reified(self.db())) } - fn reified_call_specialization(&self, call: &ruff_python_ast::ExprCall) -> Option { + fn reified_call_specialization( + &self, + call: &ruff_python_ast::ExprCall, + ) -> Option { SemanticModel::reified_call_specialization(self, call) } - fn constructor_specialization(&self, call: &ruff_python_ast::ExprCall) -> Option { + fn constructor_specialization( + &self, + call: &ruff_python_ast::ExprCall, + ) -> Option { self.reified_constructor_type_arguments(call) } @@ -1783,6 +1814,7 @@ fn spell_for_python<'db>( modules, typing_names, forward, + module_private: spelling.module_private_classes, }; resolvable_in_output( model, diff --git a/crates/by_transforms/tests/visibility_runtime.rs b/crates/by_transforms/tests/visibility_runtime.rs index 91699ac668..cafe837a3f 100644 --- a/crates/by_transforms/tests/visibility_runtime.rs +++ b/crates/by_transforms/tests/visibility_runtime.rs @@ -469,3 +469,282 @@ fn a_renamed_symbol_never_takes_a_name_the_module_already_has() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); interpreters::ran(&interpreter, target); } + +/// a module-level `private` class named by the checks a lowering writes from a type rather +/// than from the source: the soundness checks on a loop, an assignment, a return, an +/// argument and a projection — through a container, through a private type alias, from a +/// method of another class, and against a private generic's specialization, by a public +/// argument and by a private one +const SOUNDNESS_CHECKS: &str = r#" +from typing import Any + +private class Kept: + init(let n: int) + +private class Box[T]: + init(let v: T) + +private type Alias = Kept + +private var kept: list[Kept] = [Kept(1), Kept(2)] + +def dynamic(value: object) -> Any: + return value + +def looped() -> int: + var sum = 0 + for one in kept: + sum += one.n + return sum + +def assigned() -> int: + k: Kept = dynamic(Kept(3)) + return k.n + +def returned() -> Kept: + return dynamic(Kept(4)) + +def takes(k: Kept) -> int: + return k.n + +def argument() -> int: + return takes(dynamic(Kept(5))) + +def projected() -> int: + nested: dict[str, list[Kept]] = {"a": [Kept(6)]} + return nested["a"][0].n + +def aliased() -> int: + a: Alias = dynamic(Kept(7)) + return a.n + +class Reader: + def read(self) -> int: + var sum = 0 + for one in kept: + sum += one.n + return sum + +def boxed() -> int: + boxes: list[Box[int]] = [Box[int](8)] + var sum = 0 + for box in boxes: + sum += box.v + return sum + +def kept_boxes() -> int: + boxes: list[Box[Kept]] = [Box[Kept](Kept(9))] + var sum = 0 + for box in boxes: + sum += box.v.n + return sum + +def results() -> list[int]: + return [looped(), assigned(), returned().n, argument(), projected(), aliased(), Reader().read(), boxed(), kept_boxes()] +"#; + +const SOUNDNESS_CHECKS_IMPORTER: &str = r#" +import importlib +m = importlib.import_module("emitted") +assert m.results() == [3, 3, 4, 5, 6, 7, 3, 8, 9], m.results() +m.dynamic = lambda value: "not kept" +try: + m.returned() +except TypeError as error: + assert "_Kept" in str(error), error +else: + raise AssertionError("the return check let a `str` through") +assert not hasattr(m, "Kept") and not hasattr(m, "Box") +print("ok") +"#; + +#[test] +fn a_soundness_check_names_a_private_class_by_its_renamed_binding() { + // the program declares a generic class with pep 695 syntax + let Some(interpreter) = interpreters::oldest(PythonVersion::PY312, "", "") else { + return; + }; + let target = interpreter.version.min(PythonVersion::PY313); + let config = Config { + min_version: target, + ..Config::default() + }; + let transpiled = transpile(SOUNDNESS_CHECKS, &config).expect("transpile should succeed"); + + let dir = tempfile::tempdir().expect("failed to create a temp dir"); + std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); + + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(SOUNDNESS_CHECKS_IMPORTER) + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "importing the transpiled module failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} + +/// a module-level `private` class named by the other lowerings that write a type out at +/// runtime: an `is` type test against a class, a generic, a protocol's member and an enum +/// member, the checked casts, the specialization a reified generic and an erased union are +/// called with, the guard a `raises` clause is checked by, and the annotation inferred for +/// a class attribute, which python evaluates as the class body runs +const RUNTIME_TYPES: &str = r#" +from enum import Enum +from typing import Protocol + +private class Kept: + init(let n: int) + +private class Box[T]: + init(let v: T) + +private class Oops(Exception): + pass + +class Holder[T]: + init(let v: T) + + def kind(self) -> object: + return T + +def kind_of[T](value: T) -> object: + return T + +def is_kept(value: object) -> bool: + return value is Kept + +def is_int_box(value: object) -> bool: + return value is Box[int] + +private class HasKept(Protocol): + k: Kept + + def get(self, other: Kept) -> Kept: ... + +class Shaped: + k: Kept = Kept(6) + + def get(self, other: Kept) -> Kept: + return other + +def has_kept(value: object) -> bool: + return value is HasKept + +private class Color(Enum): + RED = 1 + BLUE = 2 + +def is_red(value: object) -> bool: + return value is Color.RED + +def count(values: list[Kept] | list[int]) -> int: + return len(values) + +def counted() -> int: + return count([Kept(7), Kept(8)]) + +def checked(value: object) -> int: + return (value cast! Kept).n + +def tried(value: object) -> Kept?: + return value cast? Kept + +def held() -> object: + return Holder(Kept(1)).kind() + +def called() -> object: + return kind_of(Kept(2)) + +def fails() -> int raises Oops: + raise Oops() + +def rethrow[reified E: Exception](error: E) raises E | Oops: + raise error + +def made() -> Kept: + return Kept(5) + +class Shelf: + var item = made() +"#; + +const RUNTIME_TYPES_IMPORTER: &str = r#" +import importlib +m = importlib.import_module("emitted") +kept = m._Kept(3) +assert m.is_kept(kept) and not m.is_kept(3) +assert m.is_int_box(m._Box[int](4)) and not m.is_int_box(m._Box[str]("4")) +assert m.has_kept(m.Shaped()) and not m.has_kept(object()) +assert m.is_red(m._Color.RED) and not m.is_red(m._Color.BLUE) +assert m.counted() == 2 +assert m.checked(kept) == 3 +try: + m.checked(3) +except TypeError: + pass +else: + raise AssertionError("`cast!` let an `int` through") +assert m.tried(kept) is kept and m.tried(3) is None +assert m.held() is m._Kept and m.called() is m._Kept +try: + m.fails() +except m._Oops: + pass +# the guard reads `E` off the specialization, and tests `(E, _Oops)` rather than the +# ceiling `Exception`, which would let the `KeyError` through +for raised in (ValueError("v"), m._Oops()): + try: + m.rethrow[ValueError](raised) + except type(raised): + pass +try: + m.rethrow[ValueError](KeyError("k")) +except AssertionError as error: + assert "rethrow raised KeyError" in str(error), error +else: + raise AssertionError("the `raises` guard let a `KeyError` through") +assert m.Shelf.__annotations__["item"] is m._Kept +print("ok") +"#; + +#[test] +fn a_type_written_out_at_runtime_names_a_private_class_by_its_renamed_binding() { + // the program declares a generic class with pep 695 syntax + let Some(interpreter) = interpreters::oldest(PythonVersion::PY312, "", "") else { + return; + }; + let target = interpreter.version.min(PythonVersion::PY313); + let config = Config { + min_version: target, + runtime_raises_checks: true, + ..Config::default() + }; + let transpiled = transpile(RUNTIME_TYPES, &config).expect("transpile should succeed"); + + let dir = tempfile::tempdir().expect("failed to create a temp dir"); + std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); + + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(RUNTIME_TYPES_IMPORTER) + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "importing the transpiled module failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index 141e755c09..f0c3a57075 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -68,6 +68,7 @@ pub use types::reified_infer::{ ArgVariance, ErasedTargetReason, ErasedUnion, ParametricIsPlan, ProtocolMemberCheck, TargetSpelling, }; +pub use types::runtime_spelling::RuntimeSpelling; pub use types::static_resource::{ResourceError, render_as, resolve_static_resource}; pub use types::template::finite_string_set; pub use types::visibility::private_symbols; diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 0bc8ef0341..71cc911518 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -15,6 +15,7 @@ use ty_module_resolver::{ }; use crate::Db; +use crate::RuntimeSpelling; use crate::place::implicit_globals::all_implicit_module_globals; use crate::place::imported_symbol; use crate::place::{builtins_module_scope, implicit_builtins_symbol_scope}; @@ -216,12 +217,11 @@ impl<'db> SemanticModel<'db> { .to_instance_approximation(self.db, &self.program_environment()) } - /// basedpython: the source text of the specialization step the transpiler - /// splices in after the callee of a bare reified-generic call (`f(1)` → - /// `"[int]"`). `None` when the call is not a bare reified-generic call or - /// no injectable spelling exists — the checker reports the latter as - /// `unspecialized-reified-generic` - pub fn reified_call_specialization(&self, call: &ast::ExprCall) -> Option { + /// basedpython: the specialization step the transpiler splices in after the + /// callee of a bare reified-generic call (`f(1)` → `"[int]"`). `None` when + /// the call is not a bare reified-generic call or no injectable spelling + /// exists — the checker reports the latter as `unspecialized-reified-generic` + pub fn reified_call_specialization(&self, call: &ast::ExprCall) -> Option { let env = &self.program_environment(); let db = self.db; let callee_ty = call.func.inferred_type(self)?; @@ -1052,7 +1052,10 @@ impl<'db> SemanticModel<'db> { /// `None` when the callee is not a generic class literal or no runtime /// spelling exists — reification of constructors is best-effort, never /// an error - pub fn reified_constructor_type_arguments(&self, call: &ast::ExprCall) -> Option { + pub fn reified_constructor_type_arguments( + &self, + call: &ast::ExprCall, + ) -> Option { let env = &self.program_environment(); let callee_ty = call.func.inferred_type(self)?; let class_literal = callee_ty.as_class_literal()?; diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index e08e009fc1..ec19af2b04 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -234,6 +234,7 @@ pub(crate) mod relation; mod relation_error; pub mod repeated_underscore; mod restricted; +pub(crate) mod runtime_spelling; mod safe_variance; mod set_theoretic; mod signatures; diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 41c76ad0d9..9c1a382bfe 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -32,6 +32,7 @@ use crate::types::signatures::{ }; use crate::types::tuple::{RepeatedUnit, TupleSpec, VariableLengthTuple, VariableSegment}; use crate::types::typevar::BoundTypeVarIdentity; +use crate::types::visibility::private_symbols; use crate::types::visitor::TypeVisitor; use crate::types::{ CallableType, DeferredOperation, DeferredType, DynamicType, IntersectionType, @@ -986,6 +987,9 @@ pub struct SourceSpelling { /// the outer one, `Outer.Inner`. such a path names the outer class, which is unbound while /// it is being made, so it resolves only where it is not evaluated as it is written pub reads_nested_class: bool, + /// those of [`Self::bare_classes`] that are module-level `private` symbols of the file, + /// which the transpiler emits under a name of its own + pub module_private_classes: Vec, } /// Collects every class a type names, so each can be spelled for a particular file. @@ -1153,6 +1157,23 @@ impl<'db> Type<'db> { return None; } } + // a bare class the file's global scope binds under a name the file declares + // `private` is that `private` symbol + let private = private_symbols(db, file.file(db)); + let module_private_classes = bare + .iter() + .filter(|name| { + private.contains(**name) + && matches!( + global_symbol(db, file, name).place, + Place::Defined(defined) if defined + .ty + .as_class_literal() + .is_some_and(|class| classes.contains(&class)) + ) + }) + .map(|name| (*name).to_owned()) + .collect(); let bare_classes = bare.into_iter().map(str::to_owned).collect(); let settings = DisplaySettings { @@ -1164,6 +1185,7 @@ impl<'db> Type<'db> { modules: modules.into_iter().collect(), bare_classes, reads_nested_class, + module_private_classes, }) } } diff --git a/crates/ty_python_semantic/src/types/exceptions.rs b/crates/ty_python_semantic/src/types/exceptions.rs index db211d07df..85048929cc 100644 --- a/crates/ty_python_semantic/src/types/exceptions.rs +++ b/crates/ty_python_semantic/src/types/exceptions.rs @@ -63,6 +63,7 @@ use crate::types::diagnostic::{ }; use crate::types::function::{FunctionLiteral, FunctionType, OverloadLiteral}; use crate::types::generics::{ApplySpecialization, Specialization, enclosing_binding_contexts}; +use crate::types::runtime_spelling::RuntimeSpelling; use crate::types::typevar::{BindingContext, BoundTypeVarInstance, TypeVarBoundOrConstraints}; use crate::types::visitor::any_over_type; use crate::types::{ @@ -1137,7 +1138,7 @@ pub struct RaisesRuntimeTarget { /// The `isinstance` target with every type parameter widened to its declared /// ceiling — what the guard tests when nothing says which type the caller /// chose. - pub ceiling: String, + pub ceiling: RuntimeSpelling, /// How to build the exact target at the call, when the clause names a type /// parameter that carries a runtime value. pub resolved: Option, @@ -1148,7 +1149,7 @@ pub struct RaisesRuntimeTarget { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResolvedRaisesTarget { /// The target, with each of those parameters spelled by its name. - pub expression: String, + pub expression: RuntimeSpelling, /// The guarded function's own reified parameters, read off the specialization /// it is called through. pub own: Vec, @@ -1187,7 +1188,7 @@ pub fn declared_raises_runtime_target<'db>( } if declared.is_never() { return Some(RaisesRuntimeTarget { - ceiling: "()".to_string(), + ceiling: RuntimeSpelling::text("()"), resolved: None, }); } @@ -1249,7 +1250,7 @@ fn resolved_runtime_target<'db>( let mut own = Vec::new(); let mut receiver = Vec::new(); let mut reads_any = false; - let mut parts: Vec = Vec::new(); + let mut parts: Vec = Vec::new(); for element in union_elements(db, declared) { let part = match element { Type::TypeVar(bound_typevar) if let Some(source) = source_of(bound_typevar) => { @@ -1265,7 +1266,7 @@ fn resolved_runtime_target<'db>( names.push(name.clone()); } reads_any = true; - name + RuntimeSpelling::text(name) } element => crate::types::soundness::runtime_check_target( db, @@ -1286,7 +1287,12 @@ fn resolved_runtime_target<'db>( // flattening let expression = match parts.len() { 1 => parts.pop()?, - _ => format!("({})", parts.join(", ")), + _ => { + let mut expression = RuntimeSpelling::text("("); + expression.push(&RuntimeSpelling::join(parts, ", ")); + expression.push_str(")"); + expression + } }; Some(ResolvedRaisesTarget { expression, diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index c6ec47017f..957fac1e77 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -11164,7 +11164,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { )); match runtime_check_target(db, env, self.file(), target) { Some(shallow) => diagnostic.info(format_args!( - "a runtime check can only test `{shallow}`; the type arguments are assumed" + "a runtime check can only test `{}`; the type arguments are assumed", + shallow.source() )), None => diagnostic.info("the type arguments are assumed"), } diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 64aa8f0f17..597a27238b 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -36,6 +36,7 @@ use crate::types::generics::{Specialization, combine_use_site_projections}; use crate::types::instance::Protocol; use crate::types::literal::LiteralValueTypeKind; use crate::types::protocol_class::ReifiedMember; +use crate::types::runtime_spelling::RuntimeSpelling; use crate::types::tuple::Tuple; use crate::types::typevar::TypeVarKind; use crate::types::variance::TypeVarVariance; @@ -92,7 +93,7 @@ pub(crate) fn injectable_call_specialization<'db>( function: FunctionType<'db>, positional: Vec>, keywords: Vec<(&str, Type<'db>)>, -) -> Option { +) -> Option { let arguments: CallArguments<'_, 'db> = positional .into_iter() .map(|ty| (Argument::Positional, Some(ty))) @@ -113,16 +114,22 @@ pub(crate) fn injectable_call_specialization<'db>( // declaration order: the wrapper binds them by name, not by slot let (fields, positional): (Vec<&TypeArgument>, Vec<&TypeArgument>) = rendered.iter().partition(|argument| argument.keyword); - let parts: Vec<&str> = positional - .into_iter() - .chain(fields.iter().copied()) - .map(|argument| argument.text.as_str()) - .collect(); - Some(if fields.is_empty() { - format!("[{}]", parts.join(", ")) + let parts = RuntimeSpelling::join( + positional + .into_iter() + .chain(fields.iter().copied()) + .map(|argument| argument.text.clone()), + ", ", + ); + let (open, close) = if fields.is_empty() { + ("[", "]") } else { - format!(".__getitem__({})", parts.join(", ")) - }) + (".__getitem__(", ")") + }; + let mut injected = RuntimeSpelling::text(open); + injected.push(&parts); + injected.push_str(close); + Some(injected) } /// The rendered runtime spellings to inject, in declaration order, from the @@ -214,9 +221,9 @@ fn rendered_type_arguments<'db>( /// one rendered argument of an injected specialization pub(crate) struct TypeArgument { - /// the source text of this argument — `int`, the comma-joined run of a + /// the spelling of this argument — `int`, the comma-joined run of a /// `*Ts`, or the `foo=int, bar=str` fields of a `**Kwargs` pack - text: String, + text: RuntimeSpelling, /// whether the text is keyword-spelled, and so cannot go in a subscript keyword: bool, } @@ -267,15 +274,15 @@ impl ParameterKind { } } - /// the source text this parameter's value spells as, or the empty string - /// when it stands for no arguments at all + /// the spelling of this parameter's value, or the empty spelling when it + /// stands for no arguments at all fn spelling<'db>( self, db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, - ) -> Option { + ) -> Option { let fields = match self { Self::Single => return runtime_spelling(db, env, file, ty), Self::Variadic => { @@ -285,20 +292,24 @@ impl ParameterKind { runtime_spelling(db, env, file, element.promote_in(db, env, file)) }) .collect::>>()?; - return Some(spellings.join(", ")); + return Some(RuntimeSpelling::join(spellings, ", ")); } Self::KeywordPack => ty.keyword_pack_fields(db)?, }; let spellings = fields .into_iter() .map(|(name, field)| { - Some(format!( - "{name}={}", - runtime_spelling(db, env, file, field.promote_in(db, env, file))? - )) + let mut spelling = RuntimeSpelling::text(format!("{name}=")); + spelling.push(&runtime_spelling( + db, + env, + file, + field.promote_in(db, env, file), + )?); + Some(spelling) }) .collect::>>()?; - Some(spellings.join(", ")) + Some(RuntimeSpelling::join(spellings, ", ")) } } @@ -341,9 +352,9 @@ fn runtime_spelling<'db>( env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, -) -> Option { +) -> Option { if ty.is_none(db) { - return Some("None".to_owned()); + return Some(RuntimeSpelling::text("None")); } match ty { Type::NominalInstance(instance) => spell_class(db, env, file, instance.class(db, env)), @@ -351,16 +362,16 @@ fn runtime_spelling<'db>( // into the enclosing function's closure, and the `generic` wrapper fills // that cell with the type argument, so the name evaluates to the type Type::TypeVar(bound_typevar) if is_reified_function_typevar(db, bound_typevar) => { - Some(bound_typevar.name(db).to_string()) + Some(RuntimeSpelling::text(bound_typevar.name(db).as_str())) } - Type::Union(union) => Some( + Type::Union(union) => Some(RuntimeSpelling::join( union .elements(db) .iter() .map(|element| runtime_spelling(db, env, file, *element)) - .collect::>>()? - .join(" | "), - ), + .collect::>>()?, + " | ", + )), _ => None, } } @@ -370,16 +381,19 @@ fn spell_class<'db>( env: &ProgramEnvironment<'db>, file: File, class: ClassType<'db>, -) -> Option { +) -> Option { match class { ClassType::NonGeneric(literal) => spell_class_literal(db, env, file, literal), ClassType::Generic(alias) => { let origin = ClassLiteral::Static(alias.origin(db)); - let base = spell_class_literal(db, env, file, origin)?; + let mut spelling = spell_class_literal(db, env, file, origin)?; let arguments = spell_specialization_arguments(db, env, file, origin, alias.specialization(db)) .ok()?; - Some(format!("{base}[{arguments}]")) + spelling.push_str("["); + spelling.push(&arguments); + spelling.push_str("]"); + Some(spelling) } } } @@ -521,7 +535,7 @@ fn spell_specialization_arguments<'db>( file: File, origin: ClassLiteral<'db>, specialization: Specialization<'db>, -) -> Result> { +) -> Result> { // the caller may be about to *invent* an `origin[…]` the source never wrote, // so it fires only on positive evidence. a wrong yes kills the program at // import; a wrong no costs the specialization nothing else can observe @@ -539,9 +553,9 @@ fn spell_specialization_arguments<'db>( .collect::>>() .ok_or(ReifiedInferenceError::NoBinding)?; return Ok(if elements.is_empty() { - "()".to_owned() + RuntimeSpelling::text("()") } else { - elements.join(", ") + RuntimeSpelling::join(elements, ", ") }); } let Some(generic_context) = origin.generic_context(db) else { @@ -579,7 +593,7 @@ fn spell_specialization_arguments<'db>( arguments.push(spelling); } } - Ok(arguments.join(", ")) + Ok(RuntimeSpelling::join(arguments, ", ")) } /// The specialization a construction of `class_literal` builds, or the reason @@ -597,7 +611,7 @@ fn constructor_specialization<'db>( file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, -) -> Result> { +) -> Result> { let Type::NominalInstance(instance) = constructed.promote(db, env) else { return Err(ReifiedInferenceError::NoBinding); }; @@ -624,7 +638,7 @@ pub(crate) fn constructor_specialization_display<'db>( file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, -) -> Option { +) -> Option { constructor_specialization(db, env, file, class_literal, constructed).ok() } @@ -647,34 +661,29 @@ pub(crate) fn reified_construction_error<'db>( constructor_specialization(db, env, file, class_literal, constructed).err() } -/// The full runtime spelling (`list[int]`, `tuple[int, str]`) with which the -/// transpiler makes a collection literal's inferred element types explicit -/// (`[1, 2]` → `list[int]([1, 2])`). -/// -/// `None` when the literal's (literal-promoted) type is not a plain -/// specialization of the expected builtin: empty or partially-`Unknown` -/// elements, a `TypedDict`-typed dict display, a shadowed builtin name, or -/// elements without a runtime spelling. /// the class's bare name, provided that name resolves — in the module's /// globals, else builtins — to this very class, so the injected expression -/// evaluates to the intended type object +/// evaluates to the intended type object. a module-level `private` class is +/// emitted under a name of the transpiler's, so its name is kept apart for the +/// transpiler to write ([`RuntimeSpelling::global`]) fn spell_class_literal<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, literal: ClassLiteral<'db>, -) -> Option { +) -> Option { let name = literal.name(db); - let resolved = global_symbol(db, db.program_file(file), name) + if let Some(global) = global_symbol(db, db.program_file(file), name) .place .ignore_possibly_undefined() - .or_else(|| { - builtins_symbol(db, env, name) - .place - .ignore_possibly_undefined() - })?; - let resolved_literal = resolved.as_class_literal()?; - (resolved_literal == literal).then(|| name.to_string()) + { + return (global.as_class_literal()? == literal) + .then(|| RuntimeSpelling::global(db, file, name)); + } + let builtin = builtins_symbol(db, env, name) + .place + .ignore_possibly_undefined()?; + (builtin.as_class_literal()? == literal).then(|| RuntimeSpelling::text(name.as_str())) } /// How a parametric type test (`x is C[args]`, keyword form) resolves. @@ -826,7 +835,7 @@ pub enum TargetSpelling { Written, /// nothing in the source spells this target on its own — one arm of a union /// the source named with a single word — so it is rebuilt - Rebuilt(String), + Rebuilt(RuntimeSpelling), } /// basedpython: one protocol member a parametric `is`-test checks structurally @@ -840,7 +849,7 @@ pub enum ProtocolMemberCheck { name: String, /// a python expression evaluating, in the checked module's scope, to the /// member's specialized type (`int`, `list[str]`) - expected: String, + expected: RuntimeSpelling, variance: ArgVariance, }, /// a method member: each declared positional parameter (contravariant) and, @@ -850,10 +859,10 @@ pub enum ProtocolMemberCheck { name: String, /// per-parameter `(specialized type spelling, variance)`, in declaration /// order after `self` — always contravariant - params: Vec<(String, ArgVariance)>, + params: Vec<(RuntimeSpelling, ArgVariance)>, /// the return `(specialized type spelling, variance)` — covariant, or /// `None` when the method declares no meaningful return to check - ret: Option<(String, ArgVariance)>, + ret: Option<(RuntimeSpelling, ArgVariance)>, }, } @@ -908,15 +917,15 @@ pub enum ErasedTargetReason { /// is; the specialization has to travel with the call instead pub struct ErasedUnion { /// the shared origin's runtime spelling — `list` - pub origin: String, + pub origin: RuntimeSpelling, /// which type-argument position the arms differ in pub position: usize, /// the differing argument of each arm, in declaration order — `int`, `str` - pub arms: Vec, + pub arms: Vec, /// the arguments the arms agree on, by position, so the rewritten /// annotation can put them back (`dict[str, int] | dict[str, bool]` keeps /// `str` in position 0) - pub fixed: Vec<(usize, String)>, + pub fixed: Vec<(usize, RuntimeSpelling)>, } /// classify `ty` as an [`ErasedUnion`], or `None` when it is anything else. @@ -1335,9 +1344,9 @@ fn runtime_test_plan<'db>( // the value's type is carried by a reified type parameter on the // *target* side — `x is T` where `T` is reified spells `T` itself Type::TypeVar(bound_typevar) if is_reified_function_typevar(db, bound_typevar) => { - ParametricIsPlan::Isinstance(TargetSpelling::Rebuilt( - bound_typevar.name(db).to_string(), - )) + ParametricIsPlan::Isinstance(TargetSpelling::Rebuilt(RuntimeSpelling::text( + bound_typevar.name(db).as_str(), + ))) } // `Any` really does admit every value and is worth rejecting. `Unknown` @@ -1383,7 +1392,7 @@ fn literal_target_plan<'db>( value: if written { TargetSpelling::Written } else { - TargetSpelling::Rebuilt(value) + TargetSpelling::Rebuilt(RuntimeSpelling::text(value)) }, }; match literal.kind() { @@ -1405,8 +1414,10 @@ fn literal_target_plan<'db>( // comparison `Enum.__eq__` performs anyway LiteralValueTypeKind::Enum(member) => { match written_or_spelled(target_node, false, || { - spell_class_literal(db, env, file, member.enum_class(db)) - .map(|class| format!("{class}.{}", member.name(db))) + spell_class_literal(db, env, file, member.enum_class(db)).map(|mut class| { + class.push_str(&format!(".{}", member.name(db))); + class + }) }) { Some(spelling) => ParametricIsPlan::Identity(spelling), None => unspellable, @@ -1440,7 +1451,7 @@ fn literal_target_plan<'db>( fn written_or_spelled( target_node: Option<&ast::Expr>, subscript_evaluates: bool, - spell: impl FnOnce() -> Option, + spell: impl FnOnce() -> Option, ) -> Option { if let Some(spelling) = spell() { return Some(TargetSpelling::Rebuilt(spelling)); @@ -1764,7 +1775,7 @@ fn protocol_member_spelling<'db>( env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, -) -> Option { +) -> Option { if let Type::LiteralValue(literal) = ty { let value = match literal.kind() { LiteralValueTypeKind::Bool(boolean) => { @@ -1783,7 +1794,7 @@ fn protocol_member_spelling<'db>( } _ => return None, }; - return Some(format!("_by_lit({value})")); + return Some(RuntimeSpelling::text(format!("_by_lit({value})"))); } runtime_spelling(db, env, file, ty) } @@ -1795,7 +1806,7 @@ enum ReturnCheck { /// `object`) — nothing to verify Skip, /// check the value method's return annotation against this spelling - Check(String), + Check(RuntimeSpelling), /// a meaningful return with no runtime spelling — the method can't be checked Unspellable, } @@ -1940,7 +1951,7 @@ pub(crate) fn parametric_soundness_spelling<'db>( env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, -) -> Option<(String, Box<[ArgVariance]>)> { +) -> Option<(RuntimeSpelling, Box<[ArgVariance]>)> { let Type::NominalInstance(instance) = ty else { return None; }; diff --git a/crates/ty_python_semantic/src/types/runtime_spelling.rs b/crates/ty_python_semantic/src/types/runtime_spelling.rs new file mode 100644 index 0000000000..3c2a0b99b3 --- /dev/null +++ b/crates/ty_python_semantic/src/types/runtime_spelling.rs @@ -0,0 +1,117 @@ +//! basedpython: the python the checker spells for the transpiled program +//! +//! a lowering that writes a type out at runtime — a soundness check's `isinstance` +//! target, the specialization a reified generic is called with, a type test's probe — +//! asks the checker how the type is spelled, and the checker names each class by the +//! binding the module's global scope reaches it through. a module-level `private` +//! symbol is not emitted under that binding's name, though: `private class Kept` is +//! emitted as `_Kept`, or under another name where the module already has that one, and +//! only the transpiler decides which. so a spelling keeps the names of those symbols +//! apart, and whoever writes the spelling out spells them + +use ruff_db::files::File; +use ruff_python_ast::name::Name; + +use crate::Db; +use crate::types::visibility::private_symbols; + +/// a python expression the checker spells for the transpiled program, which evaluates in +/// the module it was spelled for to the runtime object of a type (`list[int]`) or to a +/// value (`Color.RED`) +/// +/// each module-level `private` symbol it reads is kept apart, by the name the source +/// declares it under, for [`Self::spelled`] to write as the transpiled module binds it +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct RuntimeSpelling { + /// never two [`SpellingPart::Text`] in a row and never an empty one, so two spellings + /// of the same expression compare equal however they were put together + parts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum SpellingPart { + Text(String), + /// a module-level `private` symbol, by the name the source declares it under + ModulePrivate(Name), +} + +impl RuntimeSpelling { + /// a spelling that reads no module-level `private` symbol + pub(crate) fn text(text: impl Into) -> Self { + let mut spelling = Self::default(); + spelling.push_str(&text.into()); + spelling + } + + /// `name` as read through `file`'s global scope, kept apart when it is one of the + /// module's `private` symbols and written as it is otherwise + /// + /// the caller has resolved `name` to a binding of `file`'s own global scope. a name + /// that reaches a builtin because the module leaves it unbound is spelled with + /// [`Self::text`] instead: the module may declare a `private` symbol of that name + /// without binding it, and the builtin is not that symbol + pub(crate) fn global(db: &dyn Db, file: File, name: &Name) -> Self { + if private_symbols(db, file).contains(name) { + Self { + parts: vec![SpellingPart::ModulePrivate(name.clone())], + } + } else { + Self::text(name.as_str()) + } + } + + pub(crate) fn push_str(&mut self, text: &str) { + if text.is_empty() { + return; + } + if let Some(SpellingPart::Text(last)) = self.parts.last_mut() { + last.push_str(text); + } else { + self.parts.push(SpellingPart::Text(text.to_owned())); + } + } + + pub(crate) fn push(&mut self, other: &Self) { + for part in &other.parts { + match part { + SpellingPart::Text(text) => self.push_str(text), + SpellingPart::ModulePrivate(_) => self.parts.push(part.clone()), + } + } + } + + /// `spellings`, each after the first preceded by `separator` + pub(crate) fn join(spellings: impl IntoIterator, separator: &str) -> Self { + let mut joined = Self::default(); + for (index, spelling) in spellings.into_iter().enumerate() { + if index > 0 { + joined.push_str(separator); + } + joined.push(&spelling); + } + joined + } + + pub(crate) fn is_empty(&self) -> bool { + self.parts.is_empty() + } + + /// the expression as the transpiled module reads it, each module-level `private` + /// symbol written under the name `module_private` gives it + pub fn spelled(&self, module_private: impl Fn(&str) -> String) -> String { + let mut spelled = String::new(); + for part in &self.parts { + match part { + SpellingPart::Text(text) => spelled.push_str(text), + SpellingPart::ModulePrivate(name) => spelled.push_str(&module_private(name)), + } + } + spelled + } + + /// the expression as the source reads it, each symbol by the name the source declares + /// it under — for a diagnostic, and for source a later transpile phase renames itself + pub fn source(&self) -> String { + self.spelled(ToOwned::to_owned) + } +} diff --git a/crates/ty_python_semantic/src/types/soundness.rs b/crates/ty_python_semantic/src/types/soundness.rs index e549f62eb1..ce0784dbdd 100644 --- a/crates/ty_python_semantic/src/types/soundness.rs +++ b/crates/ty_python_semantic/src/types/soundness.rs @@ -21,6 +21,7 @@ use crate::types::literal::LiteralValueTypeKind; use crate::types::reified_infer::{ ArgVariance, parametric_soundness_spelling, protocol_structural_members, }; +use crate::types::runtime_spelling::RuntimeSpelling; use crate::types::signatures::CallableSignature; use crate::types::visitor::any_over_type; use crate::types::{ClassLiteral, ClassType, FunctionType, KnownClass, Type}; @@ -28,34 +29,34 @@ use ty_module_resolver::{KnownModule, file_to_module}; /// The second argument of a shallow `isinstance` soundness check. /// -/// Structured rather than spelled, because the transpiler writes it as python source -/// while the native backend resolves each class and tests against it directly; its -/// [`Display`](std::fmt::Display) is the spelling. +/// structured rather than spelled, because the transpiler writes it as python source +/// while the native backend resolves each class and tests against it directly. +/// [`Self::spelling`] is how it is written out #[derive(Debug, Clone, PartialEq, Eq)] pub enum CheckTarget { /// a class, by the name module scope reaches it through: a binding of the /// module's own, or a builtin the module leaves unbound - Class(String), + Class(RuntimeSpelling), /// `type(None)` NoneType, /// any one of several, which `isinstance` accepts as a tuple AnyOf(Vec), } -impl std::fmt::Display for CheckTarget { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl CheckTarget { + /// the second argument of the `isinstance` call + pub fn spelling(&self) -> RuntimeSpelling { match self { - Self::Class(name) => f.write_str(name), - Self::NoneType => f.write_str("type(None)"), + Self::Class(name) => name.clone(), + Self::NoneType => RuntimeSpelling::text("type(None)"), Self::AnyOf(parts) => { - f.write_str("(")?; - for (index, part) in parts.iter().enumerate() { - if index > 0 { - f.write_str(", ")?; - } - write!(f, "{part}")?; - } - f.write_str(")") + let mut spelling = RuntimeSpelling::text("("); + spelling.push(&RuntimeSpelling::join( + parts.iter().map(Self::spelling), + ", ", + )); + spelling.push_str(")"); + spelling } } } @@ -70,7 +71,10 @@ pub enum CheckKind { /// carries `__orig_class__`, its type arguments. `alias` is the runtime /// spelling of the specialization (`A[int]`); `variances` is one code per /// type parameter (0 invariant, 1 covariant, 2 contravariant, 3 bivariant) - Parametric { alias: String, variances: Vec }, + Parametric { + alias: RuntimeSpelling, + variances: Vec, + }, } /// How a checked `cast` / `cast?` validates its value against the target type. @@ -297,8 +301,8 @@ pub(crate) fn runtime_check_target<'db>( env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, -) -> Option { - target(db, env, file, ty, 0).map(|target| target.to_string()) +) -> Option { + target(db, env, file, ty, 0).map(|target| target.spelling()) } /// The runtime soundness check for a value whose declared type is `ty`. @@ -476,13 +480,13 @@ fn class_target<'db>( file: File, literal: ClassLiteral<'db>, ) -> Option { - let name = literal.name(db).as_str(); + let name = literal.name(db); match explicit_global_symbol(db, db.program_file(file), name).place { Place::Defined(defined) if defined.ty == Type::ClassLiteral(literal) => { - Some(CheckTarget::Class(name.to_owned())) + Some(CheckTarget::Class(RuntimeSpelling::global(db, file, name))) } Place::Undefined if class_is_builtin(db, literal) => { - Some(CheckTarget::Class(name.to_owned())) + Some(CheckTarget::Class(RuntimeSpelling::text(name.as_str()))) } _ => None, } @@ -497,7 +501,7 @@ fn class_is_builtin<'db>(db: &'db dyn Db, literal: ClassLiteral<'db>) -> bool { /// does not rebind that name fn builtin_target(db: &dyn Db, file: File, name: &str) -> Option { match explicit_global_symbol(db, db.program_file(file), name).place { - Place::Undefined => Some(CheckTarget::Class(name.to_owned())), + Place::Undefined => Some(CheckTarget::Class(RuntimeSpelling::text(name))), Place::Defined(_) => None, } } From 743463195fe62e3c373a2bf925275071b6c423d7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:05:30 +1000 Subject: [PATCH 02/47] =?UTF-8?q?parenthesize=20an=20expression=20written?= =?UTF-8?q?=20over=20several=20lines=20wherever=20a=20lowering=20moves=20i?= =?UTF-8?q?t=20into=20a=20statement=20it=20writes=20=E2=80=94=20the=20oper?= =?UTF-8?q?and=20of=20`or=20return`=20/=20`=3F=3F=20raise`=20/=20`if=20?= =?UTF-8?q?=E2=80=A6=20else=20raise`,=20a=20re-evaluated=20default,=20an?= =?UTF-8?q?=20`if=20let`=20subject=20or=20condition,=20a=20`let`=20subject?= =?UTF-8?q?,=20a=20conjunction=20`match`=20subject=20and=20its=20case=20gu?= =?UTF-8?q?ard=20=E2=80=94=20so=20its=20second=20line=20no=20longer=20fail?= =?UTF-8?q?s=20to=20parse=20("Unexpected=20indentation")=20or=20runs=20as?= =?UTF-8?q?=20a=20statement=20of=20its=20own=20(`xs=20=3D=20[1]`=20then=20?= =?UTF-8?q?`+=20[2]`)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/transforms/destructure.rs | 30 ++-- crates/by_transforms/src/transforms/if_let.rs | 15 +- .../src/transforms/init_method.rs | 1 + .../src/transforms/mutable_defaults.rs | 14 +- .../src/transforms/source_util.rs | 22 +++ .../src/transforms/statement_expression.rs | 23 +-- .../tests/destructure_runtime.rs | 86 ++++++++-- crates/by_transforms/tests/if_let_runtime.rs | 71 ++++++-- .../tests/mutable_defaults_runtime.rs | 47 +++++ .../tests/statement_expression_runtime.rs | 160 ++++++++++++++++++ 10 files changed, 411 insertions(+), 58 deletions(-) create mode 100644 crates/by_transforms/tests/statement_expression_runtime.rs diff --git a/crates/by_transforms/src/transforms/destructure.rs b/crates/by_transforms/src/transforms/destructure.rs index 96e98d9ac2..61c6fd408d 100644 --- a/crates/by_transforms/src/transforms/destructure.rs +++ b/crates/by_transforms/src/transforms/destructure.rs @@ -67,7 +67,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, Fragment, PassContext}; use super::repeated_underscore::WrittenNames; -use super::source_util::{line_indent, line_start, temporary_name}; +use super::source_util::{line_indent, line_start, standalone_expression, temporary_name}; pub(crate) struct DestructurePass<'src> { source: &'src str, @@ -197,7 +197,7 @@ impl DestructureLower<'_> { return; } let indent = line_indent(self.source, let_stmt.range().start()).to_owned(); - let subject = vec![Fragment::Src(let_stmt.value.range())]; + let subject = standalone_expression(self.source, let_stmt.value.range()); if let_stmt.orelse.is_empty() { let mut fragments = Vec::new(); @@ -462,13 +462,17 @@ impl DestructureLower<'_> { // the subject is evaluated once, and every case guard reads it from // there. `if True:` hosts the cases at the indentation their bodies // already have, so no body moves + let mut header = vec![Fragment::Lit(format!("{subject_name} = "))]; + header.extend(standalone_expression( + self.source, + match_stmt.subject.range(), + )); + header.push(Fragment::Lit(format!( + "\n{indent}{selector} = 0\n{indent}if True:" + ))); let mut edits = vec![( TextRange::new(match_stmt.range().start(), header_colon), - vec![ - Fragment::Lit(format!("{subject_name} = ")), - Fragment::Src(match_stmt.subject.range()), - Fragment::Lit(format!("\n{indent}{selector} = 0\n{indent}if True:")), - ], + header, )]; for (index, case) in match_stmt.cases.iter().enumerate() { @@ -488,12 +492,16 @@ impl DestructureLower<'_> { let mut fragments = vec![Fragment::Lit(format!( "if {selector} == 0:\n{guard_indent}" ))]; + let guard = case + .guard + .as_deref() + .map(|guard| standalone_expression(self.source, guard.range())); if !self.push_destructure_with_guard( &mut fragments, &guard_indent, &[Fragment::Lit(subject_name.clone())], &case.pattern, - case.guard.as_deref(), + guard.as_deref(), &format!("{selector} = {}", index + 1), ) { return; @@ -541,7 +549,7 @@ impl DestructureLower<'_> { indent: &str, subject: &[Fragment], pattern: &Pattern, - guard: Option<&Expr>, + guard: Option<&[Fragment]>, on_match: &str, ) -> bool { match push_destructure( @@ -576,7 +584,7 @@ pub(crate) fn push_destructure( indent: &str, subject: &[Fragment], pattern: &Pattern, - guard: Option<&Expr>, + guard: Option<&[Fragment]>, on_match: &str, names: &mut NameGen<'_>, ) -> Result<(), String> { @@ -624,7 +632,7 @@ pub(crate) fn push_destructure( && let Some(guard) = guard { fragments.push(Fragment::Lit(" if ".to_owned())); - fragments.push(Fragment::Src(guard.range())); + fragments.extend(guard.iter().cloned()); } step_indent = format!("{step_indent} "); fragments.push(Fragment::Lit(format!(":\n{step_indent}"))); diff --git a/crates/by_transforms/src/transforms/if_let.rs b/crates/by_transforms/src/transforms/if_let.rs index 0545e4e7d8..da173834f2 100644 --- a/crates/by_transforms/src/transforms/if_let.rs +++ b/crates/by_transforms/src/transforms/if_let.rs @@ -42,7 +42,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::destructure::{NameGen, push_destructure}; use super::repeated_underscore::WrittenNames; -use super::source_util::{line_indent, temporary_name}; +use super::source_util::{line_indent, standalone_expression, temporary_name}; use crate::type_info::TypeInfo; pub(crate) struct IfLetPass<'src> { @@ -191,7 +191,7 @@ impl IfLetLower<'_, '_> { &selector, 1, clause.pattern, - test, + &standalone_expression(self.source, test.range()), &mut self.names, ) { self.errors @@ -208,7 +208,7 @@ impl IfLetLower<'_, '_> { &selector, index + 1, clause.pattern, - test, + &standalone_expression(self.source, test.range()), &mut self.names, ) { self.errors @@ -253,21 +253,22 @@ impl IfLetLower<'_, '_> { } /// Appends the guard that records clause `value` as taken: the destructuring a -/// pattern clause stands for, a plain `if` for a condition +/// pattern clause stands for, a plain `if` for a condition. `test` is the clause's +/// subject or condition, as [`standalone_expression`] writes it fn push_guard( fragments: &mut Vec, indent: &str, selector: &str, value: usize, pattern: Option<&Pattern>, - test: &Expr, + test: &[Fragment], names: &mut NameGen<'_>, ) -> Result<(), String> { match pattern { Some(pattern) => push_destructure( fragments, indent, - &[Fragment::Src(test.range())], + test, pattern, None, &format!("{selector} = {value}"), @@ -275,7 +276,7 @@ fn push_guard( ), None => { fragments.push(Fragment::Lit("if ".to_owned())); - fragments.push(Fragment::Src(test.range())); + fragments.extend(test.iter().cloned()); fragments.push(Fragment::Lit(format!( ":\n{indent} {selector} = {value}" ))); diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index b2980b357c..2fdaf5883a 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -385,6 +385,7 @@ impl State<'_> { guards, undeclarable, } = parameter_guards( + self.source, func, self.written, &super::mutable_defaults::sentinel_name(self.written), diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index db5b12080f..a9afa2c25d 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -51,7 +51,9 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::repeated_underscore::{WrittenNames, reads_underscore, rebound_underscore}; -use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; +use super::source_util::{ + PrologueStatement, body_prologue, first_body_statement, standalone_expression, +}; use crate::type_info::TypeInfo; /// what a `_MISSING`-sentinel guard does when the argument was not supplied @@ -64,7 +66,7 @@ pub(crate) enum Guard { Reevaluate { name: String, sentinel: String, - default: TextRange, + default: Vec, }, /// raise — the parameter is required, its sentinel default only exists /// because python rejects a required parameter after a defaulted one @@ -103,7 +105,7 @@ impl PrologueStatement for Guard { frags.push(Fragment::Lit(format!( "if {name} is {sentinel}:\n{base} {name} = " ))); - frags.push(Fragment::Src(*default)); + frags.extend(default.iter().cloned()); } Guard::Required { name, @@ -218,6 +220,7 @@ pub(crate) struct ParameterGuards { /// sentinels they re-evaluate — the defaults it writes are never shared between /// calls. pub(crate) fn parameter_guards( + source: &str, f: &StmtFunctionDef, written_names: WrittenNames, sentinel: &str, @@ -244,7 +247,7 @@ pub(crate) fn parameter_guards( name: crate::python_parameter_name(params, &pw.parameter, written_names) .to_string(), sentinel: sentinel.to_owned(), - default: d.range(), + default: standalone_expression(source, d.range()), }); } } @@ -280,7 +283,7 @@ pub(crate) fn parameter_guards( name: crate::python_parameter_name(params, &pw.parameter, written_names) .to_string(), sentinel: sentinel.to_owned(), - default: d.range(), + default: standalone_expression(source, d.range()), }); } Some(_) => {} @@ -377,6 +380,7 @@ impl MutableDefaults<'_> { guards, undeclarable, } = parameter_guards( + self.source, f, self.written, &sentinel_name(self.written), diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index 95baf7453a..eb58ac9a04 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -12,6 +12,28 @@ use ruff_python_ast::{ use ruff_python_trivia::SimpleTokenizer; use ruff_text_size::{Ranged, TextRange, TextSize}; +/// the expression at `range` as fragments that stand on their own where a line break ends +/// the statement: the value of an assignment a lowering writes, the test of an `if` it +/// writes, the subject of a `match` it writes +/// +/// an expression's range stops inside any parentheses the source grouped it with, and in +/// them it may run over several lines (`(\n a\n .b()\n)`). re-emitted bare, its +/// second line ends the statement early or starts one of its own: `x = a` above `.b()` is +/// refused, and `x = [1]` above `+ [2]` runs `+[2]` as a statement of its own. so an +/// expression written over more than one line gets parentheses of its own, which python +/// accepts around any expression +pub(crate) fn standalone_expression(source: &str, range: TextRange) -> Vec { + if source[range].contains(['\n', '\r']) { + vec![ + Fragment::Lit("(".to_owned()), + Fragment::Src(range), + Fragment::Lit(")".to_owned()), + ] + } else { + vec![Fragment::Src(range)] + } +} + /// Whether `expr`, written as it is, has to be parenthesized to stand as one argument /// of a call among others: a tuple or generator expression written without its own /// parentheses (`for x in a, b:`), or a `yield`, which a call does not accept bare diff --git a/crates/by_transforms/src/transforms/statement_expression.rs b/crates/by_transforms/src/transforms/statement_expression.rs index b8a8a7348e..aa4a9eaf0c 100644 --- a/crates/by_transforms/src/transforms/statement_expression.rs +++ b/crates/by_transforms/src/transforms/statement_expression.rs @@ -56,7 +56,8 @@ use std::collections::HashSet; use super::ast_driver::{AstPass, Fragment, PassContext}; use super::repeated_underscore::WrittenNames; use super::source_util::{ - line_indent, line_start, parenthesized_value_range, temporary_name, value_separator_start, + line_indent, line_start, parenthesized_value_range, standalone_expression, temporary_name, + value_separator_start, }; pub(crate) struct StatementExpressionPass<'src> { @@ -175,7 +176,7 @@ impl Lower<'_> { if let Expr::Statement(statement) = tail { self.emit_root(statement, stmt, tail, &temp, &indent, &mut fragments); } else { - Self::emit_value(tail, &temp, &indent, &mut fragments); + Self::emit_value(self.source, tail, &temp, &indent, &mut fragments); // the replaced range starts after the line's existing indentation, // so the first emitted line must not repeat it if let Some(Fragment::Lit(first)) = fragments.first_mut() @@ -293,7 +294,7 @@ impl Lower<'_> { } /// Emits the block that computes `expr` into `temp` at `indent`. - fn emit_value(expr: &Expr, temp: &str, indent: &str, out: &mut Vec) { + fn emit_value(source: &str, expr: &Expr, temp: &str, indent: &str, out: &mut Vec) { match expr { // only the diverging forms reach here: a form with a suite is always // the whole value expression and took [`Self::emit_root`] @@ -314,26 +315,26 @@ impl Lower<'_> { out.push(Fragment::Lit(format!("\n{inner}{guard}{temp}:\n"))); inner.push_str(" "); } - Self::emit_value(value, temp, &inner, out); + Self::emit_value(source, value, temp, &inner, out); } } Expr::If(if_expr) => { let inner = format!("{indent} "); out.push(Fragment::Lit(format!("{indent}if "))); - out.push(Fragment::Src(if_expr.test.range())); + out.extend(standalone_expression(source, if_expr.test.range())); out.push(Fragment::Lit(":\n".to_string())); - Self::emit_value(&if_expr.body, temp, &inner, out); + Self::emit_value(source, &if_expr.body, temp, &inner, out); out.push(Fragment::Lit(format!("\n{indent}else:\n"))); - Self::emit_value(&if_expr.orelse, temp, &inner, out); + Self::emit_value(source, &if_expr.orelse, temp, &inner, out); } Expr::BinOp(bin_op) if matches!(bin_op.op, Operator::Coalesce) => { let inner = format!("{indent} "); - Self::emit_value(&bin_op.left, temp, indent, out); + Self::emit_value(source, &bin_op.left, temp, indent, out); out.push(Fragment::Lit(format!("\n{indent}if {temp} is None:\n"))); - Self::emit_value(&bin_op.right, temp, &inner, out); + Self::emit_value(source, &bin_op.right, temp, &inner, out); } Expr::Named(named) => { - Self::emit_value(&named.value, temp, indent, out); + Self::emit_value(source, &named.value, temp, indent, out); out.push(Fragment::Lit(format!("\n{indent}"))); out.push(Fragment::Src(named.target.range())); out.push(Fragment::Lit(format!(" = {temp}"))); @@ -341,7 +342,7 @@ impl Lower<'_> { // an ordinary operand: evaluate it into the temporary _ => { out.push(Fragment::Lit(format!("{indent}{temp} = "))); - out.push(Fragment::Src(expr.range())); + out.extend(standalone_expression(source, expr.range())); } } } diff --git a/crates/by_transforms/tests/destructure_runtime.rs b/crates/by_transforms/tests/destructure_runtime.rs index 312f01ab11..a1e670cfdc 100644 --- a/crates/by_transforms/tests/destructure_runtime.rs +++ b/crates/by_transforms/tests/destructure_runtime.rs @@ -159,24 +159,57 @@ assert not [name for name in vars(Colour) if "_by_" in name], ( print("ok") "#; -#[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] -fn destructuring_runs_correctly() { - let Some(python) = python() else { - eprintln!("skipping destructuring runtime test: no `python3` interpreter found"); - return; - }; +/// subjects and guards written over several lines inside parentheses, which are not +/// part of their ranges. the lowering moves each onto a line of a statement it writes — +/// an assignment, a `match` header, a `case` guard — where it still needs a grouping +const MULTI_LINE: &str = r#" +class Point: + __match_args__ = ("x", "y") + def __init__(self, x: int, y: int): + self.x = x + self.y = y + +def make() -> Point: + return Point(1, 2) + +def summed() -> int: + let Point(x, y) := ( + make + () + ) + return x + y + +assert summed() == 3, "a `let` subject over several lines" + +def classify(p: Point, q: int) -> str: + match ( + p if q >= 0 + else Point(0, 0) + ): + case Point(x, y) and object() if ( + x == 1 + and q > 0 + ): + return "first" + case _: + return "other" + +assert classify(Point(1, 2), 1) == "first", "a guard over several lines" +assert classify(Point(1, 2), 0) == "other" +assert classify(Point(0, 2), 1) == "other" + +print("ok") +"#; + +fn run(python: &str, program: &str) { let config = Config { min_version: PythonVersion::PY313, ..Config::default() }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(python) .arg("-c") .arg(&transpiled) .output() @@ -190,3 +223,32 @@ fn destructuring_runs_correctly() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn destructuring_runs_correctly() { + let Some(python) = python() else { + eprintln!("skipping destructuring runtime test: no `python3` interpreter found"); + return; + }; + + run(&python, PROGRAM); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_subject_written_over_several_lines_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!( + "skipping `a_subject_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" + ); + return; + }; + run(&python, MULTI_LINE); +} diff --git a/crates/by_transforms/tests/if_let_runtime.rs b/crates/by_transforms/tests/if_let_runtime.rs index bd749aeaa7..9c02f09130 100644 --- a/crates/by_transforms/tests/if_let_runtime.rs +++ b/crates/by_transforms/tests/if_let_runtime.rs @@ -77,24 +77,42 @@ assert not [name for name in dict(globals()) if name.startswith("_by_if_let")], print("ok") "#; -#[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] -fn if_let_chains_run_correctly() { - let Some(python) = python() else { - eprintln!("skipping `if let` runtime test: no `python3` interpreter found"); - return; - }; +/// a subject and a condition written over several lines inside parentheses, which +/// are not part of their ranges. the lowering moves each into a statement header it +/// writes, where it still needs a grouping +const MULTI_LINE: &str = r#" +class Box: + def __init__(self, value: int | None): + self.value = value + +def unboxed(b: Box, fallback: bool) -> int: + if let int(x) := ( + b + .value + ): + return x + elif ( + fallback + and b.value is None + ): + return -1 + return 0 + +assert unboxed(Box(3), False) == 3, "a subject over several lines" +assert unboxed(Box(None), True) == -1, "a condition over several lines" +assert unboxed(Box(None), False) == 0 +print("ok") +"#; + +fn run(python: &str, program: &str) { let config = Config { min_version: PythonVersion::PY313, ..Config::default() }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(python) .arg("-c") .arg(&transpiled) .output() @@ -108,3 +126,32 @@ fn if_let_chains_run_correctly() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn if_let_chains_run_correctly() { + let Some(python) = python() else { + eprintln!("skipping `if let` runtime test: no `python3` interpreter found"); + return; + }; + + run(&python, PROGRAM); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_subject_written_over_several_lines_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!( + "skipping `a_subject_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" + ); + return; + }; + run(&python, MULTI_LINE); +} diff --git a/crates/by_transforms/tests/mutable_defaults_runtime.rs b/crates/by_transforms/tests/mutable_defaults_runtime.rs index 5fbefdc815..bbbdcde00f 100644 --- a/crates/by_transforms/tests/mutable_defaults_runtime.rs +++ b/crates/by_transforms/tests/mutable_defaults_runtime.rs @@ -112,6 +112,38 @@ print("ok") /// second line it is an ordinary character python refuses to tokenize const BOM: &str = "\u{feff}\ndef fresh(items = []):\n items.append(1)\n return items\n\nassert fresh() == [1], \"the default is re-evaluated per call\"\n\nprint(\"ok\")\n"; +/// a default written over several lines inside parentheses, which are not part of its +/// range. moved into the body it still needs a grouping, or its second line is a +/// statement of its own: `+ [2]` runs as a unary plus and the default is just `[1]` +const MULTI_LINE: &str = r#" +def joined(xs: list[int] = ( + [1] + + [2] +)) -> list[int]: + return xs + +assert joined() == [1, 2], "the whole default is re-evaluated, not its first line" + +def keyword(*, xs: list[int] = ( + [1] + + [2] +)) -> list[int]: + return xs + +assert keyword() == [1, 2], "so is a keyword-only one" + +# an `init(...)` with no body of its own writes its guards with the body it makes +class Holder: + init(self, let xs: list[int] = ( + [1] + + [2] + )) + +assert Holder().xs == [1, 2], "so is one of a bodyless `init`" + +print("ok") +"#; + fn run(python: &str, program: &str) { let config = Config { min_version: PythonVersion::PY313, @@ -157,6 +189,21 @@ fn an_extension_members_default_is_hoisted_below_the_preamble() { run(&python, EXTENSION); } +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_default_written_over_several_lines_survives_the_move_into_the_body() { + let Some(python) = python() else { + eprintln!( + "skipping `a_default_written_over_several_lines_survives_the_move_into_the_body`: no `python3` interpreter found" + ); + return; + }; + run(&python, MULTI_LINE); +} + /// both preamble paths have to leave the BOM at offset 0: the sentinel the /// transform emits, and the `from __future__ import annotations` line every /// target below 3.10 gets diff --git a/crates/by_transforms/tests/statement_expression_runtime.rs b/crates/by_transforms/tests/statement_expression_runtime.rs new file mode 100644 index 0000000000..754737a117 --- /dev/null +++ b/crates/by_transforms/tests/statement_expression_runtime.rs @@ -0,0 +1,160 @@ +//! runtime test for the early exits a statement expression lowers to +//! +//! `x = value or return`, `x = value ?? raise E()` and `x = a if c else raise E()` +//! lower to statements: each operand is evaluated into a temporary on a line of its +//! own, and the diverging branch becomes the body of an `if`. the transform's unit +//! tests assert on the lowered *text*; this test runs it. an operand the source wrote +//! over several lines, inside parentheses that are not part of its own range, has to +//! keep a grouping when it moves onto its own line — without one, its second line +//! either fails to parse or parses as a statement of its own, and the program quietly +//! computes something else +//! +//! no third-party packages are needed, so any `python3` will do; if none is found the +//! test skips rather than fails + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; +use common::python; + +/// operands written over several lines, one per early-exit form +const MULTI_LINE_OPERANDS: &str = r#" +class Units: + def __init__(self, n: int): + self.n = n + + def of_type(self, *types: int) -> Units: + return self + + def excluding(self, n: int) -> Units: + return Units(self.n - n) + + def __bool__(self) -> bool: + return self.n > 0 + +# a parenthesised method chain, one link per line +def army(units: Units, types: list[int]) -> int: + var chosen = ( + units.of_type(*types) + .excluding(1) + .excluding(1) + ) or return -1 + return chosen.n + +assert army(Units(5), [1]) == 3, "a present chain is the value" +assert army(Units(2), [1]) == -1, "an empty chain returns" + +# a continuation line that reads as a statement on its own: `+ 1` is a unary plus +def doubled(a: int) -> int: + x = ( + a + + 1 + ) or return 0 + return x * 2 + +assert doubled(1) == 4, "the whole operand is the value, not its first line" +assert doubled(-1) == 0 + +# `and return` takes the same path +def zero(a: int) -> str: + x = ( + a + - 1 + ) and return "nonzero" + return f"zero {x}" + +assert zero(1) == "zero 0" and zero(3) == "nonzero" + +# `or raise` +def size(xs: list[int]) -> int: + n = (len(xs) + + 0) or raise ValueError("empty") + return n + +assert size([1, 2]) == 2 +try: + _ = size([]) +except ValueError: + pass +else: + raise AssertionError("an empty list raises") + +# `?? raise` with an operand that is a chain of attribute reads +class Node: + def __init__(self, value: int?): + self.value = value + self.inner = self + +def value_of(node: Node) -> int: + v = ( + node + .inner + .value + ) ?? raise KeyError("absent") + return v + +assert value_of(Node(4)) == 4 +try: + _ = value_of(Node(None)) +except KeyError: + pass +else: + raise AssertionError("an absent value raises") + +# a conditional expression whose test and value both span lines +def positive(a: int) -> int: + x = ( + a + + 1 + ) if ( + a + > 0 + ) else raise ValueError(a) + return x + +assert positive(1) == 2, "the value is the whole operand" +try: + _ = positive(-1) +except ValueError: + pass +else: + raise AssertionError("a failed test raises") + +print("ok") +"#; + +fn run(python: &str, program: &str) { + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + let output = Command::new(python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn an_operand_written_over_several_lines_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!("skipping statement expression runtime test: no `python3` interpreter found"); + return; + }; + run(&python, MULTI_LINE_OPERANDS); +} From a0bdf8fb88595ee38464b0062d2d8b32e768b2f2 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:42:38 +1000 Subject: [PATCH 03/47] =?UTF-8?q?lower=20every=20`=3F=3F`=20by=20an=20edit?= =?UTF-8?q?=20of=20its=20own,=20so=20one=20inside=20an=20operand=20of=20an?= =?UTF-8?q?other=20=E2=80=94=20`(a=20=3F=3F=20{}).get(k)=20=3F=3F=20[]`,?= =?UTF-8?q?=20an=20argument,=20a=20subscript,=20an=20f-string=20=E2=80=94?= =?UTF-8?q?=20is=20no=20longer=20left=20in=20the=20output;=20parenthesize?= =?UTF-8?q?=20the=20lowered=20conditional=20where=20python=20would=20read?= =?UTF-8?q?=20it=20as=20something=20else=20(an=20operand=20of=20`and`=20/?= =?UTF-8?q?=20`or`,=20the=20test=20or=20value=20of=20a=20conditional=20exp?= =?UTF-8?q?ression,=20a=20comprehension=20iterable=20or=20condition,=20wha?= =?UTF-8?q?t=20`*`=20/=20`**`=20unpacks);=20bind=20the=20left=20operand=20?= =?UTF-8?q?as=20a=20comprehension=20variable=20instead=20of=20with=20`:=3D?= =?UTF-8?q?`=20where=20python=20refuses=20one=20(a=20comprehension=20itera?= =?UTF-8?q?ble,=20a=20class-body=20comprehension),=20so=20`[x=20for=20x=20?= =?UTF-8?q?in=20d.get(k)=20=3F=3F=20[]]`=20no=20longer=20raises=20`SyntaxE?= =?UTF-8?q?rror`=20at=20import;=20and=20bind=20a=20unary=20left=20operand?= =?UTF-8?q?=20once=20instead=20of=20evaluating=20it=20twice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/transforms/ast_driver.rs | 13 +- .../by_transforms/src/transforms/coalesce.rs | 344 +++++++++++------- crates/by_transforms/src/transforms/mod.rs | 2 + .../src/transforms/python_scopes.rs | 298 +++++++++++++++ .../src/transforms/walrus_scope.rs | 290 +++++++++++++++ .../tests/none_coalesce_runtime.rs | 294 +++++++++++++++ docs/basedpython/features/none-coalesce.md | 30 +- 7 files changed, 1136 insertions(+), 135 deletions(-) create mode 100644 crates/by_transforms/src/transforms/python_scopes.rs create mode 100644 crates/by_transforms/src/transforms/walrus_scope.rs create mode 100644 crates/by_transforms/tests/none_coalesce_runtime.rs diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 4b4368f59d..fb3b72e521 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -183,6 +183,9 @@ pub(crate) struct PassContext { /// rendered text has to be spliced into the replacement or the operation is /// re-emitted from source and reaches the runtime pub(crate) symbolic_substitutions: Vec<(TextRange, String)>, + /// where python refuses an assignment expression in the module the type-aware + /// passes walk, which every pass binding a temporary asks + pub(crate) walrus_regions: super::walrus_scope::WalrusRegions, } /// a lowering whose edits another pass may leave out of its own, because that pass writes @@ -1429,7 +1432,7 @@ pub(crate) fn run_against_source<'a>( let callable_pass = callable::CallableSyntaxPass::new(source_ref, written, config.clone()); let protocol_type_pass = protocol_type::ProtocolTypePass::new(source_ref, written, config.clone()); - let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref); + let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref, &comments); let force_unwrap_pass = force_unwrap::ForceUnwrapPass::new(source_ref); let flexible_keyword_pass = flexible_keyword::FlexibleKeywordPass; let some_ctor_pass = some_ctor::SomeCtorPass::new(); @@ -1749,6 +1752,7 @@ pub(crate) fn run_against_source<'a>( // anon-NT spans generics polyfill leaked verbatim into class headers &anon_named_tuple_pass, ]; + ctx.walrus_regions = super::walrus_scope::WalrusRegions::collect(parsed_handle.suite()); for pass in type_aware { if config.is_stub && pass.runtime_only() { continue; @@ -2826,13 +2830,6 @@ mod subsumed_lowerings { &["continue"], &["??"], ), - case( - "NoneCoalescePass", - &[Lowering::NoneChain], - "class A:\n b: int? = None\n\ndef f(a: A?) -> int:\n return a?.b ?? 1\n", - &[], - &["?.", "??"], - ), case( "ParametricIsPass", &[Lowering::VisibilityRename], diff --git a/crates/by_transforms/src/transforms/coalesce.rs b/crates/by_transforms/src/transforms/coalesce.rs index acbcd0ffdd..2ce5303108 100644 --- a/crates/by_transforms/src/transforms/coalesce.rs +++ b/crates/by_transforms/src/transforms/coalesce.rs @@ -1,151 +1,168 @@ -use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Operator, Stmt}; +use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::visitor::{Visitor, walk_expr}; +use ruff_python_ast::{AnyNodeRef, Comprehension, Expr, ExprBinOp, Operator, Stmt}; +use ruff_python_trivia::CommentRanges; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::repeated_underscore::WrittenNames; +use super::walrus_scope::WalrusRegions; use crate::type_info::TypeInfo; /// rewrites `a ?? b` to `a if a is not None else b`. /// /// emits *template* edits: operands are `Src` passthrough spans, so any -/// sibling lowering inside an operand (`?.`, `!`, `expr.N`, …) is materialized -/// inside the rewrite instead of being clobbered by first-wins overlap dedup -pub(crate) struct NoneCoalesce<'src> { +/// sibling lowering inside an operand (`?.`, `!`, `expr.N`, a `??` of its own) +/// is materialized inside the rewrite instead of being clobbered by first-wins +/// overlap dedup +pub(crate) struct NoneCoalesce<'src, 'ast> { source: &'src str, types: &'src dyn TypeInfo, + comments: &'src CommentRanges, + regions: &'src WalrusRegions, + /// the node the next `??` visited is written directly under, when python reads + /// a conditional expression there as something else, or refuses one: an operand + /// of `and` / `or`, the test or value of a conditional expression, a + /// comprehension's iterable or condition, what a `*` or `**` unpacks + tight_parent: Option>, edits: Vec<(TextRange, Vec)>, } -impl<'src> NoneCoalesce<'src> { - fn new(source: &'src str, types: &'src dyn TypeInfo) -> Self { - Self { - source, - types, - edits: Vec::new(), +impl<'ast> Visitor<'ast> for NoneCoalesce<'_, 'ast> { + fn visit_expr(&mut self, expr: &'ast Expr) { + let tight_parent = self.tight_parent.take(); + match expr { + Expr::BinOp(coalesce) if matches!(coalesce.op, Operator::Coalesce) => { + // a literal that cannot be `None` is the whole value, and the + // fallback never runs + if is_non_none_literal(&coalesce.left) { + self.edits + .push((expr.range(), vec![Fragment::Src(coalesce.left.range())])); + self.visit_expr(&coalesce.left); + return; + } + let (mut template, conditional) = self.lower(coalesce); + if let Some(parent) = tight_parent + && conditional + && parenthesized_range(expr.into(), parent, self.comments, self.source) + .is_none() + { + template.insert(0, Fragment::Lit("(".to_owned())); + template.push(Fragment::Lit(")".to_owned())); + } + self.edits.push((expr.range(), template)); + self.visit_expr(&coalesce.left); + self.visit_expr(&coalesce.right); + } + Expr::BoolOp(bool_op) => { + for value in &bool_op.values { + self.tight_parent = Some(expr.into()); + self.visit_expr(value); + } + } + Expr::If(conditional) => { + self.tight_parent = Some(expr.into()); + self.visit_expr(&conditional.test); + self.tight_parent = Some(expr.into()); + self.visit_expr(&conditional.body); + self.visit_expr(&conditional.orelse); + } + Expr::Starred(starred) => { + self.tight_parent = Some(expr.into()); + self.visit_expr(&starred.value); + } + Expr::Dict(dict) => { + for item in &dict.items { + match &item.key { + Some(key) => self.visit_expr(key), + None => self.tight_parent = Some(expr.into()), + } + self.visit_expr(&item.value); + } + } + _ => walk_expr(self, expr), } } -} -fn expand_none_chain(expr: &Expr, source: &str, types: &dyn TypeInfo) -> Option> { - let (form, guards, base) = super::none_chain::expand_chain(expr, source, types)?; - Some(super::none_chain::build_expansion( - &guards, - &form, - &super::none_chain::temp_var(WrittenNames::new(source), 0), - base, - )) -} - -impl<'ast> Visitor<'ast> for NoneCoalesce<'_> { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - walk_stmt(self, stmt); - } - - fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::BinOp(b) = expr - && matches!(b.op, Operator::Coalesce) - { - // build the whole (possibly chained) `??` expansion in one edit over - // the outer expression's range, then stop — recursing into the - // operands here would emit overlapping edits for any nested `??` - let template = self.expand_coalesce(expr); - self.edits.push((expr.range(), template)); - return; + fn visit_comprehension(&mut self, comprehension: &'ast Comprehension) { + self.visit_expr(&comprehension.target); + self.tight_parent = Some(comprehension.into()); + self.visit_expr(&comprehension.iter); + for condition in &comprehension.ifs { + self.tight_parent = Some(comprehension.into()); + self.visit_expr(condition); } - walk_expr(self, expr); } } -impl NoneCoalesce<'_> { - /// Lower a `??` expression to a conditional template. Chained `??` - /// recurses, so `a ?? b ?? c` becomes a nested `… if … is not None else …` - /// rather than stranding the inner `??` as verbatim source. The walrus - /// temp is reused safely: a nested one only lives inside a branch that is - /// no longer referenced once the enclosing walrus is taken. - fn expand_coalesce(&self, expr: &Expr) -> Vec { - let Expr::BinOp(b) = expr else { - return self.operand_value(expr); - }; - if !matches!(b.op, Operator::Coalesce) { - return self.operand_value(expr); - } - // a literal LHS statically known to be non-None short-circuits to the LHS - // (avoids a `1 is not None` constant-fold warning) - if is_non_none_literal(&b.left) { - return self.operand_value(&b.left); - } - let rhs = self.operand_value(&b.right); +impl NoneCoalesce<'_, '_> { + /// `a ?? b`, with `a` evaluated once and `b` only when `a` is `None`. each operand + /// passes through as source, so a `??` written inside one lowers in place. the + /// flag says whether the template is a conditional expression, which binds looser + /// than any operator + fn lower(&self, coalesce: &ExprBinOp) -> (Vec, bool) { + let left = Fragment::Src(coalesce.left.range()); + let right = Fragment::Src(coalesce.right.range()); // a wrapped optional (`int??`, a generic `T?`) keeps its present value // inside the runtime wrapper — the present branch unwraps with `.value` - let unwrap = if self.types.wrapped_optional(&b.left) { + let unwrap = if self.types.wrapped_optional(&coalesce.left) { ".value" } else { "" }; - let temp = super::none_chain::temp_var(WrittenNames::new(self.source), 0); - match expand_none_chain(&b.left, self.source, self.types) { - Some(expanded) => { - let mut t = vec![Fragment::Lit(format!("{temp}{unwrap} if ({temp} := "))]; - t.extend(expanded); - t.push(Fragment::Lit(") is not None else ".to_owned())); - t.extend(rhs); - t - } - None if is_trivially_pure(&b.left) => { - let mut t = vec![ - Fragment::Src(b.left.range()), - Fragment::Lit(format!("{unwrap} if ")), - Fragment::Src(b.left.range()), - Fragment::Lit(" is not None else ".to_owned()), - ]; - t.extend(rhs); - t - } - None => { - // a chained (left-associative) `??` puts another coalesce on the - // left; recurse through `operand_value` so it is lowered rather - // than copied verbatim. hoist to walrus so the LHS runs once - let mut t = vec![Fragment::Lit(format!("{temp}{unwrap} if ({temp} := "))]; - t.extend(self.operand_value(&b.left)); - t.push(Fragment::Lit(") is not None else ".to_owned())); - t.extend(rhs); - t - } - } - } - - /// Render an operand: a nested `??` is expanded, a `?.` chain is lowered, - /// otherwise it is a passthrough span (any sibling edits inside it apply - /// when the template is materialized). - fn operand_value(&self, expr: &Expr) -> Vec { - if let Expr::BinOp(b) = expr - && matches!(b.op, Operator::Coalesce) - { - return self.expand_coalesce(expr); + if is_trivially_pure(&coalesce.left) { + let template = vec![ + left.clone(), + Fragment::Lit(format!("{unwrap} if ")), + left, + Fragment::Lit(" is not None else ".to_owned()), + right, + ]; + return (template, true); } - if let Some(expanded) = expand_none_chain(expr, self.source, self.types) { - return expanded; + // every `??` binds the same temporary. each reads it back straight after + // binding it, before anything else in the expression runs, so a `??` nested + // in an operand never overwrites a value still to be read + let temp = super::none_chain::temp_var(WrittenNames::new(self.source), 0); + if self.regions.allows(coalesce.range) { + let template = vec![ + Fragment::Lit(format!("{temp}{unwrap} if ({temp} := ")), + left, + Fragment::Lit(") is not None else ".to_owned()), + right, + ]; + (template, true) + } else { + // where python refuses `:=` the temporary is a comprehension variable, + // and the fallback is chosen by `or` on whether the comprehension kept + // the value: `a` is its first iterable and `b` is outside it, so both + // are still evaluated in the scope they were written in + let template = vec![ + Fragment::Lit(format!("([{temp}{unwrap} for {temp} in (")), + left, + Fragment::Lit(format!(",) if {temp} is not None] or [")), + right, + Fragment::Lit("])[0]".to_owned()), + ]; + (template, false) } - vec![Fragment::Src(expr.range())] } } -/// Expressions whose evaluation has no side effects and whose value is -/// stable across two reads — safe to re-emit in both branches of the -/// rewrite without changing program semantics +/// expressions whose evaluation has no side effects and whose value is stable +/// across two reads, so they are safe to re-emit in both branches of the rewrite. +/// a unary operator is not one: `-x` calls `x.__neg__()`, which may do anything pub(crate) fn is_trivially_pure(expr: &Expr) -> bool { - match expr { + matches!( + expr, Expr::Name(_) - | Expr::NumberLiteral(_) - | Expr::StringLiteral(_) - | Expr::BytesLiteral(_) - | Expr::BooleanLiteral(_) - | Expr::NoneLiteral(_) - | Expr::EllipsisLiteral(_) => true, - Expr::UnaryOp(u) => is_trivially_pure(&u.operand), - _ => false, - } + | Expr::NumberLiteral(_) + | Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + ) } /// Literals whose value cannot be `None`. `??` on these is a no-op and the @@ -166,11 +183,12 @@ fn is_non_none_literal(expr: &Expr) -> bool { pub(crate) struct NoneCoalescePass<'src> { source: &'src str, + comments: &'src CommentRanges, } impl<'src> NoneCoalescePass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, comments: &'src CommentRanges) -> Self { + Self { source, comments } } } @@ -179,13 +197,15 @@ impl TypeAwarePass for NoneCoalescePass<'_> { Some(super::ast_driver::Lowering::NoneCoalesce) } - /// an optional chain on the left of a `??` is written into the coalesce's own test - fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { - &[super::ast_driver::Lowering::NoneChain] - } - fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = NoneCoalesce::new(self.source, types); + let mut inner = NoneCoalesce { + source: self.source, + types, + comments: self.comments, + regions: &ctx.walrus_regions, + tight_parent: None, + edits: Vec::new(), + }; for stmt in stmts { inner.visit_stmt(stmt); } @@ -245,7 +265,7 @@ mod tests { "}, indoc::indoc! {" def f(a): - __by_t_0__ if (__by_t_0__ := None if a is None else a.a.b) is not None else 1 + __by_t_0__ if (__by_t_0__ := (None if a is None else a.a.b)) is not None else 1 "}, ); } @@ -261,7 +281,7 @@ mod tests { "}, indoc::indoc! {" def f(a, b): - a if a is not None else None if b is None else b.c + a if a is not None else (None if b is None else b.c) "}, ); } @@ -365,6 +385,82 @@ mod tests { ); } + /// a `??` inside an operand of another one, short of being the operand itself, is + /// lowered by its own edit, inside the source the outer one passes through + #[test] + fn coalesce_inside_an_operand() { + check( + "x = (a ?? {}).get(k) ?? []\n", + "x = __by_t_0__ if (__by_t_0__ := (a if a is not None else {}).get(k)) is not None else []\n", + ); + check( + "x = a ?? f(b ?? 1)\n", + "x = a if a is not None else f(b if b is not None else 1)\n", + ); + check( + "x = f\"{a ?? 1}\"\n", + "x = f\"{a if a is not None else 1}\"\n", + ); + } + + /// a conditional expression binds looser than every operator, so where the `??` + /// is an operand of `and` / `or`, the test or value of a conditional expression, a + /// comprehension's iterable or condition, or what a `*` / `**` unpacks, it is + /// parenthesized — unless the source already did + #[test] + fn coalesce_keeps_its_grouping() { + check( + "x = a ?? b or c\n", + "x = (a if a is not None else b) or c\n", + ); + check( + "x = a ?? b if c else d\n", + "x = (a if a is not None else b) if c else d\n", + ); + check( + "x = c if a ?? b else d\n", + "x = c if (a if a is not None else b) else d\n", + ); + check( + "x = [i for i in xs if a ?? b]\n", + "x = [i for i in xs if (a if a is not None else b)]\n", + ); + check( + "x = (a ?? b) or c\n", + "x = (a if a is not None else b) or c\n", + ); + check("x = [*a ?? b]\n", "x = [*(a if a is not None else b)]\n"); + check("x = {**a ?? b}\n", "x = {**(a if a is not None else b)}\n"); + } + + /// python refuses `:=` in a comprehension's iterable, so the value is bound as a + /// comprehension variable there instead + #[test] + fn coalesce_in_a_comprehension_iterable() { + check( + "x = [i for i in f() ?? []]\n", + "x = [i for i in ([__by_t_0__ for __by_t_0__ in (f(),) if __by_t_0__ is not None] or [[]])[0]]\n", + ); + } + + /// and in every part of a comprehension in a class body + #[test] + fn coalesce_in_a_class_body_comprehension() { + check( + "class C:\n x = [f(i) ?? 0 for i in xs]\n y = f(0) ?? 0\n", + "class C:\n x = [([__by_t_0__ for __by_t_0__ in (f(i),) if __by_t_0__ is not None] or [0])[0] for i in xs]\n y = __by_t_0__ if (__by_t_0__ := f(0)) is not None else 0\n", + ); + } + + /// a unary operator calls a method of its operand, so its result is bound once + #[test] + fn unary_lhs_hoisted_to_walrus_single_eval() { + check( + "x = -a ?? 1\n", + "x = __by_t_0__ if (__by_t_0__ := -a) is not None else 1\n", + ); + } + #[test] fn python_unchanged() { let src = "x = a if a is not None else b\n"; diff --git a/crates/by_transforms/src/transforms/mod.rs b/crates/by_transforms/src/transforms/mod.rs index beaaabc8f6..b0b8323674 100644 --- a/crates/by_transforms/src/transforms/mod.rs +++ b/crates/by_transforms/src/transforms/mod.rs @@ -63,6 +63,7 @@ pub(crate) mod postfix_await; pub(crate) mod propagate; pub(crate) mod properties; pub(crate) mod protocol_type; +pub(crate) mod python_scopes; pub(crate) mod raises_clause; pub(crate) mod reified_class; pub(crate) mod reified_generic; @@ -96,3 +97,4 @@ pub(crate) mod unique_loop_bindings; pub(crate) mod unpack; pub(crate) mod use_site_variance; pub(crate) mod visibility_rename; +pub(crate) mod walrus_scope; diff --git a/crates/by_transforms/src/transforms/python_scopes.rs b/crates/by_transforms/src/transforms/python_scopes.rs new file mode 100644 index 0000000000..f60147477a --- /dev/null +++ b/crates/by_transforms/src/transforms/python_scopes.rs @@ -0,0 +1,298 @@ +//! the scope python evaluates each part of a module in, as a walk over its syntax tree +//! +//! a definition's body is a scope of its own, and the rest of the definition is not: +//! a decorator, a parameter's default and a base class are evaluated where the +//! definition is written. a comprehension's first iterable is evaluated where the +//! comprehension is written, and everything else in it in its own scope. an annotation +//! is evaluated in a scope of its own from 3.14, and the type parameters of a generic +//! definition, with everything that can name them — a generic class's bases, a generic +//! function's annotations, a `type` alias's value — in one more +//! +//! both the lowerings that bind a temporary and the check that the emitted module +//! compiles need these rules, so they are stated once here + +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{AnyParameterRef, Comprehension, Expr, Parameters, Stmt}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Scope { + Class, + Function { + is_async: bool, + }, + Lambda, + /// a list, set or dict comprehension, or a generator expression + Comprehension { + generator: bool, + is_async: bool, + }, + /// an annotation + Annotation, + /// the type parameters of a generic definition + TypeParams, +} + +/// where the walk is: the scopes around the node, innermost last (the module is the +/// empty stack), and the constructs around it that python's rules ask about +#[derive(Default)] +pub(crate) struct ScopeState { + scopes: Vec, + /// how many comprehension iterables the node is in + iterables: usize, + /// whether the node is in a loop's body, short of a scope opened inside it + in_loop: bool, + /// whether the node is in a `try` statement + in_try: bool, +} + +impl ScopeState { + pub(crate) fn scopes(&self) -> &[Scope] { + &self.scopes + } + + pub(crate) fn in_iterable(&self) -> bool { + self.iterables > 0 + } + + pub(crate) fn in_loop(&self) -> bool { + self.in_loop + } + + pub(crate) fn in_try(&self) -> bool { + self.in_try + } +} + +/// what a walk does at each statement and expression, before it walks into it +pub(crate) trait ScopeHooks<'a> { + fn stmt(&mut self, _stmt: &'a Stmt, _state: &ScopeState) {} + fn expr(&mut self, _expr: &'a Expr, _state: &ScopeState) {} +} + +/// walks `suite`, calling `hooks` at every statement and expression with the scope +/// state it is evaluated in +pub(crate) fn walk<'a>(suite: &'a [Stmt], hooks: &mut impl ScopeHooks<'a>) { + let mut walker = Walker { + state: ScopeState::default(), + hooks, + }; + for stmt in suite { + walker.visit_stmt(stmt); + } +} + +struct Walker<'h, H> { + state: ScopeState, + hooks: &'h mut H, +} + +impl<'a, H: ScopeHooks<'a>> Walker<'_, H> { + fn in_scope(&mut self, scope: Scope, visit: impl FnOnce(&mut Self)) { + self.state.scopes.push(scope); + visit(self); + self.state.scopes.pop(); + } + + /// a body is a scope of its own, and a loop around the definition does not + /// reach into it + fn in_body(&mut self, scope: Scope, visit: impl FnOnce(&mut Self)) { + let in_loop = std::mem::replace(&mut self.state.in_loop, false); + self.in_scope(scope, visit); + self.state.in_loop = in_loop; + } + + fn in_loop_body(&mut self, visit: impl FnOnce(&mut Self)) { + let in_loop = std::mem::replace(&mut self.state.in_loop, true); + visit(self); + self.state.in_loop = in_loop; + } + + fn visit_iterable(&mut self, iterable: &'a Expr) { + self.state.iterables += 1; + self.visit_expr(iterable); + self.state.iterables -= 1; + } + + fn visit_defaults(&mut self, parameters: &'a Parameters) { + for default in parameters + .iter_non_variadic_params() + .filter_map(|parameter| parameter.default.as_deref()) + { + self.visit_expr(default); + } + } + + fn visit_parameter_annotations(&mut self, parameters: &'a Parameters) { + for parameter in parameters.iter().map(AnyParameterRef::as_parameter) { + self.visit_parameter(parameter); + } + } + + fn visit_comprehension_scope( + &mut self, + generators: &'a [Comprehension], + elements: &[&'a Expr], + generator: bool, + ) { + let Some((first, rest)) = generators.split_first() else { + return; + }; + self.visit_iterable(&first.iter); + let is_async = generators.iter().any(|generator| generator.is_async); + self.in_scope( + Scope::Comprehension { + generator, + is_async, + }, + |walker| { + walker.visit_expr(&first.target); + for condition in &first.ifs { + walker.visit_expr(condition); + } + for generator in rest { + walker.visit_expr(&generator.target); + walker.visit_iterable(&generator.iter); + for condition in &generator.ifs { + walker.visit_expr(condition); + } + } + for element in elements { + walker.visit_expr(element); + } + }, + ); + } +} + +impl<'a, H: ScopeHooks<'a>> Visitor<'a> for Walker<'_, H> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + self.hooks.stmt(stmt, &self.state); + match stmt { + Stmt::ClassDef(class) => { + for decorator in &class.decorator_list { + self.visit_decorator(decorator); + } + match &class.type_params { + Some(type_params) => self.in_scope(Scope::TypeParams, |walker| { + walker.visit_type_params(type_params); + if let Some(arguments) = &class.arguments { + walker.visit_arguments(arguments); + } + }), + None => { + if let Some(arguments) = &class.arguments { + self.visit_arguments(arguments); + } + } + } + self.in_body(Scope::Class, |walker| walker.visit_body(&class.body)); + } + Stmt::FunctionDef(function) => { + for decorator in &function.decorator_list { + self.visit_decorator(decorator); + } + self.visit_defaults(&function.parameters); + let annotations = |walker: &mut Self| { + walker.visit_parameter_annotations(&function.parameters); + if let Some(returns) = &function.returns { + walker.visit_annotation(returns); + } + }; + match &function.type_params { + Some(type_params) => self.in_scope(Scope::TypeParams, |walker| { + walker.visit_type_params(type_params); + annotations(walker); + }), + None => annotations(self), + } + let scope = Scope::Function { + is_async: function.is_async, + }; + self.in_body(scope, |walker| walker.visit_body(&function.body)); + } + Stmt::TypeAlias(alias) => { + self.visit_expr(&alias.name); + self.in_scope(Scope::TypeParams, |walker| { + if let Some(type_params) = &alias.type_params { + walker.visit_type_params(type_params); + } + walker.visit_expr(&alias.value); + for case in &alias.cases { + walker.visit_match_case(case); + } + }); + } + Stmt::For(for_stmt) => { + self.visit_expr(&for_stmt.iter); + self.visit_expr(&for_stmt.target); + if let Some(pattern) = &for_stmt.pattern { + self.visit_pattern(pattern); + } + self.in_loop_body(|walker| walker.visit_body(&for_stmt.body)); + self.visit_body(&for_stmt.orelse); + } + Stmt::While(while_stmt) => { + self.visit_expr(&while_stmt.test); + self.in_loop_body(|walker| walker.visit_body(&while_stmt.body)); + self.visit_body(&while_stmt.orelse); + } + Stmt::Try(_) => { + let in_try = std::mem::replace(&mut self.state.in_try, true); + walk_stmt(self, stmt); + self.state.in_try = in_try; + } + _ => walk_stmt(self, stmt), + } + } + + fn visit_annotation(&mut self, expr: &'a Expr) { + self.in_scope(Scope::Annotation, |walker| walker.visit_expr(expr)); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + self.hooks.expr(expr, &self.state); + match expr { + Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + self.visit_defaults(parameters); + self.visit_parameter_annotations(parameters); + } + if let Some(returns) = &lambda.returns { + self.visit_annotation(returns); + } + self.in_body(Scope::Lambda, |walker| walker.visit_expr(&lambda.body)); + } + Expr::ListComp(comprehension) => { + self.visit_comprehension_scope( + &comprehension.generators, + &[&comprehension.elt], + false, + ); + } + Expr::SetComp(comprehension) => { + self.visit_comprehension_scope( + &comprehension.generators, + &[&comprehension.elt], + false, + ); + } + Expr::Generator(comprehension) => { + self.visit_comprehension_scope( + &comprehension.generators, + &[&comprehension.elt], + true, + ); + } + Expr::DictComp(comprehension) => { + let elements: Vec<&Expr> = comprehension + .key + .as_deref() + .into_iter() + .chain([&*comprehension.value]) + .collect(); + self.visit_comprehension_scope(&comprehension.generators, &elements, false); + } + _ => walk_expr(self, expr), + } + } +} diff --git a/crates/by_transforms/src/transforms/walrus_scope.rs b/crates/by_transforms/src/transforms/walrus_scope.rs new file mode 100644 index 0000000000..a4579af30e --- /dev/null +++ b/crates/by_transforms/src/transforms/walrus_scope.rs @@ -0,0 +1,290 @@ +//! where a lowering may bind a temporary with `:=` +//! +//! a lowering that reads a value twice but may evaluate it only once — `a ?? b`, +//! `f()?.x` — binds it to a temporary with an assignment expression. python refuses +//! one: +//! +//! - anywhere inside a comprehension's iterable, at any depth, a lambda's body +//! included: `[x for x in (t := xs)]` +//! - directly in an annotation, from 3.14: `x: (t := int) = 1` +//! - directly in the scope of a generic definition's type parameters: a bound or a +//! default, a generic class's base, a generic function's annotation, a `type` +//! alias's value: `class C[T](base(t := 1)): ...` +//! - in a comprehension whose nearest enclosing scope, past comprehensions and +//! annotations, is a class body or type parameters' scope: +//! `class C: ys = [(t := x) for x in xs]` +//! +//! a lambda's body is a function scope, where none of this applies short of a +//! comprehension iterable. where python refuses `:=` the lowering writes its temporary +//! as a comprehension variable instead, which python accepts in any expression + +use std::collections::HashSet; + +use ruff_python_ast::{Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange}; + +use crate::transforms::python_scopes::{self, Scope, ScopeHooks, ScopeState}; + +/// the expressions of a module where python accepts an assignment expression +/// +/// what is recorded is where one is accepted, so a position the walk never reached +/// — or regions nobody collected — answers that it is refused, and a lowering falls +/// back to the form python accepts everywhere +#[derive(Default)] +pub(crate) struct WalrusRegions { + allowed: HashSet, + /// a range two expressions share is refused if either refuses it + refused: HashSet, +} + +impl WalrusRegions { + pub(crate) fn collect(stmts: &[Stmt]) -> Self { + let mut regions = WalrusRegions::default(); + python_scopes::walk(stmts, &mut regions); + regions + } + + /// whether an assignment expression written in place of the expression at `range` + /// is accepted + pub(crate) fn allows(&self, range: TextRange) -> bool { + self.allowed.contains(&range) && !self.refused.contains(&range) + } +} + +impl ScopeHooks<'_> for WalrusRegions { + fn expr(&mut self, expr: &Expr, state: &ScopeState) { + if walrus_allowed(state) { + self.allowed.insert(expr.range()); + } else { + self.refused.insert(expr.range()); + } + } +} + +fn walrus_allowed(state: &ScopeState) -> bool { + if state.in_iterable() { + return false; + } + match state.scopes().last() { + Some(Scope::Annotation | Scope::TypeParams) => false, + Some(Scope::Comprehension { .. }) => !matches!( + state + .scopes() + .iter() + .rev() + .find(|scope| !matches!(scope, Scope::Comprehension { .. } | Scope::Annotation)), + Some(Scope::Class | Scope::TypeParams) + ), + Some(Scope::Class | Scope::Function { .. } | Scope::Lambda) | None => true, + } +} + +#[cfg(test)] +mod tests { + use ruff_python_ast::visitor::{Visitor, walk_expr}; + use ruff_python_ast::{Expr, PySourceType}; + use ruff_python_parser::parse_unchecked_source; + use ruff_text_size::{Ranged, TextRange}; + + use super::WalrusRegions; + + /// each source with a name `hole`, and whether python 3.12, 3.13 and 3.14 all + /// compile `(t := f())` written in its place — on every version that compiles the + /// source at all. measured by compiling each one, with the walrus and with the + /// comprehension form the lowering writes instead, which compiles in all of them + const POSITIONS: &[(&str, bool)] = &[ + ("x = hole", true), + ("x = [hole for a in b]", true), + ("x = [a for a in b if hole]", true), + ("x = [a for a in hole]", false), + ("x = [a for c in d for a in hole]", false), + ("x = [a for a in (lambda: hole)()]", false), + ("x = [a for a in [hole for c in d]]", false), + ("x = {hole: 1 for a in b}", true), + ("x = {a: hole for a in b}", true), + ("x = {hole for a in b}", true), + ("x = (hole for a in b)", true), + ("x = (a for a in hole)", false), + ("x = lambda: hole", true), + ("x = lambda y=hole: y", true), + ("x = [(lambda: hole) for a in b]", true), + ("x = [(lambda y=hole: y) for a in b]", true), + ("def f():\n return [hole for a in b]", true), + ("def f():\n return [a for a in hole]", false), + ("def f(y=hole):\n pass", true), + ("def f(y: hole):\n pass", false), + ("def f() -> hole:\n pass", false), + ("def f(*y: hole):\n pass", false), + ("@dec(hole)\ndef f():\n pass", true), + ("class C:\n x = hole", true), + ("class C:\n x = [hole for a in b]", false), + ("class C:\n x = [a for a in b if hole]", false), + ("class C:\n x = [a for a in hole]", false), + ("class C:\n x = [[hole for a in b] for c in d]", false), + ("class C:\n x = [(lambda: hole) for a in b]", true), + ("class C:\n x = [(lambda y=hole: y) for a in b]", false), + ("class C:\n x = lambda: hole", true), + ("class C:\n def f(self, y=hole):\n pass", true), + ( + "class C:\n @dec(hole)\n def f(self):\n pass", + true, + ), + ("class C:\n x: hole = 1", false), + ("class C:\n x: int = hole", true), + ("class C(hole):\n pass", true), + ("class C(metaclass=hole):\n pass", true), + ("@dec(hole)\nclass C:\n pass", true), + ("class C[T](hole):\n pass", false), + ("class C[T](metaclass=hole):\n pass", false), + ("class C[T](b[[hole for a in c]]):\n pass", false), + ("class C[T](b[lambda: hole]):\n pass", true), + ("class C[T: hole]:\n pass", false), + ("class C[T = hole]:\n pass", false), + ("@dec(hole)\nclass C[T]:\n pass", true), + ("class C[T]:\n x = hole", true), + ("def f[T](y=hole):\n pass", true), + ("def f[T](y: hole):\n pass", false), + ("def f[T]() -> hole:\n pass", false), + ("def f[T: hole]():\n pass", false), + ("@dec(hole)\ndef f[T]():\n pass", true), + ("def f[T]():\n return hole", true), + ("type A = hole", false), + ("type A = b[[hole for a in c]]", false), + ("type A = b[lambda: hole]", true), + ("type A[T] = hole", false), + ("type A[T: hole] = int", false), + ("x: hole = 1", false), + ("x: int = hole", true), + ("x: b[[hole for a in c]] = 1", true), + ("x: b[lambda: hole] = 1", true), + ("x: b[[a for a in hole]] = 1", false), + ("def f():\n x: hole = 1", false), + ("def f():\n x: b[[hole for a in c]] = 1", true), + ("def f(y: b[lambda: hole]):\n pass", true), + ("for a in hole:\n pass", true), + ("with hole as a:\n pass", true), + ("x = f\"{hole}\"", true), + ("match hole:\n case _:\n pass", true), + ("match a:\n case _ if hole:\n pass", true), + ("assert hole", true), + ("x = [a for a in b if (lambda: hole)()]", true), + ("x = [[a for a in hole] for c in d]", false), + ("async def f():\n return [a for a in await hole]", false), + ("def f[T](y: b[[hole for a in c]]):\n pass", false), + ("def f[T](y: b[lambda: hole]):\n pass", true), + ("def f(y: b[[hole for a in c]]):\n pass", true), + ("class C[T]:\n x: hole = 1", false), + ("class C[T]:\n x: b[[hole for a in c]] = 1", false), + ("class C:\n x: b[[hole for a in c]] = 1", false), + ("class C:\n x = {a: hole for a in b}", false), + ("class C:\n x = (hole for a in b)", false), + ("class C:\n x = [a for a in b if (lambda: hole)()]", true), + ( + "class C:\n def f(self):\n return [hole for a in b]", + true, + ), + ( + "class C:\n class D:\n x = [hole for a in b]", + false, + ), + ( + "def f():\n class D:\n x = [hole for a in b]", + false, + ), + ( + "class C[T](b[(lambda: [hole for a in c])()]):\n pass", + true, + ), + ( + "class C[T](b[[(lambda: hole) for a in c]]):\n pass", + true, + ), + ("type A = b[[(lambda: hole) for a in c]]", true), + ("x: b[[a for a in c if hole]] = 1", true), + ("x: (lambda y=hole: y) = 1", false), + ("type A = (lambda y=hole: y)", false), + ("class C[T](b[lambda y=hole: y]):\n pass", false), + ("x = [lambda: [hole for a in b] for c in d]", true), + ( + "class C:\n x = [lambda: [hole for a in b] for c in d]", + true, + ), + ("def f(y=[hole for a in b]):\n pass", true), + ( + "class C:\n def f(self, y=[hole for a in b]):\n pass", + false, + ), + ("@dec([hole for a in b])\ndef f():\n pass", true), + ( + "class C:\n @dec([hole for a in b])\n def f(self):\n pass", + false, + ), + ( + "class C:\n def f(self, y: b[[hole for a in c]]):\n pass", + false, + ), + ( + "class C:\n def f(self) -> b[[hole for a in c]]:\n pass", + false, + ), + ( + "class C:\n def f(self, y: b[lambda: hole]):\n pass", + true, + ), + ("def f():\n x: b[[hole for a in c]] = 1", true), + ("class C:\n type A = b[lambda: hole]", true), + ("class C[T]:\n def f(self, y=hole):\n pass", true), + ("class C[T]:\n x = [hole for a in b]", false), + ("x = lambda: [hole for a in b]", true), + ("class C:\n x = lambda: [hole for a in b]", true), + ]; + + fn hole(source: &str) -> TextRange { + struct Hole(Option); + impl<'a> Visitor<'a> for Hole { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.id.as_str() == "hole" + { + self.0 = Some(name.range()); + } + walk_expr(self, expr); + } + } + let parsed = parse_unchecked_source(source, PySourceType::Python); + assert!( + parsed.errors().is_empty(), + "{source}: {:?}", + parsed.errors() + ); + let mut hole = Hole(None); + hole.visit_body(&parsed.syntax().body); + hole.0.unwrap_or_else(|| panic!("no `hole` in {source}")) + } + + /// a position nobody recorded is refused: regions left empty, or a range the + /// walk never reached + #[test] + fn an_unknown_position_is_refused() { + let source = "x = hole\n"; + assert!(!WalrusRegions::default().allows(hole(source))); + let parsed = parse_unchecked_source(source, PySourceType::Python); + let regions = WalrusRegions::collect(&parsed.syntax().body); + assert!(regions.allows(hole(source))); + // `x =` is no expression + assert!(!regions.allows(TextRange::new(0.into(), 3.into()))); + } + + #[test] + fn agrees_with_python_on_every_position() { + let mismatches: Vec = POSITIONS + .iter() + .filter_map(|&(source, python)| { + let parsed = parse_unchecked_source(source, PySourceType::Python); + let ours = WalrusRegions::collect(&parsed.syntax().body).allows(hole(source)); + (ours != python).then(|| format!("python {python}, ours {ours}: {source:?}")) + }) + .collect(); + assert!(mismatches.is_empty(), "{}", mismatches.join("\n")); + } +} diff --git a/crates/by_transforms/tests/none_coalesce_runtime.rs b/crates/by_transforms/tests/none_coalesce_runtime.rs new file mode 100644 index 0000000000..62ac371c01 --- /dev/null +++ b/crates/by_transforms/tests/none_coalesce_runtime.rs @@ -0,0 +1,294 @@ +//! runtime test for the `??` (none-coalesce) lowering +//! +//! `a ?? b` lowers to a conditional expression that evaluates `a` once, binding it to +//! a temporary when it is more than a name, and evaluates `b` only when `a` is `None`. +//! the transform's unit tests assert on the lowered *text*; this test runs it, with +//! the `??` written where the lowering has to compose with what surrounds it: +//! +//! - inside an operand of another `??` — a receiver, an argument, a subscript, an +//! f-string — where it is lowered by an edit of its own +//! - as an operand of `and` / `or`, or the test or value of a conditional +//! expression, where python reads an unparenthesized conditional as something else +//! - where python refuses `:=`: a comprehension's iterable, and any part of a +//! comprehension in a class body +//! +//! no third-party packages are needed, so any `python3` will do; if none is found the +//! test skips rather than fails + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; +use common::python; + +/// a `??` inside an operand of another one +const NESTED: &str = r#" +calls: list[str] = [] + +def probe(tag: str, value: int?) -> int?: + calls.append(tag) + return value + +def lookup(a: dict[int, list[int]]?, k: int) -> list[int]: + return (a ?? {}).get(k) ?? [] + +assert lookup(None, 1) == [] and lookup({1: [2]}, 1) == [2] and lookup({}, 1) == [] + +def right(a: int?, b: int?) -> int: + return a ?? int(str(probe("b", b) ?? 4)) + +assert right(None, None) == 4 and right(1, None) == 1 + +def indexed(xs: list[int]?, i: int?) -> int: + return (xs ?? [9])[i ?? 0] + +assert indexed(None, None) == 9 and indexed([1, 2], 1) == 2 + +def add(a: int, b: int) -> int: + return a + b + +def argument(x: dict[int, int]?, y: int?) -> int: + return add((x ?? {}).get(1) ?? 0, b=y ?? 0) + +assert argument(None, None) == 0 and argument({1: 5}, 2) == 7 + +def formatted(x: int?, d: dict[int, int]?) -> str: + return f"{x ?? 1}-{(d ?? {}).get(1) ?? 2}" + +assert formatted(None, None) == "1-2" + +# a chain evaluates each left operand once, in order, and stops at the first that +# is not `None` +def chained(a: int?, b: int?, c: int?) -> int: + return probe("a", a) ?? probe("b", b) ?? probe("c", c) ?? 0 + +calls.clear() +assert chained(None, 2, 3) == 2 and calls == ["a", "b"] +calls.clear() +assert chained(None, None, None) == 0 and calls == ["a", "b", "c"] + +# a unary operator is a call of its operand's method +class Negated: + def __init__(self): + self.calls = 0 + + def __neg__(self) -> int?: + self.calls += 1 + return 5 + +negated = Negated() +assert (-negated ?? 3) == 5 and negated.calls == 1, "the left operand is evaluated once" + +print("ok") +"#; + +/// a `??` where python reads an unparenthesized conditional expression as something +/// else +const GROUPING: &str = r#" +def body(a: int?, c: bool) -> int: + return a ?? 1 if c else 2 + +assert body(None, True) == 1 +assert body(5, False) == 2, "the condition picks between the `??` and 2" + +def operand(a: int?, b: int) -> int: + return a ?? 0 or b + +assert operand(5, 9) == 5 +assert operand(0, 9) == 9, "`or` applies to the coalesced value" + +def test(a: bool?) -> int: + return 1 if a ?? True else 2 + +assert test(None) == 1 and test(False) == 2 + +def kept(d: dict[int, bool], ks: list[int]) -> list[int]: + return [k for k in ks if d.get(k) ?? True] + +assert kept({1: False}, [1, 2]) == [2] + +def unpacked(a: list[int]?, b: list[int], d: dict[str, int]?) -> tuple[list[int], dict[str, int]]: + return [*a ?? b], {**d ?? {"k": 1}} + +assert unpacked(None, [2], None) == ([2], {"k": 1}) + +print("ok") +"#; + +/// a `??` where python refuses `:=`, and next to it the places it accepts one +const BINDING: &str = r#" +from collections.abc import Callable + +table: dict[int, list[int]] = {1: [2]} +flags: dict[int, bool] = {1: False} +values: dict[int, int?] = {1: 5} + +def iterable(k: int) -> list[int]: + return [x for x in table.get(k) ?? []] + +assert iterable(1) == [2] and iterable(3) == [] + +def nested_iterable(ks: list[int]) -> list[int]: + return [x for k in ks for x in table.get(k) ?? [0]] + +assert nested_iterable([1, 3]) == [2, 0] + +def generator(k: int) -> int: + return sum(x for x in table.get(k) ?? []) + +assert generator(1) == 2 and generator(3) == 0 + +def displays(k: int) -> tuple[set[int], dict[int, int]]: + return {x for x in table.get(k) ?? []}, {x: x for x in table.get(k) ?? [7]} + +assert displays(3) == (set(), {7: 7}) + +def lambda_in_iterable() -> list[int]: + return [x for x in (lambda: table.get(3) ?? [1])()] + +assert lambda_in_iterable() == [1] + +def parts(ks: list[int]) -> list[int]: + return [values.get(k) ?? -1 for k in ks if flags.get(k) ?? True] + +assert parts([1, 2]) == [-1] + +def counted(ks: list[int]) -> int: + return sum(values.get(k) ?? 0 for k in ks) + +assert counted([1, 2]) == 5 + +class Holder: + let direct: int = values.get(2) ?? 3 + let element: list[int] = [values.get(k) ?? 0 for k in [1, 2]] + let condition: list[int] = [k for k in [1, 2] if flags.get(k) ?? True] + let from_iterable: list[int] = [x for x in table.get(9) ?? [direct]] + let in_lambda: list[int] = [(lambda: values.get(k) ?? 0)() for k in [1, 2]] + +assert Holder.direct == 3 +assert Holder.element == [5, 0] +assert Holder.condition == [2] +assert Holder.from_iterable == [3], "the fallback still sees the class body's names" +assert Holder.in_lambda == [5, 0] + +def scaled(n: int) -> Callable[[Callable[[], int]], Callable[[], int]]: + def wrap(fn: Callable[[], int]) -> Callable[[], int]: + return lambda: fn() * n + return wrap + +@scaled(values.get(2) ?? 10) +def decorated(x: int = values.get(2) ?? 1) -> int: + return x + +assert decorated() == 10 + +print("ok") +"#; + +/// a `??` in an annotation and in the scope of a generic definition's type parameters, +/// where python refuses `:=` too +const ANNOTATION_SCOPES: &str = r#" +from typing import Annotated + +table: dict[str, int] = {} + +class Base: + pass + +def base(n: int) -> type[Base]: + return Base + +class Generic[T](base(table.get("a") ?? 0)): + pass + +annotated: Annotated[int, table.get("b") ?? 1] = 1 + +def parameter(y: Annotated[int, table.get("c") ?? 2]): + pass + +type Alias = Annotated[int, table.get("d") ?? 3] + +assert issubclass(Generic, Base) and annotated == 1 + +print("ok") +"#; + +fn run(python: &str, program: &str) { + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + let output = Command::new(python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_coalesce_inside_an_operand_is_lowered() { + let Some(python) = python() else { + eprintln!( + "skipping `a_coalesce_inside_an_operand_is_lowered`: no `python3` interpreter found" + ); + return; + }; + run(&python, NESTED); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_coalesce_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!("skipping `a_coalesce_keeps_its_grouping`: no `python3` interpreter found"); + return; + }; + run(&python, GROUPING); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_coalesce_binds_its_value_where_python_refuses_an_assignment_expression() { + let Some(python) = python() else { + eprintln!( + "skipping `a_coalesce_binds_its_value_where_python_refuses_an_assignment_expression`: no `python3` interpreter found" + ); + return; + }; + run(&python, BINDING); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_coalesce_binds_its_value_in_an_annotation_and_a_type_parameter_scope() { + let Some(python) = python() else { + eprintln!( + "skipping `a_coalesce_binds_its_value_in_an_annotation_and_a_type_parameter_scope`: no `python3` interpreter found" + ); + return; + }; + run(&python, ANNOTATION_SCOPES); +} diff --git a/docs/basedpython/features/none-coalesce.md b/docs/basedpython/features/none-coalesce.md index 28ed632f6e..ee0aad819f 100644 --- a/docs/basedpython/features/none-coalesce.md +++ b/docs/basedpython/features/none-coalesce.md @@ -16,6 +16,22 @@ a compound left operand is bound to a temp by the walrus so it is evaluated exactly once. a bare name needs no temp and is repeated directly — `a ?? b` is `a if a is not None else b` +python refuses the walrus in some places: a comprehension's iterable, a +comprehension in a class body, an annotation, and a generic definition's type +parameters, bases and `type` value. there the temp is a comprehension variable +instead, which still evaluates the left operand once and the right one only when +it is needed: + +```by +names = [n for n in user.nicknames ?? []] +``` + +transpiles to: + +```python +names = [n for n in ([__by_t_0__ for __by_t_0__ in (user.nicknames,) if __by_t_0__ is not None] or [[]])[0]] +``` + ## semantics `??` tests `is not None` (identity) — not falsiness. an empty string, zero, @@ -23,9 +39,17 @@ or an empty list is *not* coalesced. only `None` triggers the fallback ## precedence -`??` has the same precedence as a conditional expression — it binds looser -than `or` / `and` / arithmetic and tighter than assignment. parenthesize when -mixing with boolean operators +`??` has the same precedence as `or` and groups left to right with it. it binds +looser than `and`, `not`, comparisons and arithmetic, and tighter than a +conditional expression: + +```by +a ?? b or c # (a ?? b) or c +a or b ?? c # (a or b) ?? c +a ?? b and c # a ?? (b and c) +``` + +parenthesize when mixing it with boolean operators ## interaction with `?.` From 8bbb8df001d7ee850d9cc214f713455ac4591d94 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:42:47 +1000 Subject: [PATCH 04/47] lower a `?.` chain inside the receiver of another, behind a call (`n?.me()?.v`), instead of leaving it in the output; bind a chain's receivers as comprehension variables instead of with `:=` where python refuses one (a comprehension iterable, a class-body comprehension); and refuse a chain whose call or subscript reads a class body's name from a class-body comprehension's first iterable, which that comprehension would no longer see --- .../src/transforms/none_chain.rs | 225 ++++++++++++++---- crates/by_transforms/src/type_info.rs | 9 + .../tests/optional_chain_runtime.rs | 131 ++++++++++ .../ty_python_semantic/src/semantic_model.rs | 5 + .../src/types/visibility.rs | 29 +++ .../basedpython/features/optional-chaining.md | 20 ++ 6 files changed, 374 insertions(+), 45 deletions(-) create mode 100644 crates/by_transforms/tests/optional_chain_runtime.rs diff --git a/crates/by_transforms/src/transforms/none_chain.rs b/crates/by_transforms/src/transforms/none_chain.rs index e16893546d..15fb988b6d 100644 --- a/crates/by_transforms/src/transforms/none_chain.rs +++ b/crates/by_transforms/src/transforms/none_chain.rs @@ -1,12 +1,13 @@ use std::fmt::Write as _; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt}; +use ruff_python_ast::{Expr, ExprName, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::source_util::temporary_name; +use crate::transforms::walrus_scope::WalrusRegions; use crate::type_info::TypeInfo; /// rewrites `a?.b` to `(None if a is None else a.b)` and chains like `a?.b?.c` @@ -14,17 +15,9 @@ use crate::type_info::TypeInfo; pub(crate) struct NoneChain<'src> { source: &'src str, types: &'src dyn TypeInfo, + regions: &'src WalrusRegions, template_edits: Vec<(TextRange, Vec)>, -} - -impl<'src> NoneChain<'src> { - fn new(source: &'src str, types: &'src dyn TypeInfo) -> Self { - Self { - source, - types, - template_edits: Vec::new(), - } - } + errors: Vec, } /// How many temporaries a single expansion may need before it starts reusing @@ -48,16 +41,16 @@ fn pick_temp_var(written: WrittenNames<'_>, types: &dyn TypeInfo, anchor: &Expr) /// when any `?.` is present, where `python_form` has all `?.` replaced by `.`, /// `guards` is the ordered list of accumulated sub-expressions that must be /// non-None before each subsequent optional access is safe, and `base` is the -/// source range of the first `?.`'s receiver — everything before the first `?` +/// first `?.`'s receiver — everything before the first `?` /// /// a `?.` whose base is a *wrapped* optional (`int??`, a generic `T?`) /// reaches its present value through the runtime wrapper: the guard still /// tests the wrapper against `None`, but the access reads `.value` first -pub(super) fn expand_chain( - expr: &Expr, +fn expand_chain<'a>( + expr: &'a Expr, source: &str, types: &dyn TypeInfo, -) -> Option<(String, Vec, TextRange)> { +) -> Option<(String, Vec, &'a Expr)> { let Expr::Attribute(attr) = expr else { return None; }; @@ -84,7 +77,7 @@ pub(super) fn expand_chain( Some(( format!("{v_form}{unwrap}.{field}"), vec![v_form], - attr.value.range(), + &attr.value, )) } } @@ -92,7 +85,8 @@ pub(super) fn expand_chain( /// builds a `None if ... is None else ...` chain from guards and final result, /// using walrus assignment to avoid evaluating compound intermediate -/// expressions twice. +/// expressions twice. `trailers` — the calls, subscripts and attributes applied to +/// the chain's last access — pass through after the result /// /// returns fragments rather than text: a compound chain base is walrus-bound /// exactly once, and passing it through as a [`Fragment::Src`] span lets @@ -100,16 +94,26 @@ pub(super) fn expand_chain( /// instead of being clobbered. only the first guard can carry the raw base — /// every later guard is an incremental `temp.suffix` — and a name-only base /// has no interior for an edit to target, so the `Src` is needed exactly there -pub(super) fn build_expansion( +/// +/// where python refuses `:=` (see [`WalrusRegions`]), the guards from the first +/// compound one on are written as the clauses of a comprehension instead, each +/// binding the temporary as its variable — `([t.c for t in (a.b,) if t is not None +/// for t in (t.d,) if t is not None] or [None])[0]` — which evaluates each guard +/// once and stops at the first `None` the same way +fn build_expansion( guards: &[String], result: &str, temp: &str, base: TextRange, + trailers: TextRange, + walrus: bool, ) -> Vec { let mut fragments: Vec = Vec::new(); let mut s = String::new(); let mut use_t = false; let mut prev_guard: Option<&str> = None; + // the comprehension's clauses, once a compound guard has opened one + let mut clauses: Vec = Vec::new(); for (position, guard) in guards.iter().enumerate() { let guard_expr = if let Some(prev) = prev_guard.filter(|_| use_t) { @@ -121,7 +125,7 @@ pub(super) fn build_expansion( if guard_expr.chars().all(|c| c.is_alphanumeric() || c == '_') { let _ = write!(s, "None if {guard_expr} is None else "); - } else { + } else if walrus { if position == 0 { let _ = write!(s, "None if ({temp} := "); fragments.push(Fragment::Lit(std::mem::take(&mut s))); @@ -131,6 +135,18 @@ pub(super) fn build_expansion( let _ = write!(s, "None if ({temp} := {guard_expr}) is None else "); } use_t = true; + } else { + if clauses.is_empty() { + s.push_str("(["); + } + clauses.push(Fragment::Lit(format!(" for {temp} in ("))); + if position == 0 { + clauses.push(Fragment::Src(base)); + } else { + clauses.push(Fragment::Lit(guard_expr)); + } + clauses.push(Fragment::Lit(format!(",) if {temp} is not None"))); + use_t = true; } prev_guard = Some(guard.as_str()); } @@ -141,10 +157,12 @@ pub(super) fn build_expansion( } else { s.push_str(result); } - if !s.is_empty() { - fragments.push(Fragment::Lit(s)); + fragments.push(Fragment::Lit(s)); + fragments.push(Fragment::Src(trailers)); + if !clauses.is_empty() { + fragments.extend(clauses); + fragments.push(Fragment::Lit("] or [None])[0]".to_owned())); } - fragments } @@ -157,7 +175,7 @@ fn chain_head<'a>( expr: &'a Expr, source: &str, types: &dyn TypeInfo, -) -> Option<(&'a Expr, String, Vec, TextRange)> { +) -> Option<(&'a Expr, String, Vec, &'a Expr)> { if let Some((form, guards, base)) = expand_chain(expr, source, types) { return Some((expr, form, guards, base)); } @@ -185,39 +203,89 @@ impl TypeAwarePass for NoneChainPass<'_> { } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = NoneChain::new(self.source, types); + let mut inner = NoneChain { + source: self.source, + types, + regions: &ctx.walrus_regions, + template_edits: Vec::new(), + errors: Vec::new(), + }; for stmt in stmts { inner.visit_stmt(stmt); } ctx.template_edits.extend(inner.template_edits); + ctx.errors.extend(inner.errors); } } impl NoneChain<'_> { + /// where python refuses `:=` the trailers move into the comprehension that binds + /// the chain's temporary. a name they read in a class body's own scope — from a + /// comprehension's first iterable there — then no longer sees what the class body + /// bound, and would fail or quietly read a module-level name of the same name. + /// such a chain is refused rather than lowered + fn refuse_class_body_names(&mut self, expr: &Expr, head: &Expr) { + struct Names<'a>(Vec<&'a ExprName>); + impl<'a> Visitor<'a> for Names<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr { + self.0.push(name); + } + walk_expr(self, expr); + } + } + let mut names = Names(Vec::new()); + trailer_operands(expr, head, &mut |operand| names.visit_expr(operand)); + // a reference ty did not place may be one, so it is refused too + if let Some(name) = names + .0 + .into_iter() + .find(|name| self.types.reads_class_scope_binding(name) != Some(false)) + { + let line = self.source[..usize::from(expr.range().start())] + .matches('\n') + .count() + + 1; + self.errors.push(format!( + "the `?.` chain on line {line} cannot be lowered: it is in a comprehension's \ + first iterable in a class body, where python refuses `:=`, and the \ + comprehension that holds its value instead would not see the class body's \ + `{}`", + name.id + )); + } + } + /// visit the sub-expressions of the trailers between `head` and `expr` — the parts that - /// pass through as source and so can carry chains of their own. `head` itself is + /// pass through as source and so can carry chains of their own. `head`'s own links are /// rendered from its expansion, not walked - fn visit_trailers(&mut self, expr: &Expr, head: &Expr) { - if expr.range() == head.range() { - return; - } - match expr { - Expr::Attribute(attribute) => self.visit_trailers(&attribute.value, head), - Expr::Call(call) => { - self.visit_trailers(&call.func, head); - for arg in &*call.arguments.args { - self.visit_expr(arg); - } - for keyword in &*call.arguments.keywords { - self.visit_expr(&keyword.value); - } + fn visit_trailers<'a>(&mut self, expr: &'a Expr, head: &'a Expr) { + trailer_operands(expr, head, &mut |operand| self.visit_expr(operand)); + } +} + +/// calls `visit` on each operand of the trailers between `head` and `expr`: the +/// arguments of a call, the index of a subscript +fn trailer_operands<'a>(expr: &'a Expr, head: &'a Expr, visit: &mut impl FnMut(&'a Expr)) { + if expr.range() == head.range() { + return; + } + match expr { + Expr::Attribute(attribute) => trailer_operands(&attribute.value, head, visit), + Expr::Call(call) => { + trailer_operands(&call.func, head, visit); + for arg in &*call.arguments.args { + visit(arg); } - Expr::Subscript(subscript) => { - self.visit_trailers(&subscript.value, head); - self.visit_expr(&subscript.slice); + for keyword in &*call.arguments.keywords { + visit(&keyword.value); } - _ => {} } + Expr::Subscript(subscript) => { + trailer_operands(&subscript.value, head, visit); + visit(&subscript.slice); + } + _ => {} } } @@ -236,11 +304,24 @@ impl<'ast> Visitor<'ast> for NoneChain<'_> { if let Some((head, form, guards, base)) = chain_head(expr, self.source, self.types) { let temp = pick_temp_var(WrittenNames::new(self.source), self.types, expr); let trailers = TextRange::new(head.range().end(), expr.range().end()); + let walrus = self.regions.allows(expr.range()); + if !walrus { + self.refuse_class_body_names(expr, head); + } let mut fragments = vec![Fragment::Lit("(".to_owned())]; - fragments.extend(build_expansion(&guards, &form, &temp, base)); - fragments.push(Fragment::Src(trailers)); + fragments.extend(build_expansion( + &guards, + &form, + &temp, + base.range(), + trailers, + walrus, + )); fragments.push(Fragment::Lit(")".to_owned())); self.template_edits.push((expr.range(), fragments)); + // the base passes through as source, so a chain or a `??` inside it + // lowers on its own + self.visit_expr(base); self.visit_trailers(expr, head); return; } @@ -436,6 +517,60 @@ mod tests { check("x = z[a?.b]\n", "x = z[(None if a is None else a.b)]\n"); } + /// a chain's base passes through as source, so a chain inside it, behind a call, + /// lowers on its own + #[test] + fn chain_through_a_call() { + check( + "x = a?.b()?.c\n", + "x = (None if (__by_t_0__ := (None if a is None else a.b())) is None else __by_t_0__.c)\n", + ); + } + + /// python refuses `:=` in a comprehension's iterable, so the guards are the clauses + /// of a comprehension there, binding the temporary as its variable + #[test] + fn chain_in_a_comprehension_iterable() { + check( + "x = [i for i in f()?.b?.c(1)]\n", + "x = [i for i in (([__by_t_0__.c(1) for __by_t_0__ in (f(),) if __by_t_0__ is not None for __by_t_0__ in (__by_t_0__.b,) if __by_t_0__ is not None] or [None])[0])]\n", + ); + check( + "x = [i for i in a?.b?.c]\n", + "x = [i for i in (None if a is None else ([__by_t_0__.c for __by_t_0__ in (a.b,) if __by_t_0__ is not None] or [None])[0])]\n", + ); + } + + /// moving the trailers into that comprehension would take them out of a class + /// body's scope, so a name the class body binds there is refused + #[test] + fn class_body_name_in_a_moved_trailer_is_refused() { + let err = transpile( + "class C:\n k = 1\n x = [i for i in f()?.g(k)]\n", + &Config::test_default(), + ) + .unwrap_err(); + assert!( + err.contains("the `?.` chain on line 3 cannot be lowered") && err.contains("`k`"), + "got: {err}" + ); + } + + /// a builtin, a module-level name and an enclosing function's local resolve the + /// same from the comprehension, so a chain reading them is lowered + #[test] + fn a_name_the_class_body_does_not_bind_is_lowered() { + for source in [ + "class C:\n x = [i for i in f()?.g(len)]\n", + "k = 1\nclass C:\n x = [i for i in f()?.g(k)]\n", + "def outer():\n k = 3\n class C:\n x = [i for i in f()?.g(k)]\n return C\n", + "class C:\n x = [i for i in f()?.g(k)]\n k = 1\n", + ] { + transpile(source, &Config::test_default()) + .unwrap_or_else(|err| panic!("refused {source:?}: {err}")); + } + } + #[test] fn python_unchanged() { unchanged("x = None if a is None else a.b\n"); diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 92afa7add7..2177a73b56 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -438,6 +438,11 @@ pub(crate) trait TypeInfo { /// `global`. `None` when ty did not index the reference and cannot place it fn resolves_to_module_scope(&self, reference: &ExprName) -> Option; + /// whether the name `reference` reads is read in a class body's own scope and may + /// find a binding the class body made, which the same read moved into a scope + /// nested in the class body would not. `None` when ty did not index the reference + fn reads_class_scope_binding(&self, reference: &ExprName) -> Option; + /// whether `name`, read in an annotation, is a forward reference: a name the /// program binds, but not by the point python evaluates the annotation where /// it is written, as it does before 3.14 without `from __future__ import @@ -1218,6 +1223,10 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::resolves_to_module_scope(self, reference) } + fn reads_class_scope_binding(&self, reference: &ExprName) -> Option { + SemanticModel::reads_class_scope_binding(self, reference) + } + fn is_forward_reference(&self, name: &ExprName) -> Option { SemanticModel::is_forward_reference(self, name) } diff --git a/crates/by_transforms/tests/optional_chain_runtime.rs b/crates/by_transforms/tests/optional_chain_runtime.rs new file mode 100644 index 0000000000..50e3dc95ee --- /dev/null +++ b/crates/by_transforms/tests/optional_chain_runtime.rs @@ -0,0 +1,131 @@ +//! runtime test for the `?.` (optional chain) lowering +//! +//! `a?.b` lowers to a conditional expression that evaluates each receiver once, +//! binding a compound one to a temporary, and stops at the first `None`. the +//! transform's unit tests assert on the lowered *text*; this test runs it, with the +//! chain written where the lowering has to compose with what surrounds it: behind a +//! call on another chain, whose receiver it passes through as source, and where python +//! refuses the `:=` that binds the temporary — a comprehension's iterable, and any +//! part of a comprehension in a class body +//! +//! no third-party packages are needed, so any `python3` will do; if none is found the +//! test skips rather than fails + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; +use common::python; + +const CHAINS: &str = r#" +calls: list[str] = [] + +class Node: + def __init__(self, value: int, items: list[int]): + self.value = value + self.items = items + + def me(self) -> Node?: + calls.append("me") + return self + + def nothing(self) -> Node?: + return None + +def make(present: bool) -> Node?: + calls.append("make") + return Node(1, [1, 2]) if present else None + +# a chain behind a call on another chain +def through_a_call(n: Node?) -> int?: + return n?.me()?.value + +assert through_a_call(Node(3, [])) == 3 and through_a_call(None) is None +assert make(True)?.nothing()?.value is None + +# in a comprehension's iterable +def iterable(present: bool) -> list[int]: + return [x for x in make(present)?.items ?? []] + +calls.clear() +assert iterable(True) == [1, 2] and calls == ["make"], "the receiver is evaluated once" +assert iterable(False) == [] + +def nested_iterable() -> list[int]: + return [x for _ in [0] for x in make(True)?.me()?.items ?? []] + +calls.clear() +assert nested_iterable() == [1, 2] and calls == ["make", "me"] + +def in_a_display() -> list[list[int]?]: + return [y for y in [make(True)?.items, make(False)?.items]] + +assert in_a_display() == [[1, 2], None] + +class Holder: + let element: list[int?] = [make(present)?.value for present in [True, False]] + let condition: list[int] = [k for k in [1, 2] if make(k == 1)?.value] + +assert Holder.element == [1, None] +assert Holder.condition == [1] + +# from a class body comprehension's first iterable, the chain's call still reads a +# builtin and an enclosing function's local, which the comprehension it moves into sees +class Picker: + def __init__(self, items: list[int]): + self.items = items + + def pick(self, f: (list[int]) -> int) -> list[int]: + return [f(self.items)] + +def picker() -> Picker?: + return Picker([4, 5]) + +class Lengths: + let picked: list[int] = [n for n in picker()?.pick(len) ?? []] + +assert Lengths.picked == [2] + +def outer() -> list[int]: + def first(xs: list[int]) -> int: + return xs[0] + class Inner: + let picked: list[int] = [n for n in picker()?.pick(first) ?? []] + return Inner.picked + +assert outer() == [4] + +print("ok") +"#; + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn optional_chains_run_correctly() { + let Some(python) = python() else { + eprintln!("skipping optional chain runtime test: no `python3` interpreter found"); + return; + }; + + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(CHAINS, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled optional chain program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 71cc911518..24dc72dc0e 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -145,6 +145,11 @@ impl<'db> SemanticModel<'db> { crate::types::visibility::resolves_to_module_scope(self.db, self.file, reference) } + /// basedpython: see `crate::types::visibility::reads_class_scope_binding` + pub fn reads_class_scope_binding(&self, reference: &ast::ExprName) -> Option { + crate::types::visibility::reads_class_scope_binding(self.db, self.file, reference) + } + /// basedpython: the module-level names this file declares `private` — the /// set `private-import` reads, and whose references the lowering renames pub fn private_module_symbols(&self) -> Vec { diff --git a/crates/ty_python_semantic/src/types/visibility.rs b/crates/ty_python_semantic/src/types/visibility.rs index ac5e5ee2c7..9029779088 100644 --- a/crates/ty_python_semantic/src/types/visibility.rs +++ b/crates/ty_python_semantic/src/types/visibility.rs @@ -386,6 +386,35 @@ pub(crate) fn resolves_to_module_scope<'db>( Some(false) } +/// basedpython: whether the name `reference` reads is read in a class body's own +/// scope and may find a binding the class body made — which the same read, moved +/// into a scope nested in the class body, would no longer see. a read the body has +/// not bound on any path resolves outward, as python resolves it, so it answers +/// `false`. `None` when ty did not index the reference +pub(crate) fn reads_class_scope_binding<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + reference: &ast::ExprName, +) -> Option { + let index = crate::semantic_index(db, file); + let reference_scope = index.try_expression_scope_id(&ast::ExprRef::from(reference))?; + if index.scope(reference_scope).kind() != ScopeKind::Class || !reference.ctx.is_load() { + return Some(false); + } + let scope_id = reference_scope.to_scope_id(db, file); + let table = ty_python_core::place_table(db, scope_id); + let Some(symbol) = table.symbol_by_name(&reference.id) else { + return Some(false); + }; + if symbol.is_global() || symbol.is_nonlocal() || !(symbol.is_bound() || symbol.is_declared()) { + return Some(false); + } + Some(match class_scope_binding(db, scope_id, reference) { + ClassBinding::Unbound => false, + ClassBinding::Bound | ClassBinding::Maybe => true, + }) +} + /// basedpython: how a member reached through a receiver is restricted on one /// class the receiver can be #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/docs/basedpython/features/optional-chaining.md b/docs/basedpython/features/optional-chaining.md index 81d12ceec1..35f54661a6 100644 --- a/docs/basedpython/features/optional-chaining.md +++ b/docs/basedpython/features/optional-chaining.md @@ -43,6 +43,26 @@ zip = user?.address.zip `a.b?.c` works in the obvious way: only the part after `a.b` is short-circuited through a temp variable +where python refuses the walrus — a comprehension's iterable, a comprehension in +a class body, an annotation, a generic definition's type parameters — the temp is +a comprehension variable instead, one clause per guard: + +```by +class Report: + let streets: list[list[str]?] = [user.address?.streets for _ in range(2)] +``` + +transpiles to: + +```python +class Report: + streets: list[list[str] | None] = [(([__by_t_0__.streets for __by_t_0__ in (user.address,) if __by_t_0__ is not None] or [None])[0]) for _ in range(2)] +``` + +from a comprehension's first iterable in a class body, that moves the chain's +calls and subscripts out of the class body's scope, so a chain whose call reads a +name the class body binds there is refused rather than lowered + ## the chain runs to the end of the trailers a `?.` opens a chain that runs out through every trailer that follows it — From 4918d0505253ef7ed38fcc6a6a069eb2f34c7193 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:42:54 +1000 Subject: [PATCH 05/47] bind the receiver of a grapheme `first` / `last` as a comprehension variable instead of with `:=` where python refuses one, and name its temporary like every other lowering's instead of `_s`, which overwrote a module's own `_s` and was an ordinary name in a class body --- .../src/transforms/grapheme_string.rs | 32 ++++++-- .../by_transforms/tests/grapheme_runtime.rs | 73 +++++++++++++++++-- 2 files changed, 90 insertions(+), 15 deletions(-) diff --git a/crates/by_transforms/src/transforms/grapheme_string.rs b/crates/by_transforms/src/transforms/grapheme_string.rs index 768a2f7b53..480f31f2a6 100644 --- a/crates/by_transforms/src/transforms/grapheme_string.rs +++ b/crates/by_transforms/src/transforms/grapheme_string.rs @@ -52,6 +52,8 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::coalesce::is_trivially_pure; use super::repeated_underscore::WrittenNames; +use super::source_util::temporary_name; +use super::walrus_scope::WalrusRegions; use crate::type_info::TypeInfo; #[expect( @@ -61,6 +63,7 @@ use crate::type_info::TypeInfo; struct GraphemeString<'src> { types: &'src dyn TypeInfo, written: WrittenNames<'src>, + regions: &'src WalrusRegions, /// attribute nodes that are the direct callee of a call — the grapheme /// properties are not callable there, so they pass through untouched callee_attrs: HashSet, @@ -204,12 +207,26 @@ impl GraphemeString<'_> { Fragment::Lit(" else None)".to_owned()), ] } else { - vec![ - Fragment::Lit("(Character(_by_graphemes(_s)".to_owned()), - Fragment::Lit(format!("[{index}]) if (_s := ")), - Fragment::Src(receiver), - Fragment::Lit(") else None)".to_owned()), - ] + let temp = temporary_name(self.written, "s", 0); + if self.regions.allows(expr.range()) { + vec![ + Fragment::Lit(format!( + "(Character(_by_graphemes({temp})[{index}]) if ({temp} := " + )), + Fragment::Src(receiver), + Fragment::Lit(") else None)".to_owned()), + ] + } else { + // where python refuses `:=` (see `WalrusRegions`) the receiver is + // bound as a comprehension variable instead + vec![ + Fragment::Lit(format!( + "([Character(_by_graphemes({temp})[{index}]) for {temp} in (" + )), + Fragment::Src(receiver), + Fragment::Lit(format!(",) if {temp}] or [None])[0]")), + ] + } }; self.template(expr, fragments); } @@ -296,6 +313,7 @@ impl TypeAwarePass for GraphemeStringPass<'_> { let mut inner = GraphemeString { types, written: self.written, + regions: &ctx.walrus_regions, callee_attrs: HashSet::new(), needs_grapheme_helper: false, needs_character_class: false, @@ -428,7 +446,7 @@ def _by_suffix(_text, _n): "{CHAR_IMPORT}{HELPER}{}", indoc! {" def g() -> str: ... - a = (Character(_by_graphemes(_s)[0]) if (_s := g()) else None) + a = (Character(_by_graphemes(__by_s_0__)[0]) if (__by_s_0__ := g()) else None) "} ), ); diff --git a/crates/by_transforms/tests/grapheme_runtime.rs b/crates/by_transforms/tests/grapheme_runtime.rs index 91534a0f5d..b3c3e32aba 100644 --- a/crates/by_transforms/tests/grapheme_runtime.rs +++ b/crates/by_transforms/tests/grapheme_runtime.rs @@ -73,8 +73,34 @@ assert type("a") is str, "a bare literal is still a plain str" print("ok") "#; -#[test] -fn grapheme_surface_runs_correctly() { +/// `first` and `last` bind a receiver that is more than a name to a temporary, which +/// python refuses to bind with `:=` in a comprehension's iterable or anywhere in a +/// comprehension in a class body +const BOUND_RECEIVER: &str = r#" +from ty_extensions import Character + +def greeting() -> str: + return "a🇺🇸é" + +assert [c for c in [greeting().first, greeting().last]] == ["a", "é"], "in an iterable" + +class Letters: + let firsts: list[Character?] = [greeting().first for _ in [0]] + +assert Letters.firsts == ["a"], "in a class body comprehension" + +print("ok") +"#; + +fn transpiled(program: &str) -> String { + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + transpile(program, &config).expect("transpile should succeed") +} + +fn run_with_regex(program: &str) { let Some(interpreter) = interpreters::oldest( PythonVersion::PY313, "import regex", @@ -82,12 +108,7 @@ fn grapheme_surface_runs_correctly() { ) else { return; }; - - let config = Config { - min_version: PythonVersion::PY313, - ..Config::default() - }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let transpiled = transpiled(program); let output = Command::new(&interpreter.command) .arg("-c") @@ -104,3 +125,39 @@ fn grapheme_surface_runs_correctly() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); interpreters::ran(&interpreter, PythonVersion::PY313); } + +#[test] +fn grapheme_surface_runs_correctly() { + run_with_regex(PROGRAM); +} + +#[test] +fn a_bound_receiver_runs_where_python_refuses_an_assignment_expression() { + run_with_regex(BOUND_RECEIVER); +} + +/// the same module compiles, `regex` or not: an assignment expression where python +/// refuses one is a `SyntaxError` before the helpers that need `regex` ever run +#[test] +fn a_bound_receiver_compiles_where_python_refuses_an_assignment_expression() { + let Some(interpreter) = interpreters::oldest(PythonVersion::PY313, "", "") else { + return; + }; + let transpiled = transpiled(BOUND_RECEIVER); + + let output = Command::new(&interpreter.command) + .args([ + "-c", + "import sys; compile(sys.argv[1], '', 'exec')", + &transpiled, + ]) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "the transpiled grapheme program does not compile on {interpreter}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + interpreters::ran(&interpreter, PythonVersion::PY313); +} From 1e2286971979b8fe47c1c77af24464b049a6a11e Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:42:55 +1000 Subject: [PATCH 06/47] parenthesize a `^` operand written over several lines where the guard `^` lowers to evaluates it into a temporary, so its second line no longer fails to parse --- .../by_transforms/src/transforms/propagate.rs | 5 ++- .../src/transforms/source_util.rs | 11 +++++ .../tests/statement_expression_runtime.rs | 43 ++++++++++++++++++- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/crates/by_transforms/src/transforms/propagate.rs b/crates/by_transforms/src/transforms/propagate.rs index a4d98daeae..3465b6ebaf 100644 --- a/crates/by_transforms/src/transforms/propagate.rs +++ b/crates/by_transforms/src/transforms/propagate.rs @@ -28,7 +28,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{PassContext, TypeAwarePass}; use super::repeated_underscore::WrittenNames; -use super::source_util::temporary_name; +use super::source_util::{standalone_expression_text, temporary_name}; use crate::type_info::{AbsentTest, TypeInfo}; /// The guard condition that detects the "absent" value for a given operand @@ -194,8 +194,9 @@ impl<'ast> Visitor<'ast> for Propagate<'_> { let temp = temporary_name(self.written, "prop", self.counter); self.counter += 1; let cond = absent_condition(absent, &temp, self.written); + let operand = standalone_expression_text(self.source, unary.operand.range()); ( - format!("{temp} = {operand_src}\n{indent}if {cond}: return {temp}\n{indent}"), + format!("{temp} = {operand}\n{indent}if {cond}: return {temp}\n{indent}"), format!("{temp}{unwrap}"), ) }; diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index eb58ac9a04..121db15bd7 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -34,6 +34,17 @@ pub(crate) fn standalone_expression(source: &str, range: TextRange) -> Vec String { + standalone_expression(source, range) + .iter() + .map(|fragment| match fragment { + Fragment::Lit(text) => text.as_str(), + Fragment::Src(span) => &source[*span], + }) + .collect() +} + /// Whether `expr`, written as it is, has to be parenthesized to stand as one argument /// of a call among others: a tuple or generator expression written without its own /// parentheses (`for x in a, b:`), or a `yield`, which a call does not accept bare diff --git a/crates/by_transforms/tests/statement_expression_runtime.rs b/crates/by_transforms/tests/statement_expression_runtime.rs index 754737a117..71bb3f93fa 100644 --- a/crates/by_transforms/tests/statement_expression_runtime.rs +++ b/crates/by_transforms/tests/statement_expression_runtime.rs @@ -1,8 +1,11 @@ -//! runtime test for the early exits a statement expression lowers to +//! runtime test for the early exits a statement expression lowers to, and the one +//! `^` lowers to //! //! `x = value or return`, `x = value ?? raise E()` and `x = a if c else raise E()` //! lower to statements: each operand is evaluated into a temporary on a line of its -//! own, and the diverging branch becomes the body of an `if`. the transform's unit +//! own, and the diverging branch becomes the body of an `if`. `x = value^` evaluates +//! its operand into a temporary the same way, ahead of a guard that returns it when +//! it is absent. the transform's unit //! tests assert on the lowered *text*; this test runs it. an operand the source wrote //! over several lines, inside parentheses that are not part of its own range, has to //! keep a grouping when it moves onto its own line — without one, its second line @@ -125,6 +128,27 @@ else: print("ok") "#; +/// a `^` operand written over several lines +const PROPAGATE_MULTI_LINE: &str = r#" +class Node: + def __init__(self, value: int?): + self.value = value + self.inner = self + +def incremented(node: Node) -> int?: + x = ( + node + .inner + .value + )^ + return x + 1 + +assert incremented(Node(3)) == 4 +assert incremented(Node(None)) is None + +print("ok") +"#; + fn run(python: &str, program: &str) { let config = Config { min_version: PythonVersion::PY313, @@ -158,3 +182,18 @@ fn an_operand_written_over_several_lines_keeps_its_grouping() { }; run(&python, MULTI_LINE_OPERANDS); } + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_propagated_operand_written_over_several_lines_keeps_its_grouping() { + let Some(python) = python() else { + eprintln!( + "skipping `a_propagated_operand_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" + ); + return; + }; + run(&python, PROPAGATE_MULTI_LINE); +} From c58d967878068b613c082299b285c7b656fe6505 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:00:17 +1000 Subject: [PATCH 07/47] =?UTF-8?q?run=20python's=20compile-time=20checks=20?= =?UTF-8?q?over=20the=20transpiled=20module,=20on=20every=20version=20from?= =?UTF-8?q?=20its=20floor=20on,=20and=20refuse=20one=20python=20would=20no?= =?UTF-8?q?t=20compile=20=E2=80=94=20an=20assignment=20expression=20in=20a?= =?UTF-8?q?=20comprehension=20iterable=20or=20a=20class-body=20comprehensi?= =?UTF-8?q?on,=20a=20`return`=20outside=20a=20function,=20a=20parameter=20?= =?UTF-8?q?named=20twice=20=E2=80=94=20instead=20of=20writing=20out=20a=20?= =?UTF-8?q?module=20that=20raises=20`SyntaxError`=20at=20import;=20the=20t?= =?UTF-8?q?wo=20`match`=20checks=20phase=203=20already=20ran=20are=20among?= =?UTF-8?q?=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/by_transforms/src/lib.rs | 124 +++---- crates/by_transforms/src/semantic_syntax.rs | 313 ++++++++++++++++++ .../ruff_python_parser/src/semantic_errors.rs | 98 ------ .../development/how-transpilation-works.md | 6 +- 4 files changed, 369 insertions(+), 172 deletions(-) create mode 100644 crates/by_transforms/src/semantic_syntax.rs diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index b40128332f..8c0bf6e6ba 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -1,6 +1,7 @@ pub mod config; mod reverse_transforms; pub mod runtime; +mod semantic_syntax; pub(crate) mod source_map; mod transforms; pub(crate) mod type_info; @@ -890,27 +891,46 @@ fn run_version_polyfill_phase(source: String, config: &Config) -> String { } /// Re-parse the transpiled output *as the target python version* and report any -/// construct that version cannot parse. +/// construct that version cannot parse, or that python refuses to compile on any +/// version from it on /// /// [`verify_syntax`] asks whether the output is python at all; this asks whether /// it is python the file's declared floor can run. Without it, syntax no /// polyfill covers — `except*`, t-strings, a PEP 701 f-string — is emitted -/// verbatim and fails at import time in generated code the author never wrote. +/// verbatim and fails at import time in generated code the author never wrote, +/// and so does python that parses but does not compile: an assignment +/// expression in a comprehension's iterable, a capture pattern that makes the +/// remaining cases unreachable fn verify_target_syntax(source: &str, config: &Config) -> Result<(), TranspileError> { let options = ruff_python_parser::ParseOptions::from(ruff_python_ast::PySourceType::Python) .with_target_version(config.min_version); let parsed = ruff_python_parser::parse_unchecked(source, options); - let Some(first) = parsed.unsupported_syntax_errors().first() else { - return Ok(()); - }; - // the parser's own wording already names both versions: "Cannot use - // `except*` on Python 3.9 (syntax was added in Python 3.11)" - Err(TranspileError { - kind: TranspileErrorKind::InvalidSyntax, - message: first.to_string(), - output_range: Some(first.range), - by_range: None, - }) + if let Some(first) = parsed.unsupported_syntax_errors().first() { + // the parser's own wording already names both versions: "Cannot use + // `except*` on Python 3.9 (syntax was added in Python 3.11)" + return Err(TranspileError { + kind: TranspileErrorKind::InvalidSyntax, + message: first.to_string(), + output_range: Some(first.range), + by_range: None, + }); + } + // a `.by` source holds back python's two `match` checks, because a bare + // `case A:` name may be an enum member of the subject rather than the + // capture it looks like. by here every name is spelled out, so they apply + // with the rest + if let Some(module) = parsed.syntax().as_module() + && let Some(error) = + semantic_syntax::first_error(&module.body, source, config.min_version, config.is_stub) + { + return Err(TranspileError { + kind: TranspileErrorKind::InvalidSyntax, + message: format!("transpiler produced invalid Python: {error}"), + output_range: Some(error.range), + by_range: None, + }); + } + Ok(()) } /// Splice `preamble` into `body` where generated lines belong: after the module @@ -1096,19 +1116,6 @@ fn verify_syntax(source: &str, author: &AuthorNames) -> Result<(), TranspileErro }); } - // a `.by` source holds back python's two `match` checks, because a bare - // `case A:` name may be an enum member of the subject rather than the - // capture it looks like. by here every name is spelled out, so the checks - // apply again — and a `match` that fails them is python that will not parse - if let Some(error) = first_invalid_match_statement(parsed.suite()) { - return Err(TranspileError { - kind: TranspileErrorKind::InvalidSyntax, - message: format!("transpiler produced invalid Python: {error}"), - output_range: Some(error.range), - by_range: None, - }); - } - verify_runtime_helpers(parsed.suite(), &author.reads)?; verify_no_helper_name_clash(parsed.suite(), author)?; verify_no_shadowed_helper(parsed.suite(), author) @@ -1418,48 +1425,6 @@ fn verify_runtime_helpers(suite: &[Stmt], written: &HashSet) -> Result<( } } -/// The first `match` in `suite` that python's own parse-time checks reject. -fn first_invalid_match_statement( - suite: &[ruff_python_ast::Stmt], -) -> Option { - use ruff_python_ast::visitor::{Visitor, walk_stmt}; - use ruff_python_parser::semantic_errors::{SemanticSyntaxChecker, SemanticSyntaxError}; - - struct Scanner { - error: Option, - } - impl<'ast> Visitor<'ast> for Scanner { - fn visit_stmt(&mut self, stmt: &'ast ruff_python_ast::Stmt) { - if self.error.is_some() { - return; - } - if let ruff_python_ast::Stmt::Match(match_stmt) = stmt { - self.error = SemanticSyntaxChecker::python_match_statement_errors( - match_stmt, - // the checks are version-independent; `match` itself needs - // 3.10, which the parse above has already settled - ruff_python_ast::PythonVersion::latest(), - ) - .into_iter() - .next(); - if self.error.is_some() { - return; - } - } - walk_stmt(self, stmt); - } - } - - let mut scanner = Scanner { error: None }; - for stmt in suite { - scanner.visit_stmt(stmt); - if scanner.error.is_some() { - break; - } - } - scanner.error -} - /// Map a byte offset in the generated python to a byte range in the original /// `.by` source, via the line table. Returns the full `.by` line's range (line /// granularity is all the table provides today; see the sourcemap plan). @@ -2191,7 +2156,7 @@ mod transpile_error { /// The emitted python has every name spelled out, so a `match` that is /// invalid there has to be caught rather than written out. #[test] - fn verify_syntax_rejects_an_invalid_match() { + fn verify_target_syntax_rejects_an_invalid_match() { for source in [ // alternatives that bind different names "match x:\n case a | b:\n pass\n", @@ -2200,7 +2165,7 @@ mod transpile_error { // nested in a function, which the scan has to reach "def f(x):\n match x:\n case a | b:\n pass\n", ] { - let err = verify_syntax(source, &AuthorNames::none()).unwrap_err(); + let err = verify_target_syntax(source, &Config::test_default()).unwrap_err(); assert!( err.message .starts_with("transpiler produced invalid Python:"), @@ -2213,14 +2178,27 @@ mod transpile_error { /// The qualified spelling a resolved case name lowers to binds nothing, so /// the alternatives agree and the check has nothing to say. #[test] - fn verify_syntax_accepts_a_qualified_match() { - verify_syntax( + fn verify_target_syntax_accepts_a_qualified_match() { + verify_target_syntax( "match x:\n case Color.Red | Color.Green:\n pass\n case Color.Blue:\n pass\n", - &AuthorNames::none(), + &Config::test_default(), ) .unwrap(); } + /// python parses an assignment expression in a comprehension's iterable, and + /// refuses to compile it, so it is refused here rather than at import + #[test] + fn verify_target_syntax_rejects_what_python_does_not_compile() { + let err = + verify_target_syntax("[x for x in (t := xs)]\n", &Config::test_default()).unwrap_err(); + assert!( + err.message.contains("comprehension iterable"), + "got: {}", + err.message + ); + } + /// how many kinds of syntax the parser holds a python version to const KINDS: usize = 22; diff --git a/crates/by_transforms/src/semantic_syntax.rs b/crates/by_transforms/src/semantic_syntax.rs new file mode 100644 index 0000000000..e68622d10c --- /dev/null +++ b/crates/by_transforms/src/semantic_syntax.rs @@ -0,0 +1,313 @@ +//! python's compile-time checks on the emitted module +//! +//! a module python parses can still be one it refuses to compile: an assignment +//! expression in a comprehension's iterable, a parameter named twice, a `break` outside +//! a loop, a capture pattern that makes the remaining cases unreachable. the parser +//! reports these apart from its parse errors, through a checker that asks its caller +//! about the scope each node sits in. this answers it for the transpiled output, so a +//! module python would refuse is a transpile error instead of a `SyntaxError` at import +//! +//! the questions that need a binding analysis — what a `global` or `nonlocal` names, +//! whether a name is a parameter — are answered so that the checker reports nothing +//! about them. what those checks find is written by the author, not by a lowering, and +//! `by check` has already reported it on the source + +use std::cell::RefCell; + +use ruff_python_ast::{Expr, PythonVersion, Stmt}; +use ruff_python_parser::semantic_errors::{ + LazyImportContext, SemanticSyntaxChecker, SemanticSyntaxContext, SemanticSyntaxError, + SemanticSyntaxErrorKind, YieldOutsideFunctionKind, +}; +use ruff_text_size::TextRange; + +use crate::transforms::python_scopes::{self, Scope, ScopeHooks, ScopeState}; + +/// the first error python's compile-time checks find in `suite`, on any version from +/// `min_version` on. the checks changed between versions in both directions — an async +/// comprehension inside a sync one compiles from 3.11, an assignment expression in an +/// annotation stopped compiling in 3.14 — so each version is checked +pub(crate) fn first_error( + suite: &[Stmt], + source: &str, + min_version: PythonVersion, + is_stub: bool, +) -> Option { + let future_annotations_or_stub = is_stub || has_future_annotations(suite); + PythonVersion::iter() + .filter(|version| *version >= min_version) + .find_map(|python_version| { + let mut checker = Checker { + syntax: SemanticSyntaxChecker::new(), + errors: RefCell::default(), + source, + python_version, + future_annotations_or_stub, + }; + python_scopes::walk(suite, &mut checker); + checker.errors.into_inner().into_iter().next() + }) +} + +fn has_future_annotations(suite: &[Stmt]) -> bool { + suite.iter().any(|stmt| { + matches!(stmt, Stmt::ImportFrom(import) + if import.module.as_deref() == Some("__future__") + && import.names.iter().any(|alias| alias.name.as_str() == "annotations")) + }) +} + +struct Checker<'src> { + syntax: SemanticSyntaxChecker, + errors: RefCell>, + source: &'src str, + python_version: PythonVersion, + future_annotations_or_stub: bool, +} + +impl ScopeHooks<'_> for Checker<'_> { + fn stmt(&mut self, stmt: &Stmt, state: &ScopeState) { + let mut syntax = std::mem::take(&mut self.syntax); + syntax.visit_stmt( + stmt, + &Context { + checker: self, + state, + }, + ); + self.syntax = syntax; + } + + fn expr(&mut self, expr: &Expr, state: &ScopeState) { + let mut syntax = std::mem::take(&mut self.syntax); + syntax.visit_expr( + expr, + &Context { + checker: self, + state, + }, + ); + self.syntax = syntax; + } +} + +/// the checker's questions, answered at one node +struct Context<'a, 'src> { + checker: &'a Checker<'src>, + state: &'a ScopeState, +} + +impl Context<'_, '_> { + /// the scopes around the node, innermost first, past the annotation and type + /// parameter scopes, which the checker's questions look through + fn scopes(&self) -> impl Iterator { + self.state + .scopes() + .iter() + .rev() + .filter(|scope| !matches!(scope, Scope::Annotation | Scope::TypeParams)) + } +} + +impl SemanticSyntaxContext for Context<'_, '_> { + fn future_annotations_or_stub(&self) -> bool { + self.checker.future_annotations_or_stub + } + + fn lazy_import_context(&self) -> Option { + match self.scopes().next() { + Some(Scope::Function { .. }) => return Some(LazyImportContext::Function), + Some(Scope::Class) => return Some(LazyImportContext::Class), + _ => {} + } + self.state + .in_try() + .then_some(LazyImportContext::TryExceptBlocks) + } + + fn python_version(&self) -> PythonVersion { + self.checker.python_version + } + + fn source(&self) -> &str { + self.checker.source + } + + fn global(&self, _name: &str) -> Option { + None + } + + fn has_nonlocal_binding(&self, _name: &str) -> bool { + true + } + + fn in_async_context(&self) -> bool { + for scope in self.scopes() { + match scope { + Scope::Class | Scope::Lambda => return false, + Scope::Function { is_async } => return *is_async, + _ => {} + } + } + false + } + + fn in_await_allowed_context(&self) -> bool { + for scope in self.scopes() { + match scope { + Scope::Class => return false, + Scope::Function { .. } + | Scope::Lambda + | Scope::Comprehension { + generator: true, .. + } => return true, + _ => {} + } + } + false + } + + fn in_yield_allowed_context(&self) -> bool { + matches!( + self.scopes().next(), + Some(Scope::Function { .. } | Scope::Lambda) + ) + } + + fn in_sync_comprehension(&self) -> bool { + self.scopes().any(|scope| { + matches!( + scope, + Scope::Comprehension { + generator: false, + is_async: false + } + ) + }) + } + + fn in_class_body_comprehension(&self) -> bool { + for scope in self.scopes() { + match scope { + Scope::Comprehension { .. } => {} + Scope::Class => return true, + _ => return false, + } + } + false + } + + fn in_module_scope(&self) -> bool { + self.state.scopes().is_empty() + } + + fn in_function_scope(&self) -> bool { + matches!( + self.scopes().next(), + Some(Scope::Function { .. } | Scope::Lambda) + ) + } + + fn def_is_method(&self) -> bool { + // a definition is checked before the scope it opens is entered + matches!(self.state.scopes().last(), Some(Scope::Class)) + } + + fn in_generator_context(&self) -> bool { + self.scopes().any(|scope| { + matches!( + scope, + Scope::Comprehension { + generator: true, + .. + } + ) + }) + } + + fn in_notebook(&self) -> bool { + false + } + + fn report_semantic_error(&self, error: SemanticSyntaxError) { + // an `await` outside a function is reported as outside an async function too, + // which says the same thing + if matches!( + error.kind, + SemanticSyntaxErrorKind::YieldOutsideFunction(YieldOutsideFunctionKind::Await) + ) { + return; + } + self.checker.errors.borrow_mut().push(error); + } + + fn in_loop_context(&self) -> bool { + self.state.in_loop() + } + + fn is_bound_parameter(&self, _name: &str) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use ruff_python_ast::{PySourceType, PythonVersion}; + use ruff_python_parser::parse_unchecked_source; + + fn first_error(source: &str, min_version: PythonVersion) -> Option { + let parsed = parse_unchecked_source(source, PySourceType::Python); + super::first_error(parsed.suite(), source, min_version, false) + .map(|error| error.to_string()) + } + + /// what python refuses to compile, whatever scope it is written in + #[test] + fn refuses_what_python_does_not_compile() { + for source in [ + "[x for x in (t := xs)]\n", + "def f():\n return [x for y in z for x in (lambda: (t := 1))()]\n", + "class C:\n ys = [(t := x) for x in xs]\n", + "class C:\n ys = [x for x in xs if [(t := 1) for _ in z]]\n", + "def f(a, a):\n pass\n", + "def f():\n return [(yield) for x in xs]\n", + "class C:\n yield 1\n", + "break\n", + "for x in xs:\n pass\nelse:\n continue\n", + "while x:\n def f():\n break\n", + "for x in xs:\n class C:\n continue\n", + "class C[T]((t := 1)):\n pass\n", + ] { + assert!( + first_error(source, PythonVersion::PY313).is_some(), + "accepted: {source}" + ); + } + } + + /// and what it compiles + #[test] + fn accepts_what_python_compiles() { + for source in [ + "def f():\n return [(t := x) for x in xs]\n", + "class C:\n y = (t := 1)\n ys = [(lambda: (t := x))() for x in xs]\n", + "class C:\n def f(self, y=(t := 1)):\n return [(u := x) for x in xs]\n", + "def f():\n yield 1\n g = lambda: (yield)\n", + "async def f():\n return [await x for x in xs], (await x for x in xs)\n", + "def f():\n return (await x for x in xs)\n", + "x = [x for x in ([t for t in (f(),) if t is not None] or [[]])[0]]\n", + "for x in xs:\n if x:\n break\n try:\n continue\n finally:\n pass\n", + "while x:\n with y:\n break\nelse:\n pass\n", + "class C[T](b[([t for t in (f(),) if t is not None] or [0])[0]]):\n pass\n", + ] { + assert_eq!(first_error(source, PythonVersion::PY313), None, "{source}"); + } + } + + /// a check that changed between versions applies on every version the output runs on + #[test] + fn checks_every_version_the_output_runs_on() { + let source = "def f(x: (t := int)):\n pass\n"; + assert!(first_error(source, PythonVersion::PY313).is_some()); + } +} diff --git a/crates/ruff_python_parser/src/semantic_errors.rs b/crates/ruff_python_parser/src/semantic_errors.rs index 49fd6f1de5..b5f9fbe009 100644 --- a/crates/ruff_python_parser/src/semantic_errors.rs +++ b/crates/ruff_python_parser/src/semantic_errors.rs @@ -13,7 +13,6 @@ use ruff_python_ast::{ }; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use rustc_hash::{FxBuildHasher, FxHashSet}; -use std::cell::RefCell; use std::fmt::Display; #[derive(Debug, Default)] @@ -862,30 +861,6 @@ impl SemanticSyntaxChecker { } } - /// basedpython: the two `match` checks that a `.by` source holds back, run - /// over python. - /// - /// A bare `case A:` name may be an enum member of the subject rather than a - /// capture, and no syntactic rule can tell which — so for a `.by` source the - /// type checker reports both of these instead. The transpiler runs them - /// again over the python it emits, where every name is spelled out and the - /// answer is plain again, which is what keeps a `match` the checker rejected - /// from being emitted as python that cannot be parsed. - pub fn python_match_statement_errors( - stmt: &ast::StmtMatch, - python_version: PythonVersion, - ) -> Vec { - let context = CollectMatchErrors { - python_version, - errors: RefCell::default(), - }; - Self::irrefutable_match_case(stmt, &context); - for case in &stmt.cases { - Self::check_pattern_bindings(&case.pattern, &context); - } - context.errors.into_inner() - } - /// Reports a pattern that binds the same name more than once. fn check_pattern_bindings(pattern: &Pattern, ctx: &Ctx) { let mut visitor = MatchPatternVisitor { @@ -2985,79 +2960,6 @@ pub trait SemanticSyntaxContext { } } -/// The context [`SemanticSyntaxChecker::python_match_statement_errors`] runs its -/// two checks under. -/// -/// Neither reads anything about the surrounding scopes, so every other method is -/// answered with the value that makes its check a no-op — this context must -/// never be handed to the checker at large. -struct CollectMatchErrors { - python_version: PythonVersion, - errors: RefCell>, -} - -impl SemanticSyntaxContext for CollectMatchErrors { - fn report_semantic_error(&self, error: SemanticSyntaxError) { - self.errors.borrow_mut().push(error); - } - - fn python_version(&self) -> PythonVersion { - self.python_version - } - - fn future_annotations_or_stub(&self) -> bool { - false - } - fn lazy_import_context(&self) -> Option { - None - } - fn source(&self) -> &'static str { - "" - } - fn global(&self, _name: &str) -> Option { - None - } - fn has_nonlocal_binding(&self, _name: &str) -> bool { - false - } - fn in_async_context(&self) -> bool { - false - } - fn in_await_allowed_context(&self) -> bool { - false - } - fn in_yield_allowed_context(&self) -> bool { - false - } - fn in_sync_comprehension(&self) -> bool { - false - } - fn in_class_body_comprehension(&self) -> bool { - false - } - fn in_module_scope(&self) -> bool { - false - } - fn in_function_scope(&self) -> bool { - false - } - fn def_is_method(&self) -> bool { - true - } - fn in_generator_context(&self) -> bool { - false - } - fn in_notebook(&self) -> bool { - false - } - fn in_loop_context(&self) -> bool { - false - } - fn is_bound_parameter(&self, _name: &str) -> bool { - false - } -} - /// Modified version of [`std::str::EscapeDefault`] that does not escape single or double quotes. struct EscapeDefault<'a>(&'a str); diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index 327a23f2ed..b23af75c61 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -71,7 +71,11 @@ source (.by) holds a `.by` file to its target for every other kind, so a checker reports what this check would refuse. a basedpython form written as syntax of its own is a kind in that list too: a destructuring - pattern is written as a `match`, and is held to where one runs + pattern is written as a `match`, and is held to where one runs. + then run python's compile-time checks over it, on every version + from the floor on: python parses an assignment expression in a + comprehension's iterable, a parameter named twice, a capture that + makes the remaining `case`s unreachable, and refuses to compile them ``` entry points in `crates/by_transforms/src/lib.rs`: From c2c74cf03f519c983d813540313a65943c0493d7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:26:38 +1000 Subject: [PATCH 08/47] read a soft keyword after a parameter's modifiers as the parameter's name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `init(let cast: int)`, `let match`, `let type`, `let lazy` and `local type` did not parse ("Expected `,`, found `cast`"), and recovery then made a parameter named `let`. a soft keyword lexes as its own token kind, and the three modifier guards in `parse_parameter` — the binding / visibility chain, `context`, and `local` / `once` — took the keyword for a modifier only when a `Name` token followed it. they now go through `declares_a_name`, the test the statement-level declarations already use, so any of `match`, `case`, `type`, `lazy` and `cast` names the parameter. nothing can follow a parameter's name but `:`, `=`, `,` or `)`, so a keyword followed by one of those is still the parameter itself (`def f(let, match)`) --- .../src/transforms/init_method.rs | 20 ++++++ .../test/fixtures/ruff/init_method.by | 4 ++ .../test/fixtures/ruff/parameter_lifetimes.by | 4 ++ .../snapshots/format@init_method.by.snap | 8 +++ .../format@parameter_lifetimes.by.snap | 10 +++ .../src/parser/statement.rs | 8 +-- crates/ruff_python_parser/src/parser/tests.rs | 63 +++++++++++++++++++ crates/ty_ide/src/semantic_tokens.rs | 31 +++++++++ 8 files changed, 144 insertions(+), 4 deletions(-) diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index 2fdaf5883a..7db32a13b6 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -552,6 +552,26 @@ mod tests { ); } + /// a soft keyword is the name of the parameter its modifiers precede, and of the + /// attribute it declares + #[test] + fn a_soft_keyword_names_a_declaring_parameter() { + check( + indoc! {" + class A: + init(let cast: int, var match: int, private let type: int, local lazy: int, let case: int) + "}, + indoc! {" + class A: + def __init__(self, cast: int, match: int, type: int, lazy: int, case: int): + self.cast: int = cast + self.match: int = match + self.__type: int = type + self.case: int = case + "}, + ); + } + // `var` is accepted as the mutable counterpart of `let`; for an init // parameter it self-assigns identically #[test] diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/init_method.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/init_method.by index 9b66296126..0d57c3b558 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/init_method.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/init_method.by @@ -20,6 +20,10 @@ class PlainParams: init(self, a: int, let b: int) +class SoftKeywordNames: + init(let cast: int, var match: int, private let type: int, local lazy: int) + + class DefinitionModifiers: private init(self, let a: int) diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by index debd81db43..90f332ef86 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by @@ -22,6 +22,10 @@ def with_context(local fn: () -> None, context c: int) -> None: ... def not_modifiers(local, once) -> None: ... +# a soft keyword after the modifiers is the parameter's name +def soft_keyword_names(local type: int, once match: () -> None, context cast: int) -> None: ... + + # keyword-only, variadic and defaulted parameters def positions(local a: int, *, once cb: () -> None = print) -> None: ... diff --git a/crates/ruff_python_formatter/tests/snapshots/format@init_method.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@init_method.by.snap index 8ebac7ff2f..9551be94e4 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@init_method.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@init_method.by.snap @@ -26,6 +26,10 @@ class PlainParams: init(self, a: int, let b: int) +class SoftKeywordNames: + init(let cast: int, var match: int, private let type: int, local lazy: int) + + class DefinitionModifiers: private init(self, let a: int) @@ -98,6 +102,10 @@ class PlainParams: init(self, a: int, let b: int) +class SoftKeywordNames: + init(let cast: int, var match: int, private let type: int, local lazy: int) + + class DefinitionModifiers: private init(self, let a: int) diff --git a/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap index c8ed01bbd4..1c97c7557c 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap @@ -28,6 +28,10 @@ def with_context(local fn: () -> None, context c: int) -> None: ... def not_modifiers(local, once) -> None: ... +# a soft keyword after the modifiers is the parameter's name +def soft_keyword_names(local type: int, once match: () -> None, context cast: int) -> None: ... + + # keyword-only, variadic and defaulted parameters def positions(local a: int, *, once cb: () -> None = print) -> None: ... @@ -68,6 +72,12 @@ def with_context(local fn: () -> None, context c: int) -> None: ... def not_modifiers(local, once) -> None: ... +# a soft keyword after the modifiers is the parameter's name +def soft_keyword_names( + local type: int, once match: () -> None, context cast: int +) -> None: ... + + # keyword-only, variadic and defaulted parameters def positions(local a: int, *, once cb: () -> None = print) -> None: ... diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 6410073993..b9c8b19df4 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -7885,7 +7885,7 @@ impl<'src> Parser<'src> { // permissive here; the `init_method` lowering rejects combinations that // are invalid for this context while self.at(TokenKind::Name) - && self.peek() == TokenKind::Name + && declares_a_name(self.peek()) && is_param_modifier_kw(self.src_text(self.current_token_range())) { let kw = self.src_text(self.current_token_range()).to_string(); @@ -7901,7 +7901,7 @@ impl<'src> Parser<'src> { if matches!(allow_context, AllowContextModifier::Yes) && self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "context" - && self.peek() == TokenKind::Name + && declares_a_name(self.peek()) { self.error_if_not_basedpython( "`context` parameter modifier is not valid in .py files".to_string(), @@ -7916,11 +7916,11 @@ impl<'src> Parser<'src> { // order (`once local fn`). Like `let`, the keywords carry no AST field: // the `Parameter.range` covers them (it starts before this run) and the // strip transform + ty analysis detect them from the source span. The - // trailing `Name` guard keeps a parameter literally named `local`/`once` + // trailing name guard keeps a parameter literally named `local`/`once` // (`def f(local)`) from being read as a modifier. while self.at(TokenKind::Name) && matches!(self.src_text(self.current_token_range()), "local" | "once") - && self.peek() == TokenKind::Name + && declares_a_name(self.peek()) { self.error_if_not_basedpython( "`local` / `once` parameter modifiers are not valid in .py files".to_string(), diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index f9899eac1e..efbbf47364 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -600,6 +600,69 @@ fn basedpython_local_once_as_bare_param_name() { } } +/// the parameter names of the first function in `parsed`, which may be the first +/// statement of a class body +fn parameter_names(parsed: &Parsed) -> Vec { + let function = match parsed.suite().first() { + Some(Stmt::FunctionDef(function)) => function, + Some(Stmt::ClassDef(class)) => match class.body.first() { + Some(Stmt::FunctionDef(function)) => function, + other => panic!("expected a method, got {other:?}"), + }, + other => panic!("expected a function or a class, got {other:?}"), + }; + function + .parameters + .iter() + .map(|parameter| parameter.name().to_string()) + .collect() +} + +#[test] +fn basedpython_param_modifier_before_a_soft_keyword_name() { + // a soft keyword lexes as its own token kind, but after a parameter modifier it + // is the parameter's name like any other identifier + for keyword in ["match", "case", "type", "lazy", "cast"] { + for modifier in [ + "let", + "var", + "private", + "private let", + "local", + "once", + "once local", + "context", + "context local", + ] { + let source = format!("def f({modifier} {keyword}: int, {modifier} b: int): ...\n"); + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + assert_eq!(parameter_names(&parsed), [keyword, "b"], "{source}"); + } + let source = format!("class A:\n init(let {keyword}: int, var b: int = 1)\n"); + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + assert_eq!(parameter_names(&parsed), ["self", keyword, "b"], "{source}"); + } + + // a modifier keyword followed by a separator is the parameter itself, whether or + // not a soft keyword names a parameter beside it + for (source, names) in [ + ("def f(let, match): ...\n", &["let", "match"][..]), + ("def f(local, type): ...\n", &["local", "type"]), + ("def f(match, let): ...\n", &["match", "let"]), + ("def f(let let): ...\n", &["let"]), + ( + "def f(context: int, cast: int): ...\n", + &["context", "cast"], + ), + ] { + let parsed = parse_basedpython_module_with_errors(source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + assert_eq!(parameter_names(&parsed), names, "{source}"); + } +} + #[test] fn basedpython_local_once_param_rejected_in_py() { // the modifiers are `.by`-only; a `.py` file using them collects a diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index adeeedf665..00e500d575 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -6558,6 +6558,37 @@ class A: "#); } + #[test] + fn semantic_tokens_soft_keyword_parameter_after_a_modifier() { + // a soft keyword after a parameter's modifiers is the parameter's name + let test = SemanticTokenTest::new_by( + " +class A: + init(let cast: int, var match: int) + +def f(local type: int, once lazy: int): ... +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "A" @ 7..8: Class [definition] + "init" @ 14..18: Keyword + "cast" @ 23..27: Parameter [definition] + "int" @ 29..32: Class + "match" @ 38..43: Parameter [definition] + "int" @ 45..48: Class + "f" @ 55..56: Function [definition] + "local" @ 57..62: Keyword + "type" @ 63..67: Parameter [definition] + "int" @ 69..72: Class + "once" @ 74..78: Keyword + "lazy" @ 79..83: Parameter [definition] + "int" @ 85..88: Class + "#); + } + #[test] fn semantic_tokens_super_keyword() { let test = SemanticTokenTest::new_by( From 193a72fddbfe3de4cb1c6b5524802f2ef3966837 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:32:53 +1000 Subject: [PATCH 09/47] read a soft keyword after a callable-type parameter's `local` / `once` as the type it borrows `(local type[int]) -> None` did not parse: like the parameter modifiers, the callable-type modifier was taken only when a `Name` token followed it. a soft keyword now counts too, except a `cast` that is the infix cast operator, so `(local cast int)` still casts a value named `local`. the parser decides that from the token after `cast` exactly as it does when it reads the operator, through one shared `infix_cast_kind` --- .../test/fixtures/ruff/parameter_lifetimes.by | 5 +++ .../format@parameter_lifetimes.by.snap | 10 +++++ .../src/parser/expression.rs | 39 ++++++++++++------- crates/ruff_python_parser/src/parser/tests.rs | 39 ++++++++++++++++++- crates/ty_ide/src/semantic_tokens.rs | 33 ++++++++++++++++ 5 files changed, 110 insertions(+), 16 deletions(-) diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by index 90f332ef86..e6691036b9 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/parameter_lifetimes.by @@ -30,6 +30,11 @@ def soft_keyword_names(local type: int, once match: () -> None, context cast: i def positions(local a: int, *, once cb: () -> None = print) -> None: ... +# in a callable type the modifier borrows the parameter it precedes, which may be +# named by a soft keyword +def borrows(make: (local type[int], once str) -> int) -> int: ... + + # lambdas take the same modifiers handler = lambda local fn: fn diff --git a/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap index 1c97c7557c..e25707fe2f 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@parameter_lifetimes.by.snap @@ -36,6 +36,11 @@ def soft_keyword_names(local type: int, once match: () -> None, context cast: i def positions(local a: int, *, once cb: () -> None = print) -> None: ... +# in a callable type the modifier borrows the parameter it precedes, which may be +# named by a soft keyword +def borrows(make: (local type[int], once str) -> int) -> int: ... + + # lambdas take the same modifiers handler = lambda local fn: fn @@ -82,6 +87,11 @@ def soft_keyword_names( def positions(local a: int, *, once cb: () -> None = print) -> None: ... +# in a callable type the modifier borrows the parameter it precedes, which may be +# named by a soft keyword +def borrows(make: (local type[int], once str) -> int) -> int: ... + + # lambdas take the same modifiers handler = lambda local fn: fn diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index f9df346a4a..c19ef458e3 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -159,6 +159,17 @@ fn force_unwrap_before_debug_equal() -> ParseErrorType { ) } +/// basedpython: which infix cast a `cast` keyword followed by `after_cast` is, or `None` when +/// that `cast` is an ordinary name +fn infix_cast_kind(after_cast: TokenKind) -> Option { + match after_cast { + TokenKind::Exclamation => Some(CastKind::Checked), + TokenKind::Question => Some(CastKind::Try), + kind if EXPR_SET.contains(kind) || kind.is_soft_keyword() => Some(CastKind::Static), + _ => None, + } +} + impl<'src> Parser<'src> { /// Returns `true` if the parser is at a name or keyword (including soft keyword) token. pub(super) fn at_name_or_keyword(&self) -> bool { @@ -577,14 +588,7 @@ impl<'src> Parser<'src> { // expression level (where `left_precedence` is `None`). Lowered by // `transforms::checked_cast`. if current_token == TokenKind::Cast - && let Some(cast_kind) = match self.peek() { - TokenKind::Exclamation => Some(CastKind::Checked), - TokenKind::Question => Some(CastKind::Try), - peeked if EXPR_SET.contains(peeked) || peeked.is_soft_keyword() => { - Some(CastKind::Static) - } - _ => None, - } + && let Some(cast_kind) = infix_cast_kind(self.peek()) { if left_precedence > OperatorPrecedence::None { break; @@ -3584,7 +3588,9 @@ impl<'src> Parser<'src> { /// /// Only consumed when a bare name (a type) directly follows, so a /// parenthesized name (`(local)`), a call (`once(x)`), or a subscript - /// (`local[i]`) is never mistaken for a modifier. + /// (`local[i]`) is never mistaken for a modifier. A soft keyword is a name + /// there too (`(local type[int])`), except a `cast` that is the infix cast + /// operator applied to `local`. /// /// A run may spell both (`once local fn`); `once` is the stronger of the two /// (it implies the borrow) and wins. @@ -3592,11 +3598,16 @@ impl<'src> Parser<'src> { let mut borrow = ParameterBorrow::None; while self.at(TokenKind::Name) && matches!(self.src_text(self.current_token_range()), "local" | "once") - && self.peek() == TokenKind::Name - // `(local x=1)` is an anonymous named tuple *value*, which has no - // parameters to borrow — leave the name alone so it keeps its - // existing reading - && self.peek2().1 != TokenKind::Equal + && match self.peek2() { + // `(local x=1)` is an anonymous named tuple *value*, which has no + // parameters to borrow — leave the name alone so it keeps its + // existing reading + (_, TokenKind::Equal) => false, + // `(local cast int)` casts a value named `local`, as it would + // anywhere else + (TokenKind::Cast, after_cast) => infix_cast_kind(after_cast).is_none(), + (next, _) => next == TokenKind::Name || next.is_soft_keyword(), + } { self.error_if_not_basedpython( "`local` / `once` in a callable type are not valid in .py files".to_string(), diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index efbbf47364..227fa38fa3 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -3,8 +3,8 @@ use std::assert_matches; use ruff_python_ast::comparable::ComparableStmt; use ruff_python_ast::helpers::{UseSiteVariance, use_site_variance_marker}; use ruff_python_ast::{ - Expr, InterpolatedStringElement, IpyEscapeKind, ModModule, Number, Operator, Pattern, Stmt, - TypeParam, UnaryOp, is_destructure_binder, + CastKind, Expr, InterpolatedStringElement, IpyEscapeKind, ModModule, Number, Operator, Pattern, + Stmt, TypeParam, UnaryOp, is_destructure_binder, }; use ruff_text_size::Ranged; @@ -751,6 +751,10 @@ fn basedpython_local_once_in_callable_type() { // an implicit receiver is not one of `args`, so the modifiers still // line up with the written parameters ("f: int.(local str) -> None", &[Local]), + // a soft keyword after the modifier is the type it borrows + ("f: (local type[int]) -> None", &[Local]), + ("f: (int, once match) -> None", &[NoBorrow, Once]), + ("f: (local cast) -> None", &[Local]), ] { let parsed = parse_basedpython_module(source); let [Stmt::AnnAssign(assign)] = parsed.syntax().body.as_slice() else { @@ -773,6 +777,37 @@ fn basedpython_local_once_in_callable_type() { } } +/// `cast` followed by a type is the infix cast operator, so the `local` written before it +/// is the value it casts rather than a modifier on a parameter named `cast` +#[test] +fn basedpython_local_before_an_infix_cast_is_its_operand() { + for (source, kind) in [ + ("y = (local cast int)\n", CastKind::Static), + ("y = (local cast! int)\n", CastKind::Checked), + ("y = (local cast? int)\n", CastKind::Try), + ("y = (local cast int, once)\n", CastKind::Static), + ] { + let parsed = parse_basedpython_module_with_errors(source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + let [Stmt::Assign(assign)] = parsed.suite().as_slice() else { + panic!("expected an assignment for `{source}`"); + }; + let call = match assign.value.as_ref() { + Expr::Call(call) => call, + Expr::Tuple(tuple) => match tuple.elts.first() { + Some(Expr::Call(call)) => call, + other => panic!("expected a cast first in `{source}`, got {other:?}"), + }, + other => panic!("expected a cast in `{source}`, got {other:?}"), + }; + assert_eq!(call.cast_kind, Some(kind), "{source}"); + let [_, Expr::Name(value)] = &*call.arguments.args else { + panic!("expected the cast's value to be a name in `{source}`"); + }; + assert_eq!(value.id.as_str(), "local", "{source}"); + } +} + #[test] fn basedpython_local_once_in_callable_type_rejected_in_py() { // the modifiers are `.by`-only here too diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index 00e500d575..480eef2bff 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -6589,6 +6589,39 @@ def f(local type: int, once lazy: int): ... "#); } + #[test] + fn semantic_tokens_soft_keyword_after_a_callable_type_modifier() { + // a soft keyword after a callable-type parameter's modifier is the type it + // borrows, while `local cast int` stays a cast of `local` + let test = SemanticTokenTest::new_by( + " +def f(make: (local type[int], once str) -> None): ... + +local = 1 +x = (local cast int) +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "f" @ 5..6: Function [definition] + "make" @ 7..11: Parameter [definition] + "local" @ 14..19: Keyword + "type" @ 20..24: Class + "int" @ 25..28: Class + "once" @ 31..35: Keyword + "str" @ 36..39: Class + "None" @ 44..48: BuiltinConstant + "local" @ 56..61: Variable [definition] + "1" @ 64..65: Number + "x" @ 66..67: Variable [definition] + "local" @ 71..76: Variable + "cast" @ 77..81: Keyword + "int" @ 82..85: Class + "#); + } + #[test] fn semantic_tokens_super_keyword() { let test = SemanticTokenTest::new_by( From 7a1212bf80c049c53843b8b6f78e6d645e219d3e Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:42:38 +1000 Subject: [PATCH 10/47] read a soft keyword after `protocol`, `extension`, `newtype`, `implements` and `enum class` as the name it introduces `protocol match:`, `extension type:`, `newtype lazy = int`, `implements type` and `enum class case:` did not parse, for the reason a parameter modifier could not take one: each introducer was recognised only when a `Name` token followed it. they now take a soft keyword too, through `introduces_a_name`, which leaves a `cast` that the expression parser reads as the infix cast operator alone, so `protocol cast int` still casts a value named `protocol`. `@d` over `private protocol type:` panicked the parser ("left: Name, right: Class"): the decorated chain already took a soft keyword as a protocol's name, and then `parse_with_modifier`, which did not, fell through to `parse_class_definition` with `protocol` as its current token. every site that decides a protocol is being declared now asks the same question --- .../ruff/basedpython_stub_surface.byi | 9 ++ .../resources/test/fixtures/ruff/extension.by | 6 + .../format@basedpython_stub_surface.byi.snap | 18 +++ .../tests/snapshots/format@extension.by.snap | 12 ++ .../src/parser/expression.rs | 2 +- .../src/parser/statement.rs | 39 +++--- crates/ruff_python_parser/src/parser/tests.rs | 116 ++++++++++++++++++ crates/ty_ide/src/semantic_tokens.rs | 33 +++++ 8 files changed, 219 insertions(+), 16 deletions(-) diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/basedpython_stub_surface.byi b/crates/ruff_python_formatter/resources/test/fixtures/ruff/basedpython_stub_surface.byi index f851b714b1..3528de38da 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/basedpython_stub_surface.byi +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/basedpython_stub_surface.byi @@ -29,6 +29,15 @@ class Socket: let match: str var case: bytes +# it is also the name a declaration keyword introduces +protocol match: + def go(self) -> int + +newtype lazy = int + +enum class case: + case A + # the vendored typeshed spells an optional with the postfix marker, in every # position a stub writes one class Cursor: diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/extension.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/extension.by index 22ff5877ad..31e52e7881 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/extension.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/extension.by @@ -13,6 +13,12 @@ extension list[Element: int]: return sum(self) +# a soft keyword names the extended type like any other name +extension type: + def short_name(self) -> str: + return self.__name__[:2] + + # computed property extension str: @property diff --git a/crates/ruff_python_formatter/tests/snapshots/format@basedpython_stub_surface.byi.snap b/crates/ruff_python_formatter/tests/snapshots/format@basedpython_stub_surface.byi.snap index 12fb4e902e..1256174d06 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@basedpython_stub_surface.byi.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@basedpython_stub_surface.byi.snap @@ -35,6 +35,15 @@ class Socket: let match: str var case: bytes +# it is also the name a declaration keyword introduces +protocol match: + def go(self) -> int + +newtype lazy = int + +enum class case: + case A + # the vendored typeshed spells an optional with the postfix marker, in every # position a stub writes one class Cursor: @@ -77,6 +86,15 @@ class Socket: let match: str var case: bytes +# it is also the name a declaration keyword introduces +protocol match: + def go(self) -> int + +newtype lazy = int + +enum class case: + case A + # the vendored typeshed spells an optional with the postfix marker, in every # position a stub writes one class Cursor: diff --git a/crates/ruff_python_formatter/tests/snapshots/format@extension.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@extension.by.snap index 51b608fe02..8ddfbc65c2 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@extension.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@extension.by.snap @@ -19,6 +19,12 @@ extension list[Element: int]: return sum(self) +# a soft keyword names the extended type like any other name +extension type: + def short_name(self) -> str: + return self.__name__[:2] + + # computed property extension str: @property @@ -81,6 +87,12 @@ extension list[Element: int]: return sum(self) +# a soft keyword names the extended type like any other name +extension type: + def short_name(self) -> str: + return self.__name__[:2] + + # computed property extension str: @property diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index c19ef458e3..6f71e983f4 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -161,7 +161,7 @@ fn force_unwrap_before_debug_equal() -> ParseErrorType { /// basedpython: which infix cast a `cast` keyword followed by `after_cast` is, or `None` when /// that `cast` is an ordinary name -fn infix_cast_kind(after_cast: TokenKind) -> Option { +pub(super) fn infix_cast_kind(after_cast: TokenKind) -> Option { match after_cast { TokenKind::Exclamation => Some(CastKind::Checked), TokenKind::Question => Some(CastKind::Try), diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index b9c8b19df4..0f02b22271 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -18,7 +18,7 @@ use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::error::StarTupleKind; -use crate::parser::expression::{ArgumentsContext, EXPR_SET, ParsedExpr}; +use crate::parser::expression::{ArgumentsContext, EXPR_SET, ParsedExpr, infix_cast_kind}; use crate::parser::progress::ParserProgress; use crate::parser::{ FunctionKind, IpyEscapeContext, Parser, RecoveryContext, RecoveryContextKind, WithItemKind, @@ -746,6 +746,18 @@ impl<'src> Parser<'src> { } } + /// Whether the token `n` places past the next one is the name a declaration keyword + /// in front of it introduces — the `P` of `protocol P:`, or the `type` of + /// `extension type:`. A soft keyword is a name there like anywhere else, except a + /// `cast` that the expression parser would read as the infix cast operator: `protocol + /// cast int` casts a value named `protocol`, as `x cast int` casts `x` + fn introduces_a_name(&mut self, n: usize) -> bool { + match self.peek_nth(n).0 { + TokenKind::Cast => infix_cast_kind(self.peek_nth(n + 1).0).is_none(), + kind => declares_a_name(kind), + } + } + /// If the current token starts a basedpython modifier chain or introducer keyword /// (`let`, `newtype`, `protocol`, `abstract a: T`, or any combination of modifiers /// before `def`/`class`/`let`/`name = ...`), parse the corresponding statement. @@ -758,22 +770,20 @@ impl<'src> Parser<'src> { let kw = self.src_text(self.current_token_range()); // single-keyword introducers without modifier-chain support - if kw == "newtype" - && self.peek() == TokenKind::Name - && self.peek_nth(1).0 == TokenKind::Equal + if kw == "newtype" && declares_a_name(self.peek()) && self.peek_nth(1).0 == TokenKind::Equal { self.error_if_not_basedpython( "`newtype` declarations are not valid in .py files".to_string(), ); return Some(self.parse_newtype_decl(start)); } - if kw == "protocol" && self.peek() == TokenKind::Name { + if kw == "protocol" && self.introduces_a_name(0) { self.error_if_not_basedpython( "`protocol` class syntax is not valid in .py files".to_string(), ); return Some(self.parse_protocol_def(start, DecoratorList::new())); } - if kw == "extension" && self.peek() == TokenKind::Name { + if kw == "extension" && self.introduces_a_name(0) { self.error_if_not_basedpython( "`extension` declarations are not valid in .py files".to_string(), ); @@ -792,7 +802,7 @@ impl<'src> Parser<'src> { ); return Some(self.parse_build_def(start)); } - if kw == "implements" && self.peek() == TokenKind::Name { + if kw == "implements" && self.introduces_a_name(0) { self.error_if_not_basedpython( "`implements` declarations are not valid in .py files".to_string(), ); @@ -802,8 +812,7 @@ impl<'src> Parser<'src> { // sum type when its body has payload variants, an idiomatic `Enum` when // its variants are all unit). the `class` keyword is part of the // declaration - if kw == "enum" && self.peek() == TokenKind::Class && self.peek_nth(1).0 == TokenKind::Name - { + if kw == "enum" && self.peek() == TokenKind::Class && declares_a_name(self.peek_nth(1).0) { self.error_if_not_basedpython( "`enum class` declarations are not valid in .py files".to_string(), ); @@ -811,7 +820,7 @@ impl<'src> Parser<'src> { } // a bare `enum E:` (no `class`) is not valid — based enums are written // `enum class E:`. report it but still parse the body for recovery - if kw == "enum" && self.peek() == TokenKind::Name { + if kw == "enum" && self.introduces_a_name(0) { self.add_error( ParseErrorType::OtherError( "based enums must be written `enum class E:`, not `enum E:`".to_string(), @@ -1011,7 +1020,7 @@ impl<'src> Parser<'src> { // modifier chain — `private enum class E:`, `export protocol P:`. // dispatch through the modifier path so the chain is carried // as decorators on the introduced class - if (text == "protocol" && following == TokenKind::Name) + if (text == "protocol" && self.introduces_a_name(idx)) || (text == "enum" && following == TokenKind::Class) { self.error_if_not_basedpython(format!( @@ -1383,7 +1392,7 @@ impl<'src> Parser<'src> { Stmt::FunctionDef(self.parse_function_definition(decorators, start)) } else if self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "protocol" - && self.peek() == TokenKind::Name + && self.introduces_a_name(0) { // `private protocol P:` — the modifier decorators ride alongside the // synthetic `protocol_class` marker; both lower by disjoint edits @@ -2172,7 +2181,7 @@ impl<'src> Parser<'src> { let mut args: Vec = Vec::new(); loop { - if !self.at(TokenKind::Name) { + if !self.at_name_or_soft_keyword() { self.add_error( ParseErrorType::OtherError("`implements` takes an interface name".to_string()), self.current_token_range(), @@ -7556,7 +7565,7 @@ impl<'src> Parser<'src> { // appended after them. if self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "protocol" - && self.peek() == TokenKind::Name + && self.introduces_a_name(0) { self.error_if_not_basedpython( "`protocol` class syntax is not valid in .py files".to_string(), @@ -7676,7 +7685,7 @@ impl<'src> Parser<'src> { TokenKind::Name => { let text = self.src_text(range); // `protocol P:` and `enum class E:` introduce a definition - if text == "protocol" && declares_a_name(self.peek_nth(idx).0) { + if text == "protocol" && self.introduces_a_name(idx) { return true; } if text == "enum" && self.peek_nth(idx).0 == TokenKind::Class { diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 227fa38fa3..5c4877c4e7 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -373,6 +373,122 @@ fn basedpython_declares_a_soft_keyword_name() { assert!(alias.is_private); } +/// the name a declaration introduced by a keyword declares: a class or a newtype +fn introduced_name(stmt: &Stmt) -> &str { + match stmt { + Stmt::ClassDef(class) => class.name.as_str(), + Stmt::AnnAssign(assign) => match assign.target.as_ref() { + Expr::Name(name) => name.id.as_str(), + other => panic!("expected a name target, got {other:?}"), + }, + other => panic!("expected a declaration, got {other:?}"), + } +} + +/// the interface at `index` in the `implements` declaration that is the whole body of +/// the one class in `parsed` +fn implemented_interface(parsed: &Parsed, index: usize) -> &str { + let [Stmt::ClassDef(class)] = parsed.suite().as_slice() else { + panic!("expected one class, got {:?}", parsed.suite()); + }; + let [Stmt::Expr(statement)] = class.body.as_slice() else { + panic!( + "expected one `implements` declaration, got {:?}", + class.body + ); + }; + let Expr::Call(call) = statement.value.as_ref() else { + panic!( + "expected an `implements` marker call, got {:?}", + statement.value + ); + }; + match call.arguments.args.get(index) { + Some(Expr::Name(name)) => name.id.as_str(), + other => panic!("expected an interface name at {index}, got {other:?}"), + } +} + +#[test] +fn basedpython_an_introducer_declares_a_soft_keyword_name() { + // a soft keyword after `protocol`, `extension`, `newtype`, `implements` or + // `enum class` is the name the declaration introduces + for keyword in ["match", "case", "type", "lazy", "cast"] { + for source in [ + format!("protocol {keyword}:\n x: int\n"), + format!("private protocol {keyword}:\n x: int\n"), + format!("@runtime_checkable\nprotocol {keyword}:\n x: int\n"), + format!("@runtime_checkable\nprivate protocol {keyword}:\n x: int\n"), + format!("extension {keyword}:\n def f(self) -> int: ...\n"), + format!("newtype {keyword} = int\n"), + format!("enum class {keyword}:\n case A\n"), + format!("private enum class {keyword}:\n case A\n"), + ] { + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + let [stmt] = parsed.suite().as_slice() else { + panic!("expected one statement for `{source}`"); + }; + assert_eq!(introduced_name(stmt), keyword, "{source}"); + } + + for (source, index) in [ + (format!("class A:\n implements {keyword}, B\n"), 0), + (format!("class A:\n implements B, {keyword}\n"), 1), + ] { + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + assert_eq!(implemented_interface(&parsed, index), keyword, "{source}"); + } + } + + // a bare `enum E:` is reported as one error, whatever names the enum + for keyword in ["match", "type", "cast"] { + let source = format!("enum {keyword}:\n case A\n"); + let parsed = parse_basedpython_module_with_errors(&source); + let errors: Vec = parsed + .errors() + .iter() + .map(|error| error.error.to_string()) + .collect(); + assert_eq!( + errors, + ["based enums must be written `enum class E:`, not `enum E:`"], + "{source}" + ); + assert_eq!(introduced_name(&parsed.suite()[0]), keyword, "{source}"); + } + + // `cast` followed by a type is the infix cast operator, so the keyword in front of it + // is the value being cast rather than a declaration of something named `cast` + for keyword in ["protocol", "extension", "implements", "enum"] { + for operator in ["cast", "cast!", "cast?"] { + let source = format!("{keyword} {operator} int\n"); + let parsed = parse_basedpython_module_with_errors(&source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + let [Stmt::Expr(statement)] = parsed.suite().as_slice() else { + panic!("expected an expression statement for `{source}`"); + }; + let Expr::Call(call) = statement.value.as_ref() else { + panic!("expected a cast for `{source}`"); + }; + assert!(call.cast_kind.is_some(), "{source}"); + } + } + + // a decorated modifier chain decides that it ends at a protocol by the same test, so + // a cast there is reported as a decorator on something that is not a definition + let parsed = parse_basedpython_module_with_errors("@d\nprivate protocol cast int\n"); + assert!( + parsed.errors().iter().any(|error| error + .error + .to_string() + .contains("Expected a definition or a binding after decorator")), + "got {:?}", + parsed.errors() + ); +} + #[test] fn basedpython_context_composes_with_the_declaration_modifiers() { // `context` takes its place in the modifier chain rather than forming a statement of diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index 480eef2bff..80968a995e 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -6622,6 +6622,39 @@ x = (local cast int) "#); } + #[test] + fn semantic_tokens_soft_keyword_after_a_declaration_keyword() { + // a soft keyword after `protocol`, `newtype` or `enum class` is the name the + // declaration introduces + let test = SemanticTokenTest::new_by( + " +protocol match: + x: int + +newtype lazy = int + +enum class case: + case A +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "protocol" @ 1..9: Keyword + "match" @ 10..15: Class [definition] + "x" @ 21..22: Variable [definition] + "int" @ 24..27: Class + "newtype" @ 29..36: Keyword + "lazy" @ 37..41: Variable [definition] + "int" @ 44..47: Class + "enum class" @ 49..59: Keyword + "case" @ 60..64: Class [definition] + "case" @ 70..74: Keyword + "A" @ 75..76: Class [definition] + "#); + } + #[test] fn semantic_tokens_super_keyword() { let test = SemanticTokenTest::new_by( From 108136167598437d34c87ee710c954b1dd3b52c5 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:45:00 +1000 Subject: [PATCH 11/47] report a misplaced `some` before a soft keyword as one error, through the name test the introducers use `list[some type[int]]` reported "Expected `:`, found `type`" instead of the one "`some` is only allowed at the top of a parameter annotation" error, because the recovery arm took `some` for the keyword only when a `Name` token followed it. the test for "a name follows this keyword" now lives once, as `Parser::peek_is_name` beside the other token-kind predicates in `expression.rs`, and the misplaced `some`, the callable-type `local` / `once` and every declaration introducer ask it. `some cast int` still casts a variable named `some` --- .../src/parser/expression.rs | 63 +++++++++++-------- .../src/parser/statement.rs | 44 ++++++------- crates/ruff_python_parser/src/parser/tests.rs | 38 +++++++++++ 3 files changed, 96 insertions(+), 49 deletions(-) diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 6f71e983f4..66676357ce 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -161,7 +161,7 @@ fn force_unwrap_before_debug_equal() -> ParseErrorType { /// basedpython: which infix cast a `cast` keyword followed by `after_cast` is, or `None` when /// that `cast` is an ordinary name -pub(super) fn infix_cast_kind(after_cast: TokenKind) -> Option { +fn infix_cast_kind(after_cast: TokenKind) -> Option { match after_cast { TokenKind::Exclamation => Some(CastKind::Checked), TokenKind::Question => Some(CastKind::Try), @@ -186,6 +186,24 @@ impl<'src> Parser<'src> { self.current_token_kind().is_soft_keyword() } + /// basedpython: whether the token `n` places past the next one is a name, which a keyword + /// in front of it introduces or modifies — the `P` of `protocol P:`, the `type` of + /// `extension type:` or `(local type[int]) -> None`. a soft keyword is a name there like + /// anywhere else, except a `cast` that is the infix cast operator: `protocol cast int` + /// casts a value named `protocol`, as `x cast int` casts `x` + /// + /// this is the test for a keyword that could itself be an ordinary name, and so the + /// operand of that `cast`. where the position is already pinned down — the tokens after + /// the name are checked as well, or no expression is being read — the statement parser's + /// `declares_a_name` answers for a single token kind + pub(super) fn peek_is_name(&mut self, n: usize) -> bool { + match self.peek_nth(n).0 { + TokenKind::Name => true, + TokenKind::Cast => infix_cast_kind(self.peek_nth(n + 1).0).is_none(), + kind => kind.is_soft_keyword(), + } + } + /// basedpython: whether the expression just parsed ended with a suite, which /// makes it the last thing in the statement it belongs to. /// @@ -215,7 +233,8 @@ impl<'src> Parser<'src> { /// `None` if no variance keyword was present. /// /// Disambiguates `out` from a bare reference to a variable named `out`: - /// the variance form requires the next token to be a *name* (`out T`). + /// the variance form requires the next token to be a *name* (`out T`), as + /// [`Self::peek_is_name`] decides it. /// Two adjacent names are never valid Python, so `out T` is unambiguous — /// whereas `out[...]`, `out(...)`, `out.attr`, `out + 1` etc. are an /// ordinary subscript / call / attribute / arithmetic on a variable named @@ -224,7 +243,7 @@ impl<'src> Parser<'src> { &mut self, ) -> Option { use ruff_python_ast::helpers::UseSiteVariance; - let next_starts_type_expr = matches!(self.peek(), TokenKind::Name); + let next_starts_type_expr = self.peek_is_name(0); let variance = if self.at(TokenKind::In) { // `in` is a hard keyword, so its presence in subscript-start // position is unambiguously variance. @@ -289,19 +308,20 @@ impl<'src> Parser<'src> { /// range of the keyword itself. Returns `None` if no modifier is present. /// /// Disambiguated exactly like [`Self::eat_basedpython_variance_prefix`]: the - /// modifier form requires the next token to be a *name*, and two adjacent - /// names are never valid Python. So `literal str` is unambiguously the - /// modifier, while `literal`, `final[x]`, `literal.attr`, `final(x)` etc. - /// stay ordinary references to a variable of that name. The consequence is - /// that a type which does not start with a name — a parenthesized callable - /// type, a string forward reference, a starred type — cannot carry a bare - /// modifier; see the docs for the recommended spelling. + /// modifier form requires the next token to be a *name* + /// ([`Self::peek_is_name`]), and two adjacent names are never valid + /// Python. So `literal str` is unambiguously the modifier, while `literal`, + /// `final[x]`, `literal.attr`, `final(x)` etc. stay ordinary references to + /// a variable of that name. The consequence is that a type which does not + /// start with a name — a parenthesized callable type, a string forward + /// reference, a starred type — cannot carry a bare modifier; see the docs + /// for the recommended spelling. fn eat_basedpython_type_modifier_prefix( &mut self, ) -> Option<(ruff_python_ast::helpers::TypeModifier, TextRange)> { use ruff_python_ast::helpers::TypeModifier; - if !self.at(TokenKind::Name) || !matches!(self.peek(), TokenKind::Name) { + if !self.at(TokenKind::Name) || !self.peek_is_name(0) { return None; } let range = self.current_token_range(); @@ -1191,7 +1211,7 @@ impl<'src> Parser<'src> { TokenKind::Name if self.options.is_basedpython && self.src_text(self.current_token_range()) == "some" - && self.peek() == TokenKind::Name => + && self.peek_is_name(0) => { self.add_error( ParseErrorType::OtherError( @@ -3588,9 +3608,7 @@ impl<'src> Parser<'src> { /// /// Only consumed when a bare name (a type) directly follows, so a /// parenthesized name (`(local)`), a call (`once(x)`), or a subscript - /// (`local[i]`) is never mistaken for a modifier. A soft keyword is a name - /// there too (`(local type[int])`), except a `cast` that is the infix cast - /// operator applied to `local`. + /// (`local[i]`) is never mistaken for a modifier. /// /// A run may spell both (`once local fn`); `once` is the stronger of the two /// (it implies the borrow) and wins. @@ -3598,16 +3616,11 @@ impl<'src> Parser<'src> { let mut borrow = ParameterBorrow::None; while self.at(TokenKind::Name) && matches!(self.src_text(self.current_token_range()), "local" | "once") - && match self.peek2() { - // `(local x=1)` is an anonymous named tuple *value*, which has no - // parameters to borrow — leave the name alone so it keeps its - // existing reading - (_, TokenKind::Equal) => false, - // `(local cast int)` casts a value named `local`, as it would - // anywhere else - (TokenKind::Cast, after_cast) => infix_cast_kind(after_cast).is_none(), - (next, _) => next == TokenKind::Name || next.is_soft_keyword(), - } + && self.peek_is_name(0) + // `(local x=1)` is an anonymous named tuple *value*, which has no + // parameters to borrow — leave the name alone so it keeps its + // existing reading + && self.peek2().1 != TokenKind::Equal { self.error_if_not_basedpython( "`local` / `once` in a callable type are not valid in .py files".to_string(), diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 0f02b22271..2927cb9576 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -18,7 +18,7 @@ use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::error::StarTupleKind; -use crate::parser::expression::{ArgumentsContext, EXPR_SET, ParsedExpr, infix_cast_kind}; +use crate::parser::expression::{ArgumentsContext, EXPR_SET, ParsedExpr}; use crate::parser::progress::ParserProgress; use crate::parser::{ FunctionKind, IpyEscapeContext, Parser, RecoveryContext, RecoveryContextKind, WithItemKind, @@ -137,6 +137,13 @@ fn mark_context(stmt: Stmt, has_context: bool) -> Stmt { /// `let type: int` declares a field called `type`, which is what both /// `socket.SocketType` and `asyncio.TransportSocket` have. Only the token kind /// differs from a plain name — [`Parser::parse_identifier`] already reads either. +/// +/// use it only where `kind` cannot be the infix `cast` operator: either the tokens +/// around it are checked too (`let NAME =`, `class let NAME`), or it follows a +/// parameter's modifiers or an enum body's `case`, where no expression is being read +/// (`case cast(radius: float)` declares a variant named `cast`). where the word in front +/// of `kind` could be a value that a following `cast` operates on (`protocol cast +/// int`), ask [`Parser::peek_is_name`] instead fn declares_a_name(kind: TokenKind) -> bool { kind == TokenKind::Name || kind.is_soft_keyword() } @@ -746,18 +753,6 @@ impl<'src> Parser<'src> { } } - /// Whether the token `n` places past the next one is the name a declaration keyword - /// in front of it introduces — the `P` of `protocol P:`, or the `type` of - /// `extension type:`. A soft keyword is a name there like anywhere else, except a - /// `cast` that the expression parser would read as the infix cast operator: `protocol - /// cast int` casts a value named `protocol`, as `x cast int` casts `x` - fn introduces_a_name(&mut self, n: usize) -> bool { - match self.peek_nth(n).0 { - TokenKind::Cast => infix_cast_kind(self.peek_nth(n + 1).0).is_none(), - kind => declares_a_name(kind), - } - } - /// If the current token starts a basedpython modifier chain or introducer keyword /// (`let`, `newtype`, `protocol`, `abstract a: T`, or any combination of modifiers /// before `def`/`class`/`let`/`name = ...`), parse the corresponding statement. @@ -777,13 +772,13 @@ impl<'src> Parser<'src> { ); return Some(self.parse_newtype_decl(start)); } - if kw == "protocol" && self.introduces_a_name(0) { + if kw == "protocol" && self.peek_is_name(0) { self.error_if_not_basedpython( "`protocol` class syntax is not valid in .py files".to_string(), ); return Some(self.parse_protocol_def(start, DecoratorList::new())); } - if kw == "extension" && self.introduces_a_name(0) { + if kw == "extension" && self.peek_is_name(0) { self.error_if_not_basedpython( "`extension` declarations are not valid in .py files".to_string(), ); @@ -802,7 +797,7 @@ impl<'src> Parser<'src> { ); return Some(self.parse_build_def(start)); } - if kw == "implements" && self.introduces_a_name(0) { + if kw == "implements" && self.peek_is_name(0) { self.error_if_not_basedpython( "`implements` declarations are not valid in .py files".to_string(), ); @@ -820,7 +815,7 @@ impl<'src> Parser<'src> { } // a bare `enum E:` (no `class`) is not valid — based enums are written // `enum class E:`. report it but still parse the body for recovery - if kw == "enum" && self.introduces_a_name(0) { + if kw == "enum" && self.peek_is_name(0) { self.add_error( ParseErrorType::OtherError( "based enums must be written `enum class E:`, not `enum E:`".to_string(), @@ -1020,7 +1015,7 @@ impl<'src> Parser<'src> { // modifier chain — `private enum class E:`, `export protocol P:`. // dispatch through the modifier path so the chain is carried // as decorators on the introduced class - if (text == "protocol" && self.introduces_a_name(idx)) + if (text == "protocol" && self.peek_is_name(idx)) || (text == "enum" && following == TokenKind::Class) { self.error_if_not_basedpython(format!( @@ -1392,7 +1387,7 @@ impl<'src> Parser<'src> { Stmt::FunctionDef(self.parse_function_definition(decorators, start)) } else if self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "protocol" - && self.introduces_a_name(0) + && self.peek_is_name(0) { // `private protocol P:` — the modifier decorators ride alongside the // synthetic `protocol_class` marker; both lower by disjoint edits @@ -2524,9 +2519,7 @@ impl<'src> Parser<'src> { // `case` followed by a name declares variant(s) — `case A`, // `case A, B, C`, `case Circle(radius: float), Empty`. any other use of // `case` (`case = 1`, `case.x`, …) is an ordinary identifier statement - if self.at(TokenKind::Case) - && (self.peek() == TokenKind::Name || self.peek().is_soft_keyword()) - { + if self.at(TokenKind::Case) && declares_a_name(self.peek()) { self.parse_case_variants(statements); return; } @@ -7565,7 +7558,7 @@ impl<'src> Parser<'src> { // appended after them. if self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "protocol" - && self.introduces_a_name(0) + && self.peek_is_name(0) { self.error_if_not_basedpython( "`protocol` class syntax is not valid in .py files".to_string(), @@ -7685,7 +7678,7 @@ impl<'src> Parser<'src> { TokenKind::Name => { let text = self.src_text(range); // `protocol P:` and `enum class E:` introduce a definition - if text == "protocol" && self.introduces_a_name(idx) { + if text == "protocol" && self.peek_is_name(idx) { return true; } if text == "enum" && self.peek_nth(idx).0 == TokenKind::Class { @@ -7981,6 +7974,9 @@ impl<'src> Parser<'src> { let mut is_some = false; let annotation = match function_kind { FunctionKind::FunctionDef if self.eat(TokenKind::Colon) => { + // the bound is any type, including one that does not start with a name + // (`some () -> int`, `some (A | B)`), so this asks whether an expression + // follows rather than `peek_is_name` if self.at(TokenKind::Name) && self.src_text(self.current_token_range()) == "some" && (EXPR_SET.contains(self.peek()) || self.peek().is_soft_keyword()) diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 5c4877c4e7..21b6f45c1b 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -2095,6 +2095,14 @@ fn basedpython_use_site_variance_in_any_slice_element() { None, ], ), + // a soft keyword is a name after `out` like after any other keyword, but + // `out cast int` casts a variable named `out` + ("a: list[out type[int]]\n", vec![Some(UseSiteVariance::Out)]), + ( + "a: dict[int, out match]\n", + vec![None, Some(UseSiteVariance::Out)], + ), + ("a: X[out cast int]\n", vec![None]), ] { let parsed = parse_basedpython_module(source); assert_eq!( @@ -3091,6 +3099,9 @@ fn basedpython_type_modifier_prefix() { ("a: final int = 1\n", TypeModifier::Final), ("a: literal list[*]\n", TypeModifier::Literal), ("a: final dict[str, int]\n", TypeModifier::Final), + // a soft keyword is a name after the modifier like after any other keyword + ("a: final type[int]\n", TypeModifier::Final), + ("a: literal match\n", TypeModifier::Literal), ] { let parsed = parse_basedpython_module(source); let [Stmt::AnnAssign(ann)] = parsed.syntax().body.as_slice() else { @@ -3171,6 +3182,9 @@ fn basedpython_type_modifier_keywords_stay_identifiers() { "a: final[int]\n", "a: literal.Alias\n", "a: final()\n", + // `cast` followed by a type is the infix cast operator, so this casts a + // variable called `literal` + "a: literal cast int\n", ] { let parsed = parse_basedpython_module(source); let [Stmt::AnnAssign(ann)] = parsed.syntax().body.as_slice() else { @@ -3668,6 +3682,10 @@ fn a_misplaced_some_is_one_error() { "def g(xs: list[int]): ...", ), ("x: some int = 1\n", "x: int = 1"), + ( + "def h(xs: list[some type[int]]): ...\n", + "def h(xs: list[type[int]]): ...", + ), ("y = cast(some int?, 1)\n", "y = cast(int?, 1)"), ] { let parsed = parse_basedpython_module_with_errors(source); @@ -3695,6 +3713,26 @@ fn a_misplaced_some_is_one_error() { } } +/// at the top of a parameter's annotation `some` takes any type as its bound, not only one +/// that starts with a name +#[test] +fn some_at_the_top_of_an_annotation_takes_any_type() { + for source in [ + "def f(x: some int): ...\n", + "def f(x: some type[int]): ...\n", + "def f(x: some () -> int): ...\n", + "def f(x: some (int | str)): ...\n", + "def f(x: some \"Forward\"): ...\n", + ] { + let parsed = parse_basedpython_module_with_errors(source); + assert_eq!(parsed.errors(), &[], "expected a clean parse of `{source}`"); + let Some(Stmt::FunctionDef(function)) = parsed.suite().first() else { + panic!("expected a function for `{source}`"); + }; + assert!(function.parameters.args[0].parameter.is_some, "{source}"); + } +} + /// a variable named `some` followed by anything that continues an expression is that /// variable, as it is in python #[test] From 52ef3ea467390bd2c03105ef26640144234b4dd8 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:33:10 +1000 Subject: [PATCH 12/47] stop the parser panicking on a decorator over a modifier word that starts no definition in a `.py` file `@d` over `final x`, `private x = 1` or `private protocol cast int` in a `.py` file panicked the parser ("left: Name, right: Class"), so the formatter and linter crashed on a half-typed file. after a decorator, a modifier word went to `parse_with_modifier`, which reads only a definition, and only a `.by` file first asked `modifier_chain_ends_at_definition` whether one followed. both now ask, so a `.py` file gets python's own "Expected class, function definition or async function definition after decorator" recovery there, as it does over any other statement --- .../src/parser/statement.rs | 5 ++-- crates/ruff_python_parser/src/parser/tests.rs | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 2927cb9576..015d791e4b 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -7577,9 +7577,8 @@ impl<'src> Parser<'src> { // basedpython: a modifier chain also prefixes a *binding* — `export let // a = 1` — which is not what `parse_with_modifier` reads. When the chain // does not end at a definition the statement is parsed as the decorated - // binding it is instead - if at_modifier && (!self.options.is_basedpython || self.modifier_chain_ends_at_definition()) - { + // binding it is instead, or in a `.py` file gets python's own recovery + if at_modifier && self.modifier_chain_ends_at_definition() { return self.parse_with_modifier(start, decorators); } diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 21b6f45c1b..6f465d86aa 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -1022,6 +1022,30 @@ fn basedpython_decorated_protocol_keyword() { assert_eq!(names, ["runtime_checkable", "protocol_class"]); } +/// in a `.py` file a decorator over a modifier word that starts no definition gets python's +/// own recovery, as a decorator over any other statement does +#[test] +fn a_decorator_over_a_modifier_word_in_a_py_file_recovers() { + for source in [ + "@d\nfinal x\n", + "@d\nfinal = 1\n", + "@d\nprivate x = 1\n", + "@d\nprivate protocol cast int\n", + ] { + let parsed = crate::parse_unchecked(source, ParseOptions::from(Mode::Module)); + assert!( + parsed + .errors() + .iter() + .any(|error| error.error.to_string().starts_with( + "Expected class, function definition or async function definition" + )), + "{source}: {:?}", + parsed.errors() + ); + } +} + #[test] fn basedpython_final_in_modifier_chain_keeps_final_marker() { // `final` combined with another modifier must still carry the `__final__` From d35592e5f985336e6bb18dc26455bb11335b5e9e Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:54:41 +1000 Subject: [PATCH 13/47] =?UTF-8?q?spell=20a=20`float`=20type=20argument=20a?= =?UTF-8?q?s=20`float`=20in=20a=20`.by`=20module=20wherever=20the=20checke?= =?UTF-8?q?r=20writes=20a=20type=20out=20for=20the=20transpiler=20?= =?UTF-8?q?=E2=80=94=20a=20type=20test's=20or=20checked=20cast's=20probe,?= =?UTF-8?q?=20a=20reified=20call's=20specialization,=20a=20covariant=20cla?= =?UTF-8?q?ss's=20construction=20=E2=80=94=20instead=20of=20python's=20`in?= =?UTF-8?q?t=20|=20float`,=20so=20`x=20is=20L`=20with=20`type=20L=20=3D=20?= =?UTF-8?q?list[float]`=20now=20holds=20for=20a=20`list[float]`=20subclass?= =?UTF-8?q?=20and=20`argument(y)`=20with=20`y:=20float`=20reifies=20`T`=20?= =?UTF-8?q?as=20`float`=20(an=20invariant=20class=20constructed=20from=20a?= =?UTF-8?q?=20`float`=20variable=20is=20still=20specialized=20`A[int=20|?= =?UTF-8?q?=20float]`,=20because=20the=20checker=20solves=20it=20that=20wa?= =?UTF-8?q?y);=20and=20split=20promotion=20into=20three=20independent=20se?= =?UTF-8?q?ttings=20=E2=80=94=20whether=20literals=20widen,=20whether=20`f?= =?UTF-8?q?loat`=20/=20`complex`=20widen,=20and=20whether=20a=20function?= =?UTF-8?q?=20widens=20to=20its=20callable=20type=20=E2=80=94=20so=20the?= =?UTF-8?q?=20strict=20numeric=20model=20of=20a=20`.by`=20or=20`strict-flo?= =?UTF-8?q?at`=20module=20widens=20everything=20regular=20promotion=20does?= =?UTF-8?q?=20but=20`float`=20and=20`complex`=20(`self.a=20=3D=20A()`=20no?= =?UTF-8?q?w=20declares=20the=20attribute=20`A`=20rather=20than=20`final?= =?UTF-8?q?=20A`,=20and=20`self.b=20=3D=20B(1)`=20keeps=20`B[Literal[1]]`?= =?UTF-8?q?=20for=20a=20covariant=20`B`=20as=20python=20does),=20while=20a?= =?UTF-8?q?=20`.by`=20module=20keeps=20a=20function's=20own=20type,=20whos?= =?UTF-8?q?e=20reified=20type=20parameters,=20`context`=20parameters,=20`l?= =?UTF-8?q?ocal`=20parameters=20and=20`raises`=20clause=20no=20callable=20?= =?UTF-8?q?type=20carries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/transforms/type_reification.rs | 40 +++++ .../tests/parametric_is_runtime.rs | 66 ++++++++ .../tests/stored_function_runtime.rs | 86 +++++++++++ .../mdtest/basedpython_context_parameters.md | 30 ++++ .../mdtest/basedpython_escaping_local.md | 30 ++++ .../mdtest/basedpython_exceptions.md | 43 ++++++ .../mdtest/basedpython_init_method.md | 4 +- .../mdtest/basedpython_just_float.md | 83 ++++++++++ .../mdtest/basedpython_reified_generics.md | 31 ++++ crates/ty_python_semantic/src/types.rs | 146 +++++++++++++----- .../ty_python_semantic/src/types/generics.rs | 46 +++--- .../src/types/known_instance.rs | 7 +- .../src/types/reified_infer.rs | 13 +- 13 files changed, 557 insertions(+), 68 deletions(-) create mode 100644 crates/by_transforms/tests/stored_function_runtime.rs diff --git a/crates/by_transforms/src/transforms/type_reification.rs b/crates/by_transforms/src/transforms/type_reification.rs index 15450666c6..974f6becfd 100644 --- a/crates/by_transforms/src/transforms/type_reification.rs +++ b/crates/by_transforms/src/transforms/type_reification.rs @@ -318,6 +318,46 @@ mod tests { ); } + /// `float` in basedpython is just `float`, so it is what a construction is specialized + /// with, not python's `int | float`. a covariant argument goes through the promotion + /// that would apply python's special case + #[test] + fn a_float_argument_specializes_as_just_float() { + let out = out_at( + indoc! {" + class A[out T]: + def __init__(self, t: T): ... + def f(y: float): + a = A(1.5) + b = A(y) + "}, + PythonVersion::PY312, + ); + assert!(out.contains("a = A[float](1.5)"), "got: {out}"); + assert!(out.contains("b = A[float](y)"), "got: {out}"); + } + + /// backlog N15 #1, constructor calls still widen `float` in `.by`: pinned so a fix shows up + /// here. the checker solves an invariant parameter from a `float` variable through python's + /// numeric special case, so the construction is typed `A[int | float]` and specialized that + /// way. the literal is right + #[test] + fn an_invariant_float_argument_still_specializes_as_int_or_float() { + let out = out_at( + indoc! {" + class A[T]: + def __init__(self, t: T): + self.t = t + def f(y: float): + a = A(1.5) + b = A(y) + "}, + PythonVersion::PY312, + ); + assert!(out.contains("a = A[float](1.5)"), "got: {out}"); + assert!(out.contains("b = A[int | float](y)"), "got: {out}"); + } + #[test] fn generic_builtin_the_runtime_never_subscripts_stays_bare() { // a `zip` object is generic to the type checker, but no release of diff --git a/crates/by_transforms/tests/parametric_is_runtime.rs b/crates/by_transforms/tests/parametric_is_runtime.rs index 669147ee48..b4505a988d 100644 --- a/crates/by_transforms/tests/parametric_is_runtime.rs +++ b/crates/by_transforms/tests/parametric_is_runtime.rs @@ -444,6 +444,58 @@ assert len(calls) == 1, "an effectful value is evaluated once" print("ok") "#; +/// basedpython whose module-level `assert`s run what the transpiler writes out from the +/// checker's spelling of a type that holds a `float`. `float` in basedpython is just `float`, +/// not python's `int | float`, so a type test's probe compares the arguments a `list[float]` +/// subclass records against `float`, and a reified type parameter is handed `float` itself +const JUST_FLOAT_PROGRAM: &str = r#" +# a type test names the alias, so the probe is rebuilt from the alias's value +type Floats = list[float] + +class FloatList(list[float]): ... +class IntList(list[int]): ... + +def floats(v: object) -> bool: + return v is Floats + +assert floats(FloatList()) is True, "a list subclass of float is a list[float]" +assert floats(IntList()) is False, "a list subclass of int is not" +assert (FloatList() cast? Floats) is not None, "a checked cast accepts a list subclass of float" +assert (IntList() cast? Floats) is None, "and refuses a list subclass of int" + +# a reified type parameter is handed `float` itself +class Tagged[reified T]: + def __init__(self, t: T): + self.t = t + + def argument(self) -> object: + return T + +def argument[reified T](x: T) -> object: + return T + +def from_variable(y: float) -> object: + return argument(y) + +def from_floats(ys: list[float]) -> object: + return argument(ys) + +def from_complex(c: complex) -> object: + return argument(c) + +class Made: ... + +assert argument(1.5) === float, f"a float literal reifies as {argument(1.5)}" +assert from_variable(1.5) === float, f"a float variable reifies as {from_variable(1.5)}" +assert from_floats([1.5]) == list[float], f"a list of floats reifies as {from_floats([1.5])}" +assert from_complex(1j) === complex, f"a complex variable reifies as {from_complex(1j)}" +# a constructed value is exactly a `Made`, and the argument is the class +assert argument(Made()) === Made, f"a constructed value reifies as {argument(Made())}" +assert Tagged(1.5).argument() === float, f"a float literal specializes as {Tagged(1.5).argument()}" + +print("ok") +"#; + const PEP695_PROBE: &str = "type X[T] = T"; /// pep 696 defaults in native syntax plus the `has_default()` accessor the probe @@ -538,3 +590,17 @@ fn type_param_defaults_resolve_at_runtime() { }; run_program(&python, DEFAULTS_PROGRAM); } + +/// a reified class needs 3.13, so this is probed like [`DEFAULTS_PROGRAM`] +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_check_written_from_a_float_type_checks_just_float() { + let Some(python) = python_supporting(PEP696_PROBE) else { + eprintln!("skipping just-float runtime test: no 3.13+ interpreter found"); + return; + }; + run_program(&python, JUST_FLOAT_PROGRAM); +} diff --git a/crates/by_transforms/tests/stored_function_runtime.rs b/crates/by_transforms/tests/stored_function_runtime.rs new file mode 100644 index 0000000000..5d737ab96c --- /dev/null +++ b/crates/by_transforms/tests/stored_function_runtime.rs @@ -0,0 +1,86 @@ +//! Runtime test for a `.by` function read back out of where it was stored. +//! +//! A `.by` function can carry what a call has to act on — a `context` parameter the call fills, +//! reified type parameters the call specializes by subscript. The checker keeps the function's +//! own type when a list, a dict, a tuple or an attribute stores it, and the transpiler reads that +//! type to write the call: the filled `context` argument is only in the output because the +//! checker still sees the function. This test runs the result to see both arrive. + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod interpreters; + +const STORED: &str = r#" +def greet(a: int, context b: str) -> str: + return f"{b} {a}" + +def name[reified T]() -> str: + return T.__name__ + +class Holder: + def __init__(self): + self.greet = greet + self.name = name + +context salutation = "hello" + +greeters = [greet] +greeter_by_key = {"a": greet} +greeter_pair = (greet, 1) +held = Holder() +appended_greeters = [] +appended_greeters.append(greet) + +assert greeters[0](1) == "hello 1", greeters[0](1) +assert greeter_by_key["a"](2) == "hello 2", greeter_by_key["a"](2) +assert greeter_pair[0](3) == "hello 3", greeter_pair[0](3) +assert held.greet(4) == "hello 4", held.greet(4) +assert appended_greeters[0](5) == "hello 5", appended_greeters[0](5) + +names = [name] +name_by_key = {"a": name} +name_pair = (name, 1) +appended_names = [] +appended_names.append(name) + +assert names[0][int]() == "int", names[0][int]() +assert name_by_key["a"][str]() == "str", name_by_key["a"][str]() +assert name_pair[0][bytes]() == "bytes", name_pair[0][bytes]() +assert held.name[float]() == "float", held.name[float]() +assert appended_names[0][int]() == "int", appended_names[0][int]() + +print("ok") +"#; + +#[test] +fn a_stored_function_is_called_with_what_it_carries() { + // a reified generic is a pep 695 function. `oldest` says why when it skips + let Some(interpreter) = interpreters::oldest(PythonVersion::PY312, "", "") else { + return; + }; + let target = interpreter.version.min(PythonVersion::PY313); + let config = Config { + min_version: target, + ..Config::default() + }; + // `transpile` analyses the module with the checker, which is what fills the `context` + // arguments + let transpiled = transpile(STORED, &config).expect("transpile should succeed"); + + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "the transpiled program failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md b/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md index dd3f3efbf3..d8aa0ba443 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md @@ -447,6 +447,36 @@ a = A(s) a.m() # ok — bound methods resolve ``` +## a stored function still has its `context` parameters filled + +a type inferred from a stored value is widened, but a `.by` module keeps a function's own type +rather than widening it to a callable type, which has no `context` parameters. so a function read +back out of a list, a dict, a tuple or an attribute is still a plain function, and its `context` +parameters are filled where it is called + +```by +def f(a: int, context b: str) -> str: + return b + +class Holder: + def __init__(self): + self.f = f + +context s = "filled" +listed = [f] +keyed = {"a": f} +paired = (f, 1) +held = Holder() +appended = [] +appended.append(f) + +reveal_type(listed[0](1)) # revealed: str +reveal_type(keyed["a"](1)) # revealed: str +reveal_type(paired[0](1)) # revealed: str +reveal_type(held.f(1)) # revealed: str +reveal_type(appended[0](1)) # revealed: str +``` + ## an overloaded callee fills a parameter every overload agrees on Which overload a call selects is decided by the arguments it is given, so an argument may be written diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md b/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md index 2f56c6172d..65532df1bc 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md @@ -326,6 +326,36 @@ f: sink(it) ``` +## a stored function's parameters are still checked + +a type inferred from a stored value is widened, but a `.by` module keeps a function's own type +rather than widening it to a callable type. so a function read back out of a list, a dict, a tuple +or an attribute still says which of its parameters are `local`, and handing a `local` to one that is +not still escapes + +```by +def keep(cb: () -> None): + cb() + +class Holder: + def __init__(self): + self.k = keep + +listed = [keep] +keyed = {"a": keep} +paired = (keep, 1) +held = Holder() +appended = [] +appended.append(keep) + +def use(local cb: () -> None): + listed[0](cb) # error: [escaping-local] + keyed["a"](cb) # error: [escaping-local] + paired[0](cb) # error: [escaping-local] + held.k(cb) # error: [escaping-local] + appended[0](cb) # error: [escaping-local] +``` + ## an opaque callee leaves the block unconstrained When the callee's callback shape cannot be inspected, there is no declaration to read, so the block diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md b/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md index f6fafc92d8..865e4b9c31 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_exceptions.md @@ -161,6 +161,49 @@ def main(): pass ``` +## a stored function still raises what it declares + +a type inferred from a stored value is widened, but a `.by` module keeps a function's own type +rather than widening it to a callable type, which has no `raises` clause. so a call through a +function read back out of a list, a dict, a tuple or an attribute still raises what the function +declares + +```by +def p() -> int raises ValueError: + raise ValueError + +class Holder: + def __init__(self): + self.p = p + +listed = [p] +keyed = {"a": p} +paired = (p, 1) +held = Holder() +appended = [] +appended.append(p) + +def from_list() -> int raises KeyError: + # error: [undeclared-raise] "`from_list` can raise `ValueError`, which its `raises` clause does not include" + return listed[0]() + +def from_dict() -> int raises KeyError: + # error: [undeclared-raise] "`from_dict` can raise `ValueError`, which its `raises` clause does not include" + return keyed["a"]() + +def from_tuple() -> int raises KeyError: + # error: [undeclared-raise] "`from_tuple` can raise `ValueError`, which its `raises` clause does not include" + return paired[0]() + +def from_attribute() -> int raises KeyError: + # error: [undeclared-raise] "`from_attribute` can raise `ValueError`, which its `raises` clause does not include" + return held.p() + +def from_append() -> int raises KeyError: + # error: [undeclared-raise] "`from_append` can raise `ValueError`, which its `raises` clause does not include" + return appended[0]() +``` + ## `raises Never` cannot raise ```by diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md b/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md index bde4de3668..3cf04862b9 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_init_method.md @@ -26,9 +26,7 @@ class A: self.b = str(a) x = A(1) -# `str(a)` constructs, so the attribute's only assignment is exactly a `str` — see -# basedpython/features/exact-construction.md -reveal_type(x.b) # revealed: final str +reveal_type(x.b) # revealed: str ``` ## `let` parameter inside body-bearing `init` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_just_float.md b/crates/ty_python_semantic/resources/mdtest/basedpython_just_float.md index 51493fdea9..2dc7b26d78 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_just_float.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_just_float.md @@ -308,6 +308,89 @@ class Counter: reveal_type(self.label) # revealed: str ``` +## the rest of an inferred type widens as it does without the strict model + +of the numbers, the strict model holds back only the widening of `float` and `complex`. the rest of +the precision an inferred type is widened from goes as it does in a module without it: an attribute +assigned a constructor call is declared with the class rather than the exact `final` type of that +one instance, and one assigned a narrowed value is declared without what the narrowing ruled out, so +either can later hold an instance of a subclass; an attribute assigned a `range` is declared a +`range` whether or not that one was empty; and a covariant type argument keeps the literal it was +inferred from, since nothing reads a different type back out of it + +a function is the one thing a `.by` module widens differently, and not because of its numbers: it +keeps the function's own type rather than widening it to a callable type, because a `.by` function +can carry what no callable type expresses — reified type parameters, `context` parameters, `local` +parameters, a `raises` clause + +```by +class A: ... + +class B(A): ... + +class Out[out T]: + def __init__(self, t: T): ... + +class Holder: + def __init__(self, v: A): + self.a = A() + self.f = len + self.r = range(3) + self.out = Out(1) + if v is not B: + self.narrowed = v + +def replace(h: Holder): + reveal_type(h.a) # revealed: A + reveal_type(h.f) # revealed: def len(obj: Sized, /) -> int + reveal_type(h.out) # revealed: Out[1] + reveal_type(h.narrowed) # revealed: A + h.a = B() + h.narrowed = B() + h.r = range(0) +``` + +## `strict-float` gives a `.py` module the same widening + +a `.py` module that opts into the strict model widens its numbers exactly as the `.by` module above +does, holding back only `float` and `complex`. a function widens to its callable type as it does in +any `.py` module: keeping it is part of what `.by` is, not of its numeric model + +```toml +[analysis] +strict-float = true + +[environment] +python-version = "3.12" +``` + +```py +class A: ... +class B(A): ... + +class Out[T]: + def __init__(self, t: T) -> None: ... + def get(self) -> T: + raise NotImplementedError + +class Holder: + def __init__(self, v: A, x: float) -> None: + self.x = x + self.f = len + self.r = range(3) + self.out = Out(1) + if not isinstance(v, B): + self.narrowed = v + +def replace(h: Holder) -> None: + reveal_type(h.x) # revealed: float + reveal_type(h.f) # revealed: (obj: Sized, /) -> int + reveal_type(h.out) # revealed: Out[Literal[1]] + reveal_type(h.narrowed) # revealed: A + h.narrowed = B() + h.r = range(0) +``` + ## a container's element type, which is what the compiler lays out Reading an element out of a `list[float]` and iterating one both go through the same annotation, so diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md b/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md index 8e5149309a..097face132 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md @@ -325,6 +325,37 @@ def g[T](x: T) -> T: c: (int) -> int = g # ok — erased generic specializes to a plain callable ``` +## a reified generic keeps its type when stored + +a type inferred from a stored value is widened, but a `.by` module keeps a function's own type +rather than widening it to a callable type, which has no slot for the specialization step either. so +a reified generic can still be specialized once it is read back out of a list, a dict, a tuple or an +attribute + +```by +def name[reified T]() -> str: + return T.__name__ + +class Holder: + def __init__(self): + self.f = name + +listed = [name] +keyed = {"a": name} +paired = (name, 1) +held = Holder() +appended = [] +appended.append(name) + +reveal_type(listed) # revealed: list[def name[T]() -> str] +reveal_type(held.f) # revealed: def name[T]() -> str +reveal_type(listed[0][int]()) # revealed: str +reveal_type(keyed["a"][str]()) # revealed: str +reveal_type(paired[0][bytes]()) # revealed: str +reveal_type(held.f[float]()) # revealed: str +reveal_type(appended[0][int]()) # revealed: str +``` + ## overrides must keep the reified interface Specializations flow through the base type — `a.f[int]()` on `a: A` dispatches to the override at diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index ec19af2b04..ed203b3f4d 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -3924,20 +3924,28 @@ impl<'db> Type<'db> { self.apply_type_mapping( db, env, - &TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular), + &TypeMapping::Promote( + PromotionMode::On, + PromotionKind::regular( + NumericPromotion::SpecialCase, + FunctionPromotion::ToCallable, + ), + ), TypeContext::default(), ) } - /// As [`Type::promote`], but honouring the numeric model of the file the type came from. + /// As [`Type::promote`], but honouring the model of the file the type came from. /// /// An *inferred* type is widened through the same promotion that applies the numeric /// special case, so a module that opted out of the special case in its annotations would /// otherwise get it back through inference — `self.x = x` with `x: float` would declare /// the attribute `int | float` however strict the parameter was. /// - /// A `.by` file has that model by definition rather than by configuration, so it takes - /// the strict path whatever `strict-float` says. + /// A `.by` file has that numeric model by definition rather than by configuration, so it + /// takes the strict path whatever `strict-float` says. It also keeps a function's own type + /// ([`FunctionPromotion::Keep`]), which is a property of the language rather than of its + /// numbers, so `strict-float` has no say in it. #[must_use] fn promote_in( self, @@ -3945,16 +3953,24 @@ impl<'db> Type<'db> { env: &ProgramEnvironment<'db>, file: ruff_db::files::File, ) -> Type<'db> { - let kind = - if file.source_type(db).is_basedpython() || db.analysis_settings(file).strict_float { - PromotionKind::RegularStrictNumeric - } else { - PromotionKind::Regular - }; + let basedpython = file.source_type(db).is_basedpython(); + let numeric = if basedpython || db.analysis_settings(file).strict_float { + NumericPromotion::Exact + } else { + NumericPromotion::SpecialCase + }; + let functions = if basedpython { + FunctionPromotion::Keep + } else { + FunctionPromotion::ToCallable + }; self.apply_type_mapping( db, env, - &TypeMapping::Promote(PromotionMode::On, kind), + &TypeMapping::Promote( + PromotionMode::On, + PromotionKind::regular(numeric, functions), + ), TypeContext::default(), ) } @@ -10306,7 +10322,10 @@ impl<'db> Type<'db> { // callable form. TypeMapping::Promote( PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularKeepingLiterals, + PromotionKind::Regular { + functions: FunctionPromotion::ToCallable, + .. + }, ) => mapped.promote_impl(db, visitor.env), _ => mapped, } @@ -10333,14 +10352,15 @@ impl<'db> Type<'db> { ), )), - // `RegularStrictNumeric` is deliberately absent: it promotes everything - // else and leaves `float` and `complex` exact Type::NominalInstance(instance) if matches!( type_mapping, TypeMapping::Promote( PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularKeepingLiterals + PromotionKind::Regular { + numeric: NumericPromotion::SpecialCase, + .. + } ) ) => { @@ -10493,10 +10513,7 @@ impl<'db> Type<'db> { // so we don't preserve them here when regular promotion is enabled. if !matches!( type_mapping, - TypeMapping::Promote( - PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularKeepingLiterals - ) + TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular { .. }) ) { for negative in intersection.negative(db) { builder.add_negative_in_place(negative.apply_type_mapping_impl( @@ -10568,7 +10585,13 @@ impl<'db> Type<'db> { if restricted.modifier(db) == TypeModifier::Final && matches!( type_mapping, - TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular) + TypeMapping::Promote( + PromotionMode::On, + PromotionKind::Regular { + literals: LiteralPromotion::Widen, + .. + } + ) ) { return inner; @@ -10700,11 +10723,18 @@ impl<'db> Type<'db> { ) => self, TypeMapping::Promote( PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularStrictNumeric, + PromotionKind::Regular { + literals: LiteralPromotion::Widen, + .. + }, ) => self.promote_impl(db, env), - TypeMapping::Promote(PromotionMode::On, PromotionKind::RegularKeepingLiterals) => { - self.promote_impl_keeping_literals(db, env) - } + TypeMapping::Promote( + PromotionMode::On, + PromotionKind::Regular { + literals: LiteralPromotion::Keep, + .. + }, + ) => self.promote_impl_keeping_literals(db, env), }, Type::Dynamic(_) => match type_mapping { @@ -11860,19 +11890,13 @@ impl PromotionMode { #[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] pub enum PromotionKind { - /// Default promotion behaviour: recurse into nested types - Regular, - /// Like [`PromotionKind::Regular`], but literal values are left as they are. - /// - /// This is what a covariant type argument gets: widening it is sound, but nothing reads a - /// different type back out of that position, so the literal it was inferred from stands. - RegularKeepingLiterals, - /// Like [`PromotionKind::Regular`], but `float` and `complex` are left exact. - /// - /// Widening a literal is one thing; widening `float` to `int | float` is the typing spec's - /// numeric special case, and a module that set `strict-float` has opted out of it. The two - /// travel together in [`PromotionKind::Regular`] because they are usually wanted together. - RegularStrictNumeric, + /// Default promotion behaviour: recurse into nested types. what it does with a literal + /// and with `float` / `complex` are two independent choices + Regular { + literals: LiteralPromotion, + numeric: NumericPromotion, + functions: FunctionPromotion, + }, /// Promote class literals recursively without promoting other literal types. ClassLiteralsOnly, /// Singleton-only promotion recursively descends through nominal instances @@ -11880,6 +11904,54 @@ pub enum PromotionKind { SingletonsOnly, } +impl PromotionKind { + /// regular promotion that widens literals, under the given numeric model and treatment of + /// functions + const fn regular(numeric: NumericPromotion, functions: FunctionPromotion) -> Self { + Self::Regular { + literals: LiteralPromotion::Widen, + numeric, + functions, + } + } +} + +/// what regular promotion does with a literal value +#[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] +pub enum LiteralPromotion { + /// widen it to the class it is an instance of (`Literal[1]` → `int`), and drop the exactness + /// a constructor call infers (`final A` → `A`), which is the same kind of precision + Widen, + /// leave it as it is. this is what a covariant type argument gets: widening it is sound, + /// but nothing reads a different type back out of that position, so the literal it was + /// inferred from stands + Keep, +} + +/// what regular promotion does with `float` and `complex` +#[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] +pub enum NumericPromotion { + /// the typing spec's numeric special case: `float` widens to `int | float` and `complex` + /// to `int | float | complex` + SpecialCase, + /// `float` and `complex` are left exact. a `.by` module has this model by definition, and + /// a module that set `strict-float` has opted into it + Exact, +} + +/// what regular promotion does with a function +#[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] +pub enum FunctionPromotion { + /// widen it to its callable type (`def f() -> int` → `() -> int`) + ToCallable, + /// leave it as it is. a function declared in a `.by` module can carry what no callable + /// type expresses — reified type parameters a call specializes by subscript, `context` + /// parameters a call fills, `local` / `once` parameters, a `raises` clause — and a call + /// through the stored value relies on them, so a `.by` module keeps the function's own + /// type + Keep, +} + /// Returns the [`ClassLiteral`] that "owns" a `Self` typevar (i.e., the class from its upper bound). fn self_typevar_owner_class_literal<'db>( db: &'db dyn Db, diff --git a/crates/ty_python_semantic/src/types/generics.rs b/crates/ty_python_semantic/src/types/generics.rs index 13e0c31b1b..e9df8bab33 100644 --- a/crates/ty_python_semantic/src/types/generics.rs +++ b/crates/ty_python_semantic/src/types/generics.rs @@ -38,10 +38,10 @@ use crate::types::visitor::{ use crate::types::{ ApplyTypeMappingVisitor, BindingContext, BoundTypeVarInstance, CallableType, CallableTypes, ClassLiteral, ErrorContext, FindLegacyTypeVarsVisitor, IntersectionType, KnownClass, - KnownInstanceType, MaterializationKind, PromotionKind, PromotionMode, SubclassOfInner, Type, - TypeAliasType, TypeContext, TypeMapping, TypeRecursionContext, TypeVarBoundOrConstraints, - TypeVarKind, TypeVarVariance, UnionAccumulator, UnionType, binding_type, - infer_definition_types, inferred_declaration, + KnownInstanceType, LiteralPromotion, MaterializationKind, PromotionKind, PromotionMode, + SubclassOfInner, Type, TypeAliasType, TypeContext, TypeMapping, TypeRecursionContext, + TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, UnionAccumulator, UnionType, + binding_type, infer_definition_types, inferred_declaration, }; use crate::{Db, FxIndexMap, FxOrderMap, FxOrderSet}; use ty_python_core::definition::{Definition, DefinitionKind}; @@ -1759,20 +1759,30 @@ impl<'db> Specialization<'db> { // reads a different type back out of it, so widening its literals only loses // precision. (an invariant or contravariant argument keeps its literals too — the // arm below flips promotion off, because widening it would be unsound) - (variance, TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular)) - if variance.is_covariant() => - { - ty.apply_type_mapping_impl( - db, - env, - &TypeMapping::Promote( - PromotionMode::On, - PromotionKind::RegularKeepingLiterals, - ), - tcx, - visitor, - ) - } + ( + variance, + TypeMapping::Promote( + PromotionMode::On, + PromotionKind::Regular { + literals: LiteralPromotion::Widen, + numeric, + functions, + }, + ), + ) if variance.is_covariant() => ty.apply_type_mapping_impl( + db, + env, + &TypeMapping::Promote( + PromotionMode::On, + PromotionKind::Regular { + literals: LiteralPromotion::Keep, + numeric: *numeric, + functions: *functions, + }, + ), + tcx, + visitor, + ), (variance, _) if variance.is_covariant() => { ty.apply_type_mapping_impl(db, env, type_mapping, tcx, visitor) } diff --git a/crates/ty_python_semantic/src/types/known_instance.rs b/crates/ty_python_semantic/src/types/known_instance.rs index 403d63e0d3..845b470b29 100644 --- a/crates/ty_python_semantic/src/types/known_instance.rs +++ b/crates/ty_python_semantic/src/types/known_instance.rs @@ -519,10 +519,9 @@ impl<'db> KnownInstanceType<'db> { )) } KnownInstanceType::Range { .. } => match type_mapping { - TypeMapping::Promote( - PromotionMode::On, - PromotionKind::Regular | PromotionKind::RegularKeepingLiterals, - ) => self.instance_fallback(db, env), + TypeMapping::Promote(PromotionMode::On, PromotionKind::Regular { .. }) => { + self.instance_fallback(db, env) + } _ => Type::KnownInstance(self), }, KnownInstanceType::FunctoolsPartialCall(partial) => { diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 597a27238b..0628430ff9 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -203,7 +203,7 @@ fn rendered_type_arguments<'db>( let ty = parameter .value .ok_or_else(|| ReifiedInferenceError::Unsolved(parameter.name.clone()))?; - let promoted = ty.promote(db, env); + let promoted = ty.promote_in(db, env, file); parameter .kind .spelling(db, env, file, promoted) @@ -571,10 +571,11 @@ fn spell_specialization_arguments<'db>( if matches!(kind, ParameterKind::KeywordPack) { return Err(ReifiedInferenceError::NoBinding); } - // an argument is promoted one by one: a covariant parameter keeps the - // literal type it was inferred from, and only a class object can be - // written at runtime - let promoted = argument.promote(db, env); + // a covariant parameter keeps the literal type it was inferred from + // (`B[Literal[1]]`), and a literal has no runtime spelling, so each argument + // is promoted on its own, outside the class's variance, to the class it is + // an instance of — under the numeric model of the module it is written into + let promoted = argument.promote_in(db, env, file); // an unsolved parameter reads as `Never`, and one solved from something // the checker cannot see reads as `Unknown`. neither is an argument // anybody wrote down, so both say the same thing: name one @@ -612,7 +613,7 @@ fn constructor_specialization<'db>( class_literal: ClassLiteral<'db>, constructed: Type<'db>, ) -> Result> { - let Type::NominalInstance(instance) = constructed.promote(db, env) else { + let Type::NominalInstance(instance) = constructed.promote_in(db, env, file) else { return Err(ReifiedInferenceError::NoBinding); }; let ClassType::Generic(alias) = instance.class(db, env) else { From b27411d911d4f5754023780e76a9a8453549a498 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:06:14 +1000 Subject: [PATCH 14/47] =?UTF-8?q?transpile=20a=20checked=20cast=20or=20a?= =?UTF-8?q?=20type=20test=20whose=20target=20another=20lowering=20rewrites?= =?UTF-8?q?=20where=20it=20is=20written=20=E2=80=94=20`float`,=20`int=3F`,?= =?UTF-8?q?=20a=20literal,=20a=20tuple=20type,=20`typeof`,=20a=20callable?= =?UTF-8?q?=20type,=20`dynamic`,=20a=20template=20literal,=20an=20inline?= =?UTF-8?q?=20typed=20dict=20or=20named=20tuple,=20a=20symbolic=20type=20o?= =?UTF-8?q?peration=20=E2=80=94=20instead=20of=20refusing=20it=20as=20a=20?= =?UTF-8?q?transform=20conflict:=20the=20check=20is=20built=20from=20the?= =?UTF-8?q?=20checker's=20reading=20of=20the=20target,=20or=20settled=20wi?= =?UTF-8?q?thout=20it=20where=20the=20operand=20cannot=20hold=20the=20targ?= =?UTF-8?q?et,=20so=20`def=20f(x:=20int)=20->=20float:=20return=20x=20cast?= =?UTF-8?q?!=20float`=20now=20builds=20and=20raises=20at=20runtime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/transforms/ast_driver.rs | 153 +++++++++++++++++ .../src/transforms/checked_cast.rs | 23 ++- .../src/transforms/parametric_is.rs | 12 +- .../src/transforms/template_type.rs | 4 + .../src/transforms/typed_dict_literal.rs | 4 + .../tests/parametric_is_runtime.rs | 154 ++++++++++++++++++ 6 files changed, 345 insertions(+), 5 deletions(-) diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index fb3b72e521..c65b276a23 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -217,7 +217,9 @@ pub(crate) enum Lowering { RuntimeUnion, StatementExpression, SymbolicTypeOp, + TemplateType, TupleLiteralType, + TypedDictLiteral, Typeof, Unpack, VarianceStrip, @@ -2847,6 +2849,156 @@ mod subsumed_lowerings { ] } + /// a checked cast and a type test build their check from the checker's reading of the + /// target, so a lowering of the target as written is not what they print: the check + /// spells what the target means, or, where the operand's type cannot hold the target, + /// is settled without it + fn type_test_target_cases() -> Vec { + let cast = |operand: &str, target: &str| { + format!("y = 1\n\ndef f(x: {operand}) -> object:\n return x cast! {target}\n") + }; + let test = |operand: &str, target: &str| { + format!("y = 1\n\ndef f(x: {operand}) -> bool:\n return x is {target}\n") + }; + let never = "_checked_cast_pred(x, lambda _by_cast_value: False)"; + let mut cases = vec![ + case( + "CheckedCastPass", + &[Lowering::AnonNamedTuple], + &cast("str", "(a: int, b: str)"), + &[never], + &["(a: int"], + ), + case( + "CheckedCastPass", + &[Lowering::Callable], + &cast("str", "list[(int) -> str]"), + &[never], + &["(int) ->"], + ), + case( + "CheckedCastPass", + &[Lowering::DynamicKeyword], + &cast("object", "list[dynamic]"), + &["_checked_cast(x, list)"], + &["dynamic"], + ), + case( + "CheckedCastPass", + &[Lowering::FloatConst], + &cast("str", "float.inf"), + &[never], + &["float.inf"], + ), + case( + "CheckedCastPass", + &[Lowering::JustFloat], + &cast("int", "float"), + &[never], + &["cast! float"], + ), + case( + "CheckedCastPass", + &[Lowering::JustFloat], + &cast("object", "list[float]"), + &["_parametric_is_lenient(_by_cast_value, list[float], (0,))"], + &["list[JustFloat]"], + ), + case( + "CheckedCastPass", + &[Lowering::LiteralType], + &cast("str", "1"), + &[never], + &["Literal[1]"], + ), + case( + "CheckedCastPass", + &[Lowering::OptionalType], + &cast("object", "int?"), + &["(isinstance(_by_cast_value, int) or _by_cast_value is None)"], + &["int?"], + ), + case( + "CheckedCastPass", + &[Lowering::SymbolicTypeOp], + &cast("object", "typeof y + 1"), + &["_by_cast_value == 2)"], + &["typeof"], + ), + case( + "CheckedCastPass", + &[Lowering::TemplateType], + &cast("object", "f\"a{int}b\""), + &["_by_pattern_is(_by_cast_value, \"a(?:0|-?[1-9][0-9]*)b\")"], + &["f\"a{int}b\""], + ), + case( + "CheckedCastPass", + &[Lowering::TupleLiteralType], + &cast("object", "(int, str)"), + &["_parametric_is_lenient(_by_cast_value, tuple[int, str], (1,))"], + &["cast! (int, str)"], + ), + case( + "CheckedCastPass", + &[Lowering::TypedDictLiteral], + &cast("object", "{\"a\": int}"), + &["_checked_cast(x, dict)"], + &["{\"a\": int}"], + ), + case( + "CheckedCastPass", + &[Lowering::Typeof], + &cast("object", "typeof y"), + &["_by_cast_value == 1)"], + &["typeof"], + ), + ]; + cases.extend([ + case( + "ParametricIsPass", + &[Lowering::AnonNamedTuple], + &test("str", "(a: int, b: str)"), + &["return False"], + &["(a: int"], + ), + case( + "ParametricIsPass", + &[Lowering::Callable], + &test("str", "list[(int) -> str]"), + &["return False"], + &["(int) ->"], + ), + case( + "ParametricIsPass", + &[Lowering::DynamicKeyword], + &test("object", "list[dynamic]"), + &["isinstance(x, list)"], + &["dynamic"], + ), + case( + "ParametricIsPass", + &[Lowering::JustFloat], + &test("object", "list[float]"), + &["_parametric_is(x, list[float], (0,))"], + &["list[JustFloat]"], + ), + case( + "ParametricIsPass", + &[Lowering::Typeof], + &test("object", "typeof y"), + &["x == 1)"], + &["typeof"], + ), + ]); + cases + } + + #[test] + fn a_type_test_target_is_written_as_what_it_means() { + check(&type_test_target_cases(), PythonVersion::PY312); + } + #[test] fn a_type_expression_a_pass_moves_is_written_lowered() { check(&type_expression_cases(), PythonVersion::PY310); @@ -2868,6 +3020,7 @@ mod subsumed_lowerings { let cases: Vec = type_expression_cases() .into_iter() .chain(whole_construct_cases()) + .chain(type_test_target_cases()) .collect(); let mut failures = Vec::new(); diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 436b2e3390..6f024cbf29 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -252,10 +252,27 @@ pub(crate) struct CheckedCastPass<'src> { } impl TypeAwarePass for CheckedCastPass<'_> { - /// a checked cast prints the type it checks against as the runtime check it builds, - /// the emitted name of a `private` class included + /// a checked cast builds its runtime check from the checker's reading of the type it + /// checks against, and leaves the type out where the checker settles the check without + /// it (an operand whose type cannot hold the target), so it writes these lowerings of the + /// type as written itself. where it falls back to the type as written, the lowerings + /// inside pass through with it. the emitted name of a `private` class is included fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { - &[super::ast_driver::Lowering::VisibilityRename] + &[ + super::ast_driver::Lowering::AnonNamedTuple, + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::FloatConst, + super::ast_driver::Lowering::JustFloat, + super::ast_driver::Lowering::LiteralType, + super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::SymbolicTypeOp, + super::ast_driver::Lowering::TemplateType, + super::ast_driver::Lowering::TupleLiteralType, + super::ast_driver::Lowering::TypedDictLiteral, + super::ast_driver::Lowering::Typeof, + super::ast_driver::Lowering::VisibilityRename, + ] } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index 8008427d43..1a6bc6ea6f 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -747,12 +747,20 @@ impl TypeAwarePass for ParametricIsPass<'_> { Some(super::ast_driver::Lowering::ParametricIs) } - /// the type it tests against is printed as the runtime probe it builds, literals, the - /// `None` of an optional and the emitted name of a `private` class included + /// the runtime probe is built from the checker's reading of the type it tests against, + /// and the type is left out where the checker settles the test without it, so it writes + /// these lowerings of the type as written itself — literals, the `None` of an optional + /// and the emitted name of a `private` class among them. where the probe passes the + /// type as written through, the lowerings inside pass through with it fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { &[ + super::ast_driver::Lowering::AnonNamedTuple, + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::JustFloat, super::ast_driver::Lowering::LiteralType, super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::Typeof, super::ast_driver::Lowering::VisibilityRename, ] } diff --git a/crates/by_transforms/src/transforms/template_type.rs b/crates/by_transforms/src/transforms/template_type.rs index 07038f4ae3..a6780c97a5 100644 --- a/crates/by_transforms/src/transforms/template_type.rs +++ b/crates/by_transforms/src/transforms/template_type.rs @@ -45,6 +45,10 @@ pub(crate) struct TemplateTypePass<'src> { } impl TypeAwarePass for TemplateTypePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::TemplateType) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = TemplateType { types, diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index 8e8dae60d4..c388373d6b 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -484,6 +484,10 @@ fn flatten_bitand(expr: &Expr) -> Vec<&Expr> { } impl AstPass for TypedDictLiteralPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::TypedDictLiteral) + } + /// a typed dict type it replaces moves into a class, whose field types the shared /// type-expression lowerer prints, a `typeof` among them fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { diff --git a/crates/by_transforms/tests/parametric_is_runtime.rs b/crates/by_transforms/tests/parametric_is_runtime.rs index b4505a988d..c9fe2c9ecb 100644 --- a/crates/by_transforms/tests/parametric_is_runtime.rs +++ b/crates/by_transforms/tests/parametric_is_runtime.rs @@ -496,6 +496,147 @@ assert Tagged(1.5).argument() === float, f"a float literal specializes as {Tagge print("ok") "#; +/// basedpython whose module-level `assert`s run checked casts and type tests against targets +/// another lowering rewrites where they are written — `float`, `int?`, a literal, a tuple +/// type, `typeof`, a callable type, `dynamic` and the rest. the check is built from the +/// checker's reading of the target, so it tests what the target means rather than how it +/// is spelled; and where the operand's type cannot hold the target, the checker settles the +/// check without the target at all +const LOWERED_TARGETS_PROGRAM: &str = r#" +y = 1 + +def raises(f: () -> object) -> bool: + try: + f() + except TypeError: + return True + return False + +# the operand's type cannot hold the target, so the checker settles the cast without the +# target: it never succeeds +def int_to_float(x: int) -> float: + return x cast! float + +def int_to_float_or_none(x: int) -> float?: + return x cast? float + +assert raises(lambda: int_to_float(3)), "an int is not a float" +assert int_to_float_or_none(3) is None, "an int is not a float" + +def str_to_one(x: str) -> object: + return x cast? 1 + +def str_to_pair(x: str) -> object: + return x cast? (int, str) + +def str_to_named(x: str) -> object: + return x cast? (a: int, b: str) + +def str_to_inf(x: str) -> object: + return x cast? float.inf + +def str_to_floats(x: str) -> object: + return x cast? tuple[float, ...] + +assert str_to_one("1") is None +assert str_to_pair("a") is None +assert str_to_named("a") is None +assert str_to_inf("inf") is None +assert str_to_floats("a") is None + +# the operand may hold the target, so the check is built from the checker's reading of it +def to_float(x: object) -> float: + return x cast! float + +assert to_float(3.5) == 3.5 +assert raises(lambda: to_float(3)), "an int is not a float" + +def to_optional(x: object) -> int?: + return x cast! int? + +assert to_optional(None) is None +assert to_optional(1) == 1 +assert raises(lambda: to_optional("a")) + +class FloatList(list[float]): ... +class IntList(list[int]): ... + +def to_pair(x: object) -> object: + return x cast? (int, str) + +def to_floats(x: object) -> object: + return x cast? list[float] + +def to_y(x: object) -> object: + return x cast? typeof y + +def to_y_plus_one(x: object) -> object: + return x cast? typeof y + 1 + +def to_float_class(x: object) -> object: + return x cast? type[float] + +def to_pattern(x: object) -> object: + return x cast? f"a{int}b" + +def to_dynamic_list(x: object) -> object: + return x cast? list[dynamic] + +def to_callbacks(x: object) -> object: + return x cast? list[(int) -> str] + +assert to_pair((1, "a")) is not None +assert to_pair([1, "a"]) is None +assert to_floats(FloatList()) is not None +assert to_floats(IntList()) is None +assert to_y(1) == 1 +assert to_y(2) is None +assert to_y(True) is None +assert to_y_plus_one(2) == 2 +assert to_y_plus_one(1) is None +assert to_float_class(float) === float +assert to_float_class(int) is None +assert to_pattern("a1b") == "a1b" +assert to_pattern("a1c") is None +assert to_dynamic_list([1]) == [1] +assert to_dynamic_list((1,)) is None +assert to_callbacks([str]) is not None +assert to_callbacks("s") is None + +# a type test is built the same way, and `float` is just `float` there too +def is_float(v: object) -> bool: + return v is float + +assert is_float(3.5) is True +assert is_float(3) is False, "an int is not a float" + +def is_floats(v: object) -> bool: + return v is list[float] + +def is_y(v: object) -> bool: + return v is typeof y + +def is_callbacks(v: object) -> bool: + return v is list[(int) -> str] + +def is_dynamic_list(v: object) -> bool: + return v is list[dynamic] + +def str_is_named(v: str) -> bool: + return v is (a: int, b: str) + +assert is_floats(FloatList()) is True +assert is_floats(IntList()) is False +assert is_y(1) is True +assert is_y(2) is False +assert is_callbacks([]) is False +assert is_dynamic_list([1]) is True +assert is_dynamic_list((1,)) is False +assert str_is_named("a") is False + +print("ok") +"#; + const PEP695_PROBE: &str = "type X[T] = T"; /// pep 696 defaults in native syntax plus the `has_default()` accessor the probe @@ -604,3 +745,16 @@ fn a_check_written_from_a_float_type_checks_just_float() { }; run_program(&python, JUST_FLOAT_PROGRAM); } + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_target_another_lowering_rewrites_checks_correctly_at_runtime() { + let Some(python) = python_supporting(PEP695_PROBE) else { + eprintln!("skipping lowered-target runtime test: no PEP 695-capable interpreter found"); + return; + }; + run_program(&python, LOWERED_TARGETS_PROGRAM); +} From aa5099c39004e59905aadc44982ab9d10f4b0f69 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:01:53 +1000 Subject: [PATCH 15/47] build a union of methods bound to different literals without comparing every pair of them: methods that differ only in a receiver holding one value (`"a".lower`, `"a".startswith`, an int or enum member's method) are grouped by receiver like the literals themselves, so calling a method on a large enum's `name` (`kind.name.lower()` over 2000 members) is checked and transpiled in linear time instead of quadratic, with a union of the same members as before, possibly listed in a different order --- .../resources/mdtest/union_types.md | 122 ++++++++ crates/ty_python_semantic/src/types.rs | 11 +- .../ty_python_semantic/src/types/literal.rs | 21 ++ .../ty_python_semantic/src/types/relation.rs | 5 + .../src/types/set_theoretic/builder.rs | 296 +++++++++++++++++- 5 files changed, 436 insertions(+), 19 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/union_types.md b/crates/ty_python_semantic/resources/mdtest/union_types.md index 0ea3eddaf2..4f673fb78a 100644 --- a/crates/ty_python_semantic/resources/mdtest/union_types.md +++ b/crates/ty_python_semantic/resources/mdtest/union_types.md @@ -150,6 +150,128 @@ def _( reveal_type(u2) # revealed: Literal[Color.RED] ``` +## Methods bound to literals + +```toml +[environment] +python-version = "3.12" +``` + +looking up a method on a union of literals gives a union of bound methods, one for each literal. a +method bound to `"a"` is not redundant with the same method bound to `"b"`, so each one is kept +along with its receiver + +```py +from enum import Enum +from typing import Literal + +class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + +def _(s: Literal["a", "b"], n: Literal[1, 2], color: Literal[Color.RED, Color.GREEN]): + # revealed: (Overload[def lower() -> LiteralString, def lower() -> str]) | (Overload[def lower() -> LiteralString, def lower() -> str]) + reveal_type(s.lower) + reveal_type(s.lower.__self__) # revealed: Literal["a", "b"] + reveal_type(s.lower()) # revealed: LiteralString + # revealed: () | () + reveal_type(s.startswith) + reveal_type(s.startswith("a")) # revealed: bool + reveal_type(n.bit_length.__self__) # revealed: Literal[1, 2] + reveal_type(color.__eq__.__self__) # revealed: Literal[Color.RED, Color.GREEN] +``` + +an enum's `name` is the union of its members' names, so a method called on it is bound to each name + +```py +def _(color: Color): + reveal_type(color.name.lower.__self__) # revealed: Literal["RED", "GREEN", "BLUE"] + reveal_type(color.name.lower()) # revealed: LiteralString +``` + +the same method bound to the same literal appears once. when one receiver is the promotable form of +the literal (the `"a"` written below) and the other is not (the declared `a`), the union keeps the +method bound to the unpromotable one, as a union of the two literals keeps the unpromotable literal. +promotion shows in the inferred list type + +```py +def _(flag: bool, a: Literal["a"]): + reveal_type(a.lower if flag else a.lower) # revealed: Overload[() -> LiteralString, () -> str] + reveal_type(a.lower if flag else "a".lower) # revealed: Overload[() -> LiteralString, () -> str] + reveal_type("a".lower if flag else a.lower) # revealed: Overload[() -> LiteralString, () -> str] + reveal_type(["a"]) # revealed: list[str] + reveal_type([(a.lower if flag else "a".lower).__self__]) # revealed: list[Literal["a"]] + reveal_type([("a".lower if flag else a.lower).__self__]) # revealed: list[Literal["a"]] +``` + +a type that holds every one of the bound methods subsumes them, whichever side of the union it is +on. a method bound to a `LiteralString` is such a type for the same method bound to a string literal + +```py +from typing import Callable +from typing_extensions import LiteralString + +def _(flag: bool, s: Literal["a", "b"], f: Callable[[], str], x: LiteralString): + reveal_type(s.lower if flag else f) # revealed: () -> str + reveal_type(f if flag else s.lower) # revealed: () -> str + reveal_type(s.lower if flag else x.lower) # revealed: Overload[() -> LiteralString, () -> str] + reveal_type(x.lower if flag else s.lower) # revealed: Overload[() -> LiteralString, () -> str] +``` + +literals of different kinds can share a method: `True`, `1` and a member of an `IntEnum` or +`IntFlag` all have `int.bit_length`, and a `StrEnum` member has the methods of `str`. a literal of +one kind is never a literal of another, even where the two compare equal like `True` and `1`, so +neither method stands for the other and both are kept + +```py +from enum import IntEnum, IntFlag, StrEnum + +class Code(IntEnum): + ONE = 1 + TWO = 2 + +class Bits(IntFlag): + ONE = 1 + TWO = 2 + +class Name(StrEnum): + A = "a" + B = "b" + +def _(flag: bool, b: Literal[True], i: Literal[1], a: Literal["a"]): + # revealed: (bound method Literal[True].bit_length() -> int) | (bound method Literal[1].bit_length() -> int) + reveal_type(b.bit_length if flag else i.bit_length) + reveal_type((b.bit_length if flag else i.bit_length).__self__) # revealed: Literal[True, 1] + reveal_type((i.bit_length if flag else b.bit_length).__self__) # revealed: Literal[1, True] + # revealed: (bound method Literal[Code.ONE].bit_length() -> int) | (bound method Literal[1].bit_length() -> int) + reveal_type(Code.ONE.bit_length if flag else i.bit_length) + reveal_type((Code.ONE.bit_length if flag else i.bit_length).__self__) # revealed: Literal[Code.ONE, 1] + reveal_type((i.bit_length if flag else Code.ONE.bit_length).__self__) # revealed: Literal[1, Code.ONE] + reveal_type((Bits.ONE.bit_length if flag else i.bit_length).__self__) # revealed: Literal[Bits.ONE, 1] + reveal_type((i.bit_length if flag else Bits.ONE.bit_length).__self__) # revealed: Literal[1, Bits.ONE] + # revealed: (bound method Literal[Name.A].lower() -> str) | (Overload[() -> LiteralString, () -> str]) + reveal_type(Name.A.lower if flag else a.lower) + reveal_type((Name.A.lower if flag else a.lower).__self__) # revealed: Literal[Name.A, "a"] + reveal_type((a.lower if flag else Name.A.lower).__self__) # revealed: Literal["a", Name.A] +``` + +## Methods bound to template literals + +a template literal holds many strings, like `LiteralString`. a method bound to `f"v{int}"` subsumes +the same method bound to `"v1"` or `"v2"`, but not the one bound to `"x"`, which the template does +not produce + +```by +def _(flag: bool, s: "v1" | "v2" | "x", t: f"v{int}"): + # revealed: (Overload[() -> LiteralString, () -> str]) | (bound method f"v{int}".lower() -> str) + reveal_type(s.lower if flag else t.lower) + # revealed: (bound method f"v{int}".lower() -> str) | (Overload[() -> LiteralString, () -> str]) + reveal_type(t.lower if flag else s.lower) + reveal_type((s.lower if flag else t.lower).__self__) # revealed: "x" | f"v{int}" + reveal_type((t.lower if flag else s.lower).__self__) # revealed: f"v{int}" | "x" +``` + ## Do not erase `Unknown` ```py diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index ed203b3f4d..3bc25d653e 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -3537,16 +3537,7 @@ impl<'db> Type<'db> { .elements(db) .iter() .all(|ty| ty.is_literal_or_union_of_literals(db, env)), - Type::LiteralValue(literal) => match literal.kind() { - LiteralValueTypeKind::String(_) - | LiteralValueTypeKind::Bytes(_) - | LiteralValueTypeKind::Int(_) - | LiteralValueTypeKind::Bool(_) - | LiteralValueTypeKind::Enum(_) - | LiteralValueTypeKind::Float(_) - | LiteralValueTypeKind::Complex(_) => true, - LiteralValueTypeKind::LiteralString | LiteralValueTypeKind::Template(_) => false, - }, + Type::LiteralValue(literal) => literal.kind().holds_one_value(), Type::NominalInstance(_) => { self.is_none(db) || self.is_bool(db) || self.is_enum(db, env) } diff --git a/crates/ty_python_semantic/src/types/literal.rs b/crates/ty_python_semantic/src/types/literal.rs index 1df32bd228..e49ce9fee2 100644 --- a/crates/ty_python_semantic/src/types/literal.rs +++ b/crates/ty_python_semantic/src/types/literal.rs @@ -101,6 +101,27 @@ pub(crate) enum LiteralValueTypeKind<'db> { Template(TemplateLiteralType<'db>), } +impl LiteralValueTypeKind<'_> { + /// basedpython: whether a literal of this kind holds exactly one value + /// + /// two literals that each hold one value are related only when their kinds are equal (the + /// last `LiteralValue` arms of `TypeRelationChecker::check_type_pair`). the union builder + /// relies on that to group methods bound to different literals without comparing them, and + /// `one_valued_literals_are_related_only_by_kind` pins it + pub(crate) const fn holds_one_value(self) -> bool { + match self { + LiteralValueTypeKind::Int(_) + | LiteralValueTypeKind::Bool(_) + | LiteralValueTypeKind::String(_) + | LiteralValueTypeKind::Enum(_) + | LiteralValueTypeKind::Bytes(_) + | LiteralValueTypeKind::Float(_) + | LiteralValueTypeKind::Complex(_) => true, + LiteralValueTypeKind::LiteralString | LiteralValueTypeKind::Template(_) => false, + } + } +} + impl<'db> LiteralValueType<'db> { pub(crate) fn new( kind: impl Into>, diff --git a/crates/ty_python_semantic/src/types/relation.rs b/crates/ty_python_semantic/src/types/relation.rs index 5179c54e5d..5f4bb9ef17 100644 --- a/crates/ty_python_semantic/src/types/relation.rs +++ b/crates/ty_python_semantic/src/types/relation.rs @@ -2522,6 +2522,11 @@ impl<'a, 'c, 'db> TypeRelationChecker<'a, 'c, 'db> { // For union simplification, we want to preserve the unpromotable form of a literal value, // and so redundancy is not symmetric. + // + // basedpython: this arm and the next one say that two literals holding one value each + // (`LiteralValueTypeKind::holds_one_value`) are related only when their kinds are equal. + // the union builder's `LiteralReceiverMethod` groups rely on that, and + // `one_valued_literals_are_related_only_by_kind` pins it (Type::LiteralValue(source), Type::LiteralValue(target)) if matches!(self.relation, TypeRelation::Redundancy { pure: false }) => { diff --git a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs index c976ecf25f..047ff4b7b3 100644 --- a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs +++ b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs @@ -38,19 +38,28 @@ //! reason, we group literal types in `UnionBuilder`. Since every different string literal type //! shares exactly the same possible super-types, and none of them are subtypes of each other //! (unless exactly the same literal type), we can avoid many unnecessary redundancy checks. +//! +//! basedpython: the same holds one step removed, for a method bound to a literal such as +//! `"a".lower`. looking up an attribute on a large literal union gives a union of as many bound +//! methods, one per receiver (`kind.name.lower` for an enum `kind`, whose `name` is a string +//! literal per member). two methods that differ only in their receiver are redundant with each +//! other only if their receivers are, which two different literals never are, so these methods +//! are grouped by everything but their receiver; see [`LiteralReceiverMethod`] use std::hint::cold_path; use super::RecursivelyDefined; use super::generic_gradual_intersections::{GenericIntersection, generic_gradual_intersection}; use crate::types::enums::EnumComplement; +use crate::types::function::FunctionType; use crate::types::regex; use crate::types::set_theoretic::expand_intersection_typevars_and_newtypes; use crate::types::visitor::any_over_type; use crate::types::{ - BytesLiteralType, ClassLiteral, EnumLiteralType, IntersectionType, KnownClass, - KnownInstanceType, LiteralValueType, LiteralValueTypeKind, NegativeIntersectionElements, - StringLiteralType, SubclassOfType, Type, TypePair, TypeVarBoundOrConstraints, UnionType, + BytesLiteralType, ClassLiteral, EnumLiteralType, IntersectionType, KnownBoundMethodType, + KnownClass, KnownInstanceType, LiteralValueType, LiteralValueTypeKind, + NegativeIntersectionElements, StringLiteralType, SubclassOfType, Type, TypePair, + TypeVarBoundOrConstraints, UnionType, }; use crate::{Db, FxOrderMap, FxOrderSet, ProgramEnvironment}; use rustc_hash::FxHashSet; @@ -309,6 +318,42 @@ impl<'db> Type<'db> { } } +/// basedpython: a method bound to a literal that holds exactly one value, without its receiver +/// +/// two bound methods with the same [`LiteralReceiverMethod`] are related only through their +/// receivers: a bound method is a subtype of another only if its receiver is a subtype of the +/// other's (see `check_bound_method_pair` and `check_known_bound_method_pair`). two different +/// literals that each hold one value are never redundant with each other (see +/// `LiteralValueTypeKind::holds_one_value`), so neither are two such methods with different +/// receivers, and the union builder compares them by receiver instead of by relation +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum LiteralReceiverMethod<'db> { + /// a function bound to its receiver, as in `"a".lower` + Function(FunctionType<'db>), + /// `str.startswith` bound to a string literal, as in `"a".startswith` + StrStartswith, +} + +impl<'db> LiteralReceiverMethod<'db> { + /// split `ty` into the method and the literal it is bound to, when it is a method bound to + /// a literal that holds exactly one value + fn of(db: &'db dyn Db, ty: Type<'db>) -> Option<(Self, LiteralValueTypeKind<'db>)> { + let (method, receiver) = match ty { + Type::BoundMethod(method) => { + let Type::LiteralValue(receiver) = method.self_instance(db) else { + return None; + }; + (Self::Function(method.function(db)), receiver.kind()) + } + Type::KnownBoundMethod(KnownBoundMethodType::StrStartswith(receiver)) => { + (Self::StrStartswith, LiteralValueTypeKind::String(receiver)) + } + _ => return None, + }; + receiver.holds_one_value().then_some((method, receiver)) + } +} + #[derive(Debug)] enum UnionElement<'db> { Type(Type<'db>), @@ -323,6 +368,12 @@ enum UnionElement<'db> { enum_class: ClassLiteral<'db>, literals: FxOrderMap, bool>, }, + /// basedpython: methods bound to different literals, keyed by the value of the literal each + /// is bound to + LiteralReceiverMethods { + method: LiteralReceiverMethod<'db>, + methods: FxOrderMap, Type<'db>>, + }, } impl<'db> UnionElement<'db> { @@ -333,6 +384,7 @@ impl<'db> UnionElement<'db> { UnionElement::StringLiterals(literals) => literals.len(), UnionElement::BytesLiterals(literals) => literals.len(), UnionElement::EnumLiterals { literals, .. } => literals.len(), + UnionElement::LiteralReceiverMethods { methods, .. } => methods.len(), } } @@ -368,6 +420,9 @@ impl<'db> UnionElement<'db> { .as_nominal_instance() .is_none_or(|instance| instance.class_literal(db, env) != *enum_class), ), + // recovery makes no relation-based simplification, and only an identical + // method could stand for one of these + UnionElement::LiteralReceiverMethods { .. } => ReduceResult::KeepIf(true), UnionElement::Type(_) => unreachable!("ordinary types are handled before recovery"), }; } @@ -469,6 +524,10 @@ impl<'db> UnionElement<'db> { .is_redundant_with(db, env, other_type) } } + UnionElement::LiteralReceiverMethods { methods, .. } => { + methods.retain(|_, method| should_retain_type(*method)); + !methods.is_empty() + } UnionElement::Type(_) => unreachable!("ordinary types are handled before reduction"), }; @@ -636,7 +695,7 @@ impl<'db> UnionBuilder<'db> { let (enum_literal, _) = literals.first().unwrap(); replace_with.push(enum_literal.enum_class_instance(db, &self.env)); } - UnionElement::Type(_) => {} + UnionElement::LiteralReceiverMethods { .. } | UnionElement::Type(_) => {} } } for ty in replace_with { @@ -684,7 +743,7 @@ impl<'db> UnionBuilder<'db> { UnionElement::StringLiterals(literals) => acc + literals.len(), UnionElement::BytesLiterals(literals) => acc + literals.len(), UnionElement::EnumLiterals { literals, .. } => acc + literals.len(), - UnionElement::Type(_) => acc, + UnionElement::LiteralReceiverMethods { .. } | UnionElement::Type(_) => acc, }); if should_widen(literals, self.recursively_defined) { self.widen_literal_types(seen_aliases); @@ -1070,7 +1129,57 @@ impl<'db> UnionBuilder<'db> { let mut ty_negated: Option = None; let mut to_remove = SmallVec::<[usize; 2]>::new(); + // basedpython: a method bound to a literal joins the group of methods that differ from + // it only in their receiver, which it needs no relation query against + let original_ty = ty; + let mut receiver_group = LiteralReceiverMethod::of(db, ty); + let mut group_methods = None; + for (i, element) in self.elements.iter_mut().enumerate() { + // the group's methods are handed over for the rest of the push, so the element is + // moved into the match rather than reborrowed, and the test that it is the group is + // made first: a guard would hold a borrow of it into the other arm + let joins_group = receiver_group.is_some_and(|(method, _)| { + matches!( + element, + UnionElement::LiteralReceiverMethods { method: group_method, .. } + if *group_method == method + ) + }); + let element = match (joins_group, receiver_group, element) { + ( + true, + Some((_, receiver)), + UnionElement::LiteralReceiverMethods { methods, .. }, + ) => { + // every other method in the group has a different receiver, so only one bound + // to the same receiver can be redundant with `ty`, or `ty` with it. it can + // differ from `ty`, e.g. bound to the promotable form of the literal where + // `ty` is bound to the unpromotable one, and only the relation can tell which + // of the two stands for the other + match methods.get(&receiver).copied() { + None => {} + Some(existing) if existing == ty => return, + Some(existing) + if !self.cycle_recovery && ty.is_redundant_with(db, env, existing) => + { + return; + } + // `ty` takes the place of the method redundant with it + Some(existing) + if !self.cycle_recovery && existing.is_redundant_with(db, env, ty) => {} + // neither stands for the other, so `ty` is kept as an element of its own + Some(_) => { + receiver_group = None; + continue; + } + } + group_methods = Some(methods); + continue; + } + (_, _, element) => element, + }; + let element_type = match element.try_reduce(db, &self.env, ty, self.cycle_recovery) { ReduceResult::KeepIf(keep) => { if !keep { @@ -1194,15 +1303,34 @@ impl<'db> UnionBuilder<'db> { } } + // the loop can replace `ty` with a merged type, which is not the method it was grouped as + let new_element = match (receiver_group, group_methods) { + (Some((_, receiver)), Some(methods)) if ty == original_ty => { + methods.insert(receiver, ty); + // descending, to keep the remaining indices valid after `swap_remove` + for index in to_remove.into_iter().rev() { + self.elements.swap_remove(index); + } + return; + } + (Some((method, receiver)), None) if ty == original_ty => { + UnionElement::LiteralReceiverMethods { + method, + methods: FxOrderMap::from_iter([(receiver, ty)]), + } + } + _ => UnionElement::Type(ty), + }; + let mut to_remove = to_remove.into_iter(); if let Some(first) = to_remove.next() { - self.elements[first] = UnionElement::Type(ty); + self.elements[first] = new_element; // We iterate in descending order to keep remaining indices valid after `swap_remove`. for index in to_remove.rev() { self.elements.swap_remove(index); } } else { - self.elements.push(UnionElement::Type(ty)); + self.elements.push(new_element); } } @@ -1253,6 +1381,9 @@ impl<'db> UnionBuilder<'db> { ) })); } + UnionElement::LiteralReceiverMethods { methods, .. } => { + types.extend(methods.into_values()); + } // basedpython: a literal without a group of its own — a float or a complex — // is carried as an ordinary element, so it is the only place the union's // `recursively_defined` can be recorded for it. The grouped kinds above are @@ -2169,13 +2300,19 @@ mod tests { RecursivelyDefined, Type, UnionBuilder, UnionType, }; - use crate::db::tests::{TestDb, setup_db}; + use std::fmt::Write as _; + + use crate::db::tests::{TestDb, TestDbBuilder, setup_db}; use crate::place::{global_symbol, known_module_symbol}; use crate::types::enums::enum_member_literals; use crate::types::type_alias::TypeAliasType; - use crate::types::{KnownClass, KnownInstanceType, Truthiness}; + use crate::types::{ + KnownClass, KnownInstanceType, LiteralValueType, LiteralValueTypeKind, Truthiness, + }; use ruff_db::system::DbWithWritableSystem as _; + use ruff_db::testing::assert_function_query_was_not_run_by_name; + use ruff_python_ast::PythonVersion; use ty_module_resolver::KnownModule; use ty_python_core::ProgramFile; @@ -2369,6 +2506,147 @@ mod tests { } } + /// basedpython: a module of enums whose members are literals of their own kind + fn literal_receiver_enums_db() -> TestDb { + let mut source = String::from( + "from enum import Enum, IntEnum, StrEnum\n\n\ + class Plain(Enum):\n ONE = 1\n TWO = 2\n\n\ + class Name(StrEnum):\n ONE = \"1\"\n TWO = \"2\"\n\n\ + class Code(IntEnum):\n", + ); + for index in 0..64 { + writeln!(source, " M{index} = {index}").unwrap(); + } + let mut db = TestDbBuilder::new() + .with_python_version(PythonVersion::PY312) + .build() + .expect("valid TestDb setup"); + db.write_dedented("/src/enums.py", &source).unwrap(); + db + } + + fn enum_members<'db>(db: &'db TestDb, name: &str) -> Vec> { + let file = ruff_db::files::system_path_to_file(db, "/src/enums.py").unwrap(); + let module = ProgramFile::new(db, file, db.program_environment().program(db)); + let class = global_symbol(db, module, name) + .place + .expect_type() + .expect_class_literal(); + enum_member_literals(db, class, None) + .expect("an enum") + .collect() + } + + /// basedpython: a union of methods bound to many different literals, like `kind.name.lower` + /// for a large enum, is built without a single query: in particular no + /// `is_redundant_with_impl` for each pair of methods, which is what it used to cost + #[test] + fn methods_bound_to_literals_need_no_queries() { + let db = literal_receiver_enums_db(); + // a handle on the same database and its event log, which the types below borrow + let mut events_db = db.clone(); + let db = &db; + let env = db.program_environment(); + + let strings: Vec = (0..64) + .map(|index| Type::string_literal(db, format!("m{index}").as_str())) + .collect(); + let ints: Vec = (0..64).map(Type::int_literal).collect(); + let enum_members = enum_members(db, "Code"); + assert_eq!(enum_members.len(), 64); + + for (receivers, method) in [ + (&strings, "lower"), + (&strings, "startswith"), + (&ints, "bit_length"), + (&enum_members, "bit_length"), + ] { + let methods: Vec = receivers + .iter() + .map(|receiver| receiver.member(db, &env, method).place.expect_type()) + .collect(); + events_db.take_salsa_events(); + + let union = UnionType::from_elements(db, &env, methods.iter().copied()); + + let events = events_db.take_salsa_events(); + assert_function_query_was_not_run_by_name(db, "is_redundant_with_impl", None, &events); + let queries: Vec<_> = events + .iter() + .filter(|event| matches!(event.kind, salsa::EventKind::WillExecute { .. })) + .collect(); + assert!(queries.is_empty(), "`{method}` ran {queries:#?}"); + assert_eq!(union.expect_union().elements(db), methods.as_slice()); + } + } + + /// basedpython: two literals that each hold one value are related only when their kinds are + /// equal, the promotable one being redundant with the unpromotable one. grouping methods + /// bound to literals relies on it, see `LiteralValueTypeKind::holds_one_value` + #[test] + fn one_valued_literals_are_related_only_by_kind() { + let db = literal_receiver_enums_db(); + let db = &db; + let env = db.program_environment(); + + // the same value as far as possible, in every kind holding one value + let literals = [ + Type::int_literal(1), + Type::bool_literal(true), + Type::string_literal(db, "1"), + Type::bytes_literal(db, b"1"), + Type::float_literal(1.0), + Type::complex_literal(db, 1.0, 0.0), + enum_members(db, "Plain")[0], + enum_members(db, "Name")[0], + enum_members(db, "Code")[1], + ] + .map(|ty| ty.as_literal_value().expect("a literal").kind()); + + let mut kinds_seen = [false; 7]; + for kind in literals { + assert!(kind.holds_one_value()); + let index = match kind { + LiteralValueTypeKind::Int(_) => 0, + LiteralValueTypeKind::Bool(_) => 1, + LiteralValueTypeKind::String(_) => 2, + LiteralValueTypeKind::Bytes(_) => 3, + LiteralValueTypeKind::Float(_) => 4, + LiteralValueTypeKind::Complex(_) => 5, + LiteralValueTypeKind::Enum(_) => 6, + LiteralValueTypeKind::LiteralString | LiteralValueTypeKind::Template(_) => { + panic!("`{kind:?}` holds many values") + } + }; + kinds_seen[index] = true; + } + assert_eq!( + kinds_seen, [true; 7], + "a kind holding one value is not sampled" + ); + + let forms = |kind| { + [true, false].map(|promotable| Type::from(LiteralValueType::new(kind, promotable))) + }; + for left in literals { + let [left_promotable, left_unpromotable] = forms(left); + assert!(left_promotable.is_redundant_with(db, &env, left_unpromotable)); + assert!(!left_unpromotable.is_redundant_with(db, &env, left_promotable)); + for right in literals.into_iter().filter(|right| *right != left) { + for left_form in forms(left) { + for right_form in forms(right) { + assert!( + !left_form.is_redundant_with(db, &env, right_form), + "{} is redundant with {}", + left_form.display(db, &env), + right_form.display(db, &env), + ); + } + } + } + } + } + #[test] fn union_common_literal_supertype() { let db = setup_db(); From 78f8250943c2cd23d2b7d3e226f271a8f76e6378 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:01:55 +1000 Subject: [PATCH 16/47] add a `ty_micro[enum_name_method]` benchmark calling `lower` and `startswith` on a 512-member enum's `name` --- crates/ruff_benchmark/benches/ty.rs | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/ruff_benchmark/benches/ty.rs b/crates/ruff_benchmark/benches/ty.rs index 937c172cdc..be5c1aee51 100644 --- a/crates/ruff_benchmark/benches/ty.rs +++ b/crates/ruff_benchmark/benches/ty.rs @@ -548,6 +548,38 @@ fn benchmark_large_enum_membership(criterion: &mut Criterion) { }); } +/// basedpython: regression benchmark for calling a method on a large enum's `name` +/// +/// `name` is a union of one string literal per member, so `kind.name.lower` is a union of as many +/// bound methods, one per name. none of them is redundant with another, and building that union +/// should not have to compare every pair of them to find that out +fn benchmark_enum_name_method(criterion: &mut Criterion) { + const NUM_ENUM_MEMBERS: usize = 512; + + setup_rayon(); + + let mut code = "from enum import IntEnum\n\nclass E(IntEnum):\n".to_string(); + for i in 0..NUM_ENUM_MEMBERS { + writeln!(&mut code, " m{i} = {i}").ok(); + } + code.push_str( + "\ndef lower(kind: E) -> str:\n return kind.name.lower()\n\n\ + def starts(kind: E) -> bool:\n return kind.name.startswith(\"m\")\n", + ); + + criterion.bench_function("ty_micro[enum_name_method]", |b| { + b.iter_batched_ref( + || setup_micro_case(&code), + |case| { + let Case { db } = case; + let result = db.check(); + assert_eq!(result.len(), 0); + }, + BatchSize::SmallInput, + ); + }); +} + fn benchmark_enum_comparison(criterion: &mut Criterion, name: &str, code: &str) { setup_rayon(); @@ -1909,6 +1941,7 @@ criterion_group!( benchmark_complex_constrained_attributes_3, benchmark_gradual_vararg_call, benchmark_large_enum_membership, + benchmark_enum_name_method, benchmark_many_enum_members, benchmark_narrowed_str_enum_comparison, benchmark_optional_str_enum_comparison, From 4218c42c83d2faa2d18682bccdb1e5ea4863760a Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:34:45 +1000 Subject: [PATCH 17/47] record during inference which modules' extensions and conformances it relied on: an extension member read as an attribute or as a bare name in a receiver block, an operator an extension supplies, a conversion whose route an extension or a conformance supplies, a protocol extension member reached through a conformance, a type test answered from the conformance registry, and a protocol extension default a conformance's witness table takes; each module is kept with the diagnostics of the inference that relied on it --- .../src/transforms/parametric_is.rs | 4 +- .../ty_python_semantic/src/types/call/bind.rs | 2 +- .../src/types/conformance.rs | 44 ++++++++++---- .../ty_python_semantic/src/types/context.rs | 21 ++++++- .../src/types/conversions.rs | 55 ++++++++++++++--- .../src/types/diagnostic.rs | 29 ++++++++- .../src/types/extensions.rs | 32 ++++++++-- .../src/types/infer/builder.rs | 29 +++++++-- .../infer/builder/attribute_assignment.rs | 13 ++-- .../src/types/infer/builder/function.rs | 2 +- .../src/types/reified_infer.rs | 60 +++++++++++++++---- 11 files changed, 235 insertions(+), 56 deletions(-) diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index 1a6bc6ea6f..5d52f0fec3 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -298,7 +298,9 @@ fn fragments_for_plan( // `isinstance`: a conforming type is not a subclass. the registry // answers first, and a value nothing registered falls back to carrying // the requirements - ParametricIsPlan::Conformance { target, members } => { + ParametricIsPlan::Conformance { + target, members, .. + } => { needs.references_value = true; needs.conformance_runtime = true; let mut spelled = String::from(", ("); diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index 536c456b17..009e6b4bb5 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -10741,7 +10741,7 @@ impl<'db> BindingError<'db> { Self::get_argument_node(node, *argument_index).map(ArgOrKeyword::value), ) { - crate::types::conversions::report_ambiguous_conversion(context, range, &repair); + crate::types::conversions::accept_conversion(context, range, &repair); return; } diff --git a/crates/ty_python_semantic/src/types/conformance.rs b/crates/ty_python_semantic/src/types/conformance.rs index f4be840436..aa3babd84d 100644 --- a/crates/ty_python_semantic/src/types/conformance.rs +++ b/crates/ty_python_semantic/src/types/conformance.rs @@ -155,10 +155,10 @@ fn imported_module_names(db: &dyn Db, file: File) -> Vec { .collect() } -/// the protocol a conformance extension in `file` makes `receiver_class` -/// conform to, when that protocol's class literal is `protocol`. +/// the conformance extension visible in `file` that makes `receiver_class` conform to +/// `protocol`, with the protocol it declares. /// -/// The answer is the protocol *as the conformance declared it*, so a specialized +/// The protocol is the one *as the conformance declared it*, so a specialized /// conformance (`extension MyBox(Container[int]):`) hands back `Container[int]` /// — which is what a protocol extension's members must be specialized at when /// they are reached through the conforming type @@ -168,20 +168,21 @@ pub(crate) fn conformance_for<'db>( file: File, receiver_class: ClassType<'db>, protocol: ClassLiteral<'db>, -) -> Option> { +) -> Option<(StaticClassLiteral<'db>, ClassType<'db>)> { for &(extension, declared) in visible_conformances(db, file) { if declared.class_literal(db) != protocol { continue; } if extension_applies(db, env, extension, receiver_class).is_some() { - return Some(declared); + return Some((extension, declared)); } } None } /// would a visible conformance make `source` acceptable where `target` is asked -/// for? the protocol it conforms to, for the diagnostic that reports two routes. +/// for? the conformance, and the protocol it conforms to for the diagnostic that +/// reports two routes. /// /// Unlike a conversion this materializes nothing: the value already answers /// every requirement, through the witness table its conformance registered at @@ -194,7 +195,7 @@ pub(crate) fn repair_with_conformance<'db>( file: File, source: Type<'db>, target: Type<'db>, -) -> Option> { +) -> Option<(StaticClassLiteral<'db>, ClassType<'db>)> { if !file.source_type(db).is_basedpython() { return None; } @@ -215,7 +216,7 @@ pub(crate) fn repair_with_conformance<'db>( continue; } if Type::instance(db, env, protocol).is_assignable_to(db, env, target) { - return Some(protocol); + return Some((extension, protocol)); } } None @@ -452,8 +453,14 @@ pub(crate) fn validate_conformance_declaration<'db>( } // two conformances of one pair would register two witness tables against - // the same key, and which one survives would depend on import order - if let Some(other) = conflicting_conformance(db, extension, target, *interface) + // the same key, and which one survives would depend on import order. the + // import bringing the other one into scope is what puts the pair in that + // state, so it is relied on until one of the two is dropped + let conflicting = conflicting_conformance(db, extension, target, *interface); + if let Some(other) = conflicting { + context.record_extension_use(other); + } + if let Some(other) = conflicting && let Some(builder) = context.report_lint(&INVALID_CONFORMANCE, node.range()) { let mut diagnostic = builder.into_diagnostic(format_args!( @@ -587,6 +594,23 @@ pub(crate) fn validate_conformance_declaration<'db>( } } + // a default the protocol's own extension supplies goes into the witness + // table this conformance registers, so the import that makes that extension + // applicable here is relied on + for requirement in interface_requirements(db, *interface) { + let name = requirement.as_str(); + if own_member(db, extension, name).is_none() + && let Some(default) = protocol_extension_supplying( + db, + extension.file(db), + interface.class_literal(db), + name, + ) + { + context.record_extension_use(default); + } + } + // every requirement has to be answered by something: the block itself, a // default on the protocol's own extension, or a member the type already // has. an unanswered one is an `AttributeError` the moment anything diff --git a/crates/ty_python_semantic/src/types/context.rs b/crates/ty_python_semantic/src/types/context.rs index bbe77164ad..604265a8f0 100644 --- a/crates/ty_python_semantic/src/types/context.rs +++ b/crates/ty_python_semantic/src/types/context.rs @@ -17,6 +17,7 @@ use super::{Type, TypeCheckDiagnostics, infer_definition_types}; use crate::diagnostic::DiagnosticGuard; use crate::lint::LintSource; use crate::reachability::is_range_reachable; +use crate::types::class::StaticClassLiteral; use crate::types::diagnostic::{INVALID_TYPE_FORM, UNBOUND_TYPE_VARIABLE}; use crate::types::function::FunctionDecorators; use crate::types::infer::InferenceFlags; @@ -235,7 +236,25 @@ impl<'db, 'ast> InferContext<'db, 'ast> { } pub(super) fn has_diagnostics(&self) -> bool { - !self.diagnostics.borrow().is_empty() + self.diagnostics.borrow().has_reports() + } + + /// basedpython: records that `extension` supplied something this inference relied on, so + /// the import that makes it applicable is a used one. + /// + /// Recorded even where diagnostics are suppressed: a speculative inference whose result is + /// kept relied on the extension just the same, and one whose result is dropped takes its + /// record with it + pub(crate) fn record_extension_use(&self, extension: StaticClassLiteral<'db>) { + self.record_extension_module_use(extension.file(self.db)); + } + + /// basedpython: [`Self::record_extension_use`], for a use known only by the module that + /// declares the extension + pub(crate) fn record_extension_module_use(&self, module: File) { + self.diagnostics + .borrow_mut() + .mark_extension_module_used(module); } /// Prevents diagnostic construction for this inference context. diff --git a/crates/ty_python_semantic/src/types/conversions.rs b/crates/ty_python_semantic/src/types/conversions.rs index 6bc670d61c..02df942a60 100644 --- a/crates/ty_python_semantic/src/types/conversions.rs +++ b/crates/ty_python_semantic/src/types/conversions.rs @@ -79,7 +79,10 @@ pub(crate) enum Route<'db> { /// a visible conformance extension makes the source's class conform to the /// protocol the target asks for. nothing is emitted: the value is already /// the object the protocol dispatches on - Conformance(ClassType<'db>), + Conformance { + extension: StaticClassLiteral<'db>, + protocol: ClassType<'db>, + }, /// `T.__from__(value)`, where `T` is the target class From(ClassType<'db>, DunderSource<'db>), /// `T.__of__(value)`, where `T` is the target class @@ -94,6 +97,19 @@ pub(crate) enum Route<'db> { } impl<'db> Route<'db> { + /// the extension the route is supplied by, when one is + fn extension(self) -> Option> { + match self { + Route::Conformance { extension, .. } + | Route::From(_, DunderSource::Extension(extension)) + | Route::Of(_, DunderSource::Extension(extension)) => Some(extension), + Route::From(_, DunderSource::Declared) + | Route::Of(_, DunderSource::Declared) + | Route::Into(_) + | Route::DiscardReturn => None, + } + } + /// how the route reads in a diagnostic fn describe(self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> String { let dunder = |class: ClassType<'db>, source: DunderSource<'db>, dunder: &str| match source { @@ -107,7 +123,9 @@ impl<'db> Route<'db> { ), }; match self { - Route::Conformance(protocol) => format!("conformance to `{}`", protocol.name(db)), + Route::Conformance { protocol, .. } => { + format!("conformance to `{}`", protocol.name(db)) + } Route::From(class, source) => dunder(class, source, FROM), Route::Of(class, source) => dunder(class, source, OF), Route::Into(source) => format!("{}.{INTO}", source.display(db, env)), @@ -156,10 +174,13 @@ pub(crate) fn repair_conversion<'db>( let source = source.erase_restriction(db); let mut routes: Vec> = Vec::new(); - if let Some(protocol) = + if let Some((extension, protocol)) = super::conformance::repair_with_conformance(db, env, file, source, target) { - routes.push(Route::Conformance(protocol)); + routes.push(Route::Conformance { + extension, + protocol, + }); } dunder_routes(db, env, file, source, target, value, &mut routes); if discards_return(db, env, source, target) { @@ -897,14 +918,15 @@ pub(crate) fn call_parameter_types<'db>( bindings.plain_callee_parameter_types(arguments.len()) } -/// report a conversion site served by more than one route. the site is otherwise -/// accepted — any of the conversions would work — but which one runs must not -/// depend on ordering -pub(crate) fn report_ambiguous_conversion<'db>( +/// accept a conversion site's repair: record the extensions it relies on, and report +/// a site served by more than one route. the site is otherwise accepted — any of the +/// conversions would work — but which one runs must not depend on ordering +pub(crate) fn accept_conversion<'db>( context: &InferContext<'db, '_>, node: impl Ranged, repair: &ConversionRepair<'db>, ) { + record_conversion_uses(context, repair); let env = context.program_environment(); let db = context.db(); if repair.ambiguous_with.is_empty() { @@ -922,6 +944,21 @@ pub(crate) fn report_ambiguous_conversion<'db>( diagnostic.help("Remove all but one of them, or write the conversion you want explicitly"); } +/// record the extensions a repair relies on: the one supplying its route, and every rival +/// route's. a site served by two is reported rather than resolved, and each import keeps +/// the site in that state, so neither is removable on its own +pub(crate) fn record_conversion_uses<'db>( + context: &InferContext<'db, '_>, + repair: &ConversionRepair<'db>, +) { + for extension in std::iter::once(repair.route) + .chain(repair.ambiguous_with.iter().copied()) + .filter_map(Route::extension) + { + context.record_extension_use(extension); + } +} + /// the conversions the value expression `value` needs so that it satisfies /// `declared`, as `(range to wrap, conversion)` in source order. /// @@ -1130,7 +1167,7 @@ pub(crate) fn conversion_info<'db>( // conformance is not a conversion at runtime: the value already answers // every member the protocol asks for, through the witness table its // conformance registered. the site emits nothing at all - Route::Conformance(_) => ConversionInfo::Call { + Route::Conformance { .. } => ConversionInfo::Call { prefix: String::new(), suffix: String::new(), replaces_value: false, diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 27330e1130..c400048e45 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -4117,6 +4117,11 @@ declare_lint! { pub struct TypeCheckDiagnostics { diagnostics: Vec, used_suppressions: FxHashSet, + /// basedpython: the modules whose `extension` or conformance supplied something inference + /// relied on — a member, an operator's dunder, a conversion, a conformance. none of these + /// reads a name the module's import binds, so the import that makes the module's + /// extensions applicable is used only through this record (see `unused-import`) + used_extension_modules: FxHashSet, } pub(crate) fn report_mismatched_type_name<'db>( @@ -4154,6 +4159,8 @@ impl TypeCheckDiagnostics { pub(super) fn extend(&mut self, other: &TypeCheckDiagnostics) { self.diagnostics.extend_from_slice(&other.diagnostics); self.used_suppressions.extend(&other.used_suppressions); + self.used_extension_modules + .extend(&other.used_extension_modules); } pub(super) fn extend_diagnostics(&mut self, diagnostics: impl IntoIterator) { @@ -4172,8 +4179,15 @@ impl TypeCheckDiagnostics { self.used_suppressions.len() } + /// basedpython: records that an extension or conformance declared in `module` supplied + /// something inference relied on + pub(crate) fn mark_extension_module_used(&mut self, module: File) { + self.used_extension_modules.insert(module); + } + pub(crate) fn shrink_to_fit(&mut self) { self.used_suppressions.shrink_to_fit(); + self.used_extension_modules.shrink_to_fit(); self.diagnostics.shrink_to_fit(); } @@ -4182,7 +4196,13 @@ impl TypeCheckDiagnostics { } pub(crate) fn is_empty(&self) -> bool { - self.diagnostics.is_empty() && self.used_suppressions.is_empty() + !self.has_reports() && self.used_extension_modules.is_empty() + } + + /// whether anything was reported, suppressed or not. a recorded extension use is not a + /// report + pub(crate) fn has_reports(&self) -> bool { + !self.diagnostics.is_empty() || !self.used_suppressions.is_empty() } fn iter(&self) -> std::slice::Iter<'_, Diagnostic> { @@ -4907,8 +4927,11 @@ pub(super) fn report_invalid_assignment<'db>( value_node, target_ty, ); - if let Some((range, repair)) = conversions.first() { - crate::types::conversions::report_ambiguous_conversion(context, *range, repair); + if let Some(((range, repair), rest)) = conversions.split_first() { + crate::types::conversions::accept_conversion(context, *range, repair); + for (_, repair) in rest { + crate::types::conversions::record_conversion_uses(context, repair); + } return; } } diff --git a/crates/ty_python_semantic/src/types/extensions.rs b/crates/ty_python_semantic/src/types/extensions.rs index 7f5ca2e7f4..c3070f5e7e 100644 --- a/crates/ty_python_semantic/src/types/extensions.rs +++ b/crates/ty_python_semantic/src/types/extensions.rs @@ -563,6 +563,21 @@ pub(crate) struct ExtensionMemberResolution<'db> { /// another applicable extension that also supplies the member — an /// ambiguity the checker reports at the access site pub(crate) ambiguous_with: Option>, + /// the conformance the receiver reached the member through, when the member is a + /// protocol extension's and the receiver is not an instance of the protocol + pub(crate) conformance: Option>, +} + +impl<'db> ExtensionMemberResolution<'db> { + /// every extension the resolution relied on: the one supplying the member, the + /// conformance it was reached through, and a rival that supplies it too. a file's + /// import of each one's module is what makes it applicable, so each makes that import + /// a used one + pub(crate) fn relied_on(&self) -> impl Iterator> { + std::iter::once(self.extension) + .chain(self.conformance) + .chain(self.ambiguous_with) + } } /// resolve an attribute access that found no regular member: consult the @@ -663,9 +678,13 @@ pub(crate) fn resolve_extension_members<'db>( } resolved.push(ExtensionMemberResolution { extension, - ty: bind(member, applicable.conformed_as), + ty: bind( + member, + applicable.conformed_as.map(|(_, protocol)| protocol), + ), kind, ambiguous_with: None, + conformance: applicable.conformed_as.map(|(conformance, _)| conformance), }); } @@ -947,10 +966,10 @@ pub(crate) fn own_member<'db>( /// an extension member that applies to one receiver struct ApplicableMember<'db> { member: Type<'db>, - /// the interface the receiver reached this member through, when it was - /// reached by conformance rather than by being an instance of the extended + /// the conformance and the interface the receiver reached this member through, when + /// it was reached by conformance rather than by being an instance of the extended /// type. that interface is what the member's `self` is written as - conformed_as: Option>, + conformed_as: Option<(StaticClassLiteral<'db>, ClassType<'db>)>, } /// if `extension` applies to `receiver_class` and declares `name`, return the @@ -1000,12 +1019,13 @@ fn applicable_member<'db>( }); } let target = extended_class(db, extension)?; - let conformed = conformance::conformance_for(db, env, file, receiver_class, target)?; + let (conformance, conformed) = + conformance::conformance_for(db, env, file, receiver_class, target)?; let application = applied_at(db, env, extension, target, conformed)?; let member = own_member(db, extension, name)?; Some(ApplicableMember { member: application.apply(db, env, extension, member), - conformed_as: Some(conformed), + conformed_as: Some((conformance, conformed)), }) } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 957fac1e77..33bcf654bf 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -13929,6 +13929,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Some(name_node), ) { + if let receivers::ImplicitReceiverName::ExtensionMember { resolution, .. } = &resolved { + self.record_extension_resolution(resolution); + } return resolved.ty(); } @@ -15045,6 +15048,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if let Some(resolution) = extensions::resolve_extension_member(db, env, self.file(), value_type, &attr.id) { + self.record_extension_resolution(&resolution); if let Some(other) = resolution.ambiguous_with && let Some(builder) = self .context @@ -15759,11 +15763,18 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return None; } let db = self.db(); - extensions::unary_extension_operator(db, env, self.file(), op, operand)?.return_type( - db, - env, - &CallArguments::none(), - ) + let operator = extensions::unary_extension_operator(db, env, self.file(), op, operand)?; + let returned = operator.return_type(db, env, &CallArguments::none())?; + self.record_extension_resolution(&operator.resolution); + Some(returned) + } + + /// basedpython: records every extension an extension-member resolution relied on, so + /// the imports that make them applicable count as used + fn record_extension_resolution(&self, resolution: &extensions::ExtensionMemberResolution<'db>) { + for extension in resolution.relied_on() { + self.context.record_extension_use(extension); + } } /// basedpython: the type a binary operator evaluates to when an applicable @@ -15782,7 +15793,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let operator = extensions::binary_extension_operator(db, env, self.file(), left, op, right)?; let argument = if operator.reflected { left } else { right }; - operator.return_type(db, env, &CallArguments::positional([argument])) + let returned = operator.return_type(db, env, &CallArguments::positional([argument]))?; + self.record_extension_resolution(&operator.resolution); + Some(returned) } /// basedpython: the type a comparison evaluates to when an applicable @@ -15804,6 +15817,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { extensions::comparison_extension_operator(db, env, self.file(), left, op, right)?; let argument = if operator.reflected { left } else { right }; let returned = operator.return_type(db, env, &CallArguments::positional([argument]))?; + self.record_extension_resolution(&operator.resolution); Some(match op { ast::CmpOp::In | ast::CmpOp::NotIn => KnownClass::Bool.to_instance(db, env), _ => returned, @@ -16578,6 +16592,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { target, Some(right), ); + for module in plan.conformance_modules() { + self.context.record_extension_module_use(module); + } if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = plan { let source = ruff_db::source::source_text(db, self.file()); self.report_erased_type_check( diff --git a/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs b/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs index 220d6a65c0..85dd4c22d8 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs @@ -545,13 +545,14 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { } else { Vec::new() }; - if let Some((range, repair)) = conversions.first() { + if let Some(((range, repair), rest)) = conversions.split_first() { if emit_diagnostics { - crate::types::conversions::report_ambiguous_conversion( - &self.builder.context, - *range, - repair, - ); + crate::types::conversions::accept_conversion(&self.builder.context, *range, repair); + } else { + crate::types::conversions::record_conversion_uses(&self.builder.context, repair); + } + for (_, repair) in rest { + crate::types::conversions::record_conversion_uses(&self.builder.context, repair); } return true; } diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index dde10d8215..4c57181333 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -585,7 +585,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ), ) .is_some_and(|repair| { - crate::types::conversions::report_ambiguous_conversion( + crate::types::conversions::accept_conversion( &self.context, return_statement.range, &repair, diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 0628430ff9..aef65d7c07 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -772,6 +772,10 @@ pub enum ParametricIsPlan { /// the interface's required member names, for the runtime check to look /// for on a value nothing registered members: Vec, + /// the modules declaring the conformances to the interface that are visible + /// here. the test is answered from the registry only because one is, so the + /// import making each one visible is relied on + declared_in: Vec, }, /// nothing is known about the target — an error elsewhere left `Unknown` /// behind. the test is lowered as the plain `isinstance` the source spells @@ -824,6 +828,27 @@ impl ParametricIsPlan { _ => false, } } + + /// every module whose conformance this plan answers from, in any arm + pub(crate) fn conformance_modules(&self) -> Vec { + match self { + Self::Conformance { declared_in, .. } => declared_in.clone(), + Self::Union(arms) => arms.iter().flat_map(Self::conformance_modules).collect(), + Self::Fold(_) + | Self::TokenEq(_) + | Self::Probe { .. } + | Self::ProtocolStructural(_) + | Self::ErasedTarget(_) + | Self::Isinstance(_) + | Self::IsCallable + | Self::IsNone + | Self::Equality { .. } + | Self::Identity(_) + | Self::Subclass(_) + | Self::Pattern(_) + | Self::Unresolved => Vec::new(), + } + } } /// basedpython: how a type test's target is written into the emitted python @@ -1550,11 +1575,15 @@ fn class_target_plan<'db>( // an interface something visibly conforms to is answered by the // registry rather than by the class hierarchy, so it is checkable // even though a conforming type is not a subclass - if let Some(members) = conformance_members(db, file, class) { + if let Some((members, declared_in)) = conformance_members(db, file, class) { return match written_or_spelled(target_node, false, || { spell_class(db, env, file, class) }) { - Some(target) => ParametricIsPlan::Conformance { target, members }, + Some(target) => ParametricIsPlan::Conformance { + target, + members, + declared_in, + }, None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), }; } @@ -1586,21 +1615,28 @@ fn class_target_plan<'db>( } /// The required member names of `class` when something in `file`'s scope -/// visibly conforms to it, or `None` when it is not a conformance interface. +/// visibly conforms to it, with the modules declaring those conformances, or +/// `None` when it is not a conformance interface. fn conformance_members<'db>( db: &'db dyn Db, file: File, class: ClassType<'db>, -) -> Option> { - let conformed = crate::types::conformance::visible_conformances(db, file) +) -> Option<(Vec, Vec)> { + let mut declared_in: Vec = Vec::new(); + for (extension, declared) in crate::types::conformance::visible_conformances(db, file) { + let module = extension.file(db); + if declared.class_literal(db) == class.class_literal(db) && !declared_in.contains(&module) { + declared_in.push(module); + } + } + if declared_in.is_empty() { + return None; + } + let members = crate::types::conformance::interface_requirements(db, class) .iter() - .any(|(_, declared)| declared.class_literal(db) == class.class_literal(db)); - conformed.then(|| { - crate::types::conformance::interface_requirements(db, class) - .iter() - .map(ToString::to_string) - .collect() - }) + .map(ToString::to_string) + .collect(); + Some((members, declared_in)) } /// whether a callable signature says nothing beyond "this is callable" — the From 0108e6d268b86790d7a2b18fcaa68fabcca1b668 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:35:33 +1000 Subject: [PATCH 18/47] add a per-file query listing each import that makes another module's extensions applicable, with the module it grants: `import a.b` grants `a.b` and `a`, `from a import x` grants `a` --- .../src/types/extensions.rs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/crates/ty_python_semantic/src/types/extensions.rs b/crates/ty_python_semantic/src/types/extensions.rs index c3070f5e7e..c9936dc7c4 100644 --- a/crates/ty_python_semantic/src/types/extensions.rs +++ b/crates/ty_python_semantic/src/types/extensions.rs @@ -24,11 +24,13 @@ //! [`ClassDef`]: ruff_python_ast::StmtClassDef use ruff_db::files::File; +use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; use ruff_python_ast as ast; use ruff_python_ast::name::Name; use rustc_hash::{FxHashMap, FxHashSet}; use ty_module_resolver::{ModuleName, resolve_module}; +use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_core::{global_scope, place_table, semantic_index}; use crate::Db; @@ -181,6 +183,74 @@ pub(crate) fn applicable_extensions(db: &dyn Db, file: File) -> Box<[StaticClass extensions.into_boxed_slice() } +/// an import in a file that makes another module's extensions applicable there +#[derive(Debug, Clone, Copy, PartialEq, Eq, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) struct ExtensionGrant<'db> { + /// the module whose extensions the import makes applicable + pub(crate) module: File, + /// the import's definition: one alias of an `import` or a `from … import` + pub(crate) import: Definition<'db>, +} + +/// every import in `file` that makes another module's extensions applicable, with +/// the module each one grants. +/// +/// This is [`applicable_extensions`]'s visibility rule read backwards, so it has to +/// grant exactly what that one does: `import a.b` grants `a.b` and `a`, and +/// `from a import x` grants `a`, wherever in the file the statement is written. A +/// module declaring no extension is left out, because no extension use can name it +#[salsa::tracked(returns(deref), heap_size = ruff_memory_usage::heap_size)] +pub(crate) fn extension_granting_imports(db: &dyn Db, file: File) -> Box<[ExtensionGrant<'_>]> { + if !file.source_type(db).is_basedpython() { + return Box::default(); + } + let program_file = db.program_file(file); + let index = semantic_index(db, program_file); + let parsed = parsed_module(db, program_file.python_file(db)).load(db); + let importing = ImportingFile::File(file, program_file.resolver_environment(db)); + let mut grants = Vec::new(); + for scope in index.scope_ids() { + let use_def = index.use_def_map(scope.file_scope_id(db)); + for (_, definition, _) in use_def.definitions_with_usage() { + let modules: Vec = match definition.kind(db) { + DefinitionKind::Import(import) => { + let alias = import.alias(&parsed); + if alias.is_resource { + continue; + } + let Some(name) = ModuleName::new(&alias.name) else { + continue; + }; + name.ancestors().collect() + } + DefinitionKind::ImportFrom(import) => { + let Ok(name) = + ModuleName::from_import_statement(db, importing, import.import(&parsed)) + else { + continue; + }; + vec![name] + } + _ => continue, + }; + for name in modules { + let Some(module) = + resolve_module(db, importing, &name).and_then(|module| module.file(db)) + else { + continue; + }; + if module != file && !extensions_in_module(db, module).is_empty() { + grants.push(ExtensionGrant { + module, + import: definition, + }); + } + } + } + } + grants.into_boxed_slice() +} + /// the classes an applicable extension supplies a conversion dunder for. /// /// The transpiler's hot-path gate asks whether a type might convert before doing From bdcc01d37265c9759d352a811919f76cb0d3cc37 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:53:31 +1000 Subject: [PATCH 19/47] record during inference the names it reads as every binding their scope makes, which the semantic index has no record of: a receiver callable read as a method (`x.fn()`), a name in a lazily evaluated annotation, a name read through `global`, the enum a context-sensitive name or case name is lowered to qualify with, and the module bindings the runtime spelling of a type test, a checked cast or a `raises` guard reads --- .../ty_python_semantic/src/types/context.rs | 30 +++++- .../src/types/context_sensitive.rs | 55 +++++++--- .../src/types/diagnostic.rs | 65 +++++++++-- .../src/types/infer/builder.rs | 101 +++++++++++++++++- .../src/types/infer/builder/function.rs | 19 ++++ .../ty_python_semantic/src/types/receivers.rs | 2 +- .../src/types/reified_infer.rs | 37 +++++++ .../src/types/runtime_spelling.rs | 31 ++++-- .../ty_python_semantic/src/types/soundness.rs | 20 ++++ 9 files changed, 322 insertions(+), 38 deletions(-) diff --git a/crates/ty_python_semantic/src/types/context.rs b/crates/ty_python_semantic/src/types/context.rs index 604265a8f0..42981c3b16 100644 --- a/crates/ty_python_semantic/src/types/context.rs +++ b/crates/ty_python_semantic/src/types/context.rs @@ -21,6 +21,7 @@ use crate::types::class::StaticClassLiteral; use crate::types::diagnostic::{INVALID_TYPE_FORM, UNBOUND_TYPE_VARIABLE}; use crate::types::function::FunctionDecorators; use crate::types::infer::InferenceFlags; +use crate::types::runtime_spelling::RuntimeSpelling; use crate::{ Db, Program, lint::{LintId, LintMetadata}, @@ -28,8 +29,9 @@ use crate::{ }; use ty_module_resolver::ResolverEnvironment; use ty_python_core::definition::Definition; -use ty_python_core::scope::ScopeId; -use ty_python_core::{ProgramFile, semantic_index}; +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::symbol::ScopedSymbolId; +use ty_python_core::{ProgramFile, global_scope, place_table, semantic_index}; /// The lazily resolved program used by a semantic operation. #[derive(Clone)] @@ -249,6 +251,30 @@ impl<'db, 'ast> InferContext<'db, 'ast> { self.record_extension_module_use(extension.file(self.db)); } + /// basedpython: records that inference read every binding `symbol` has in `scope`, at a + /// site the semantic index records no read for + pub(crate) fn record_scope_read(&self, scope: FileScopeId, symbol: ScopedSymbolId) { + self.diagnostics.borrow_mut().mark_scope_read(scope, symbol); + } + + /// basedpython: [`Self::record_scope_read`], for the symbol named `name` + pub(crate) fn record_scope_read_of_name(&self, scope: ScopeId<'db>, name: &str) { + let db = self.db; + if let Some(symbol) = place_table(db, scope).symbol_id(name) { + self.record_scope_read(scope.file_scope_id(db), symbol); + } + } + + /// basedpython: records the bindings of the module's global scope a spelling the + /// transpiled program writes reads. the checker spells it while it checks, exactly as + /// the lowering will, and the source spells none of those reads + pub(crate) fn record_spelling(&self, spelling: &RuntimeSpelling) { + let global = global_scope(self.db, self.program_file); + for name in spelling.global_reads() { + self.record_scope_read_of_name(global, name); + } + } + /// basedpython: [`Self::record_extension_use`], for a use known only by the module that /// declares the extension pub(crate) fn record_extension_module_use(&self, module: File) { diff --git a/crates/ty_python_semantic/src/types/context_sensitive.rs b/crates/ty_python_semantic/src/types/context_sensitive.rs index c462f7feda..589eef4df7 100644 --- a/crates/ty_python_semantic/src/types/context_sensitive.rs +++ b/crates/ty_python_semantic/src/types/context_sensitive.rs @@ -28,7 +28,8 @@ use std::ops::ControlFlow; use ruff_db::files::File; use ruff_python_ast::name::Name; use ty_python_core::predicate::CaseNamePredicateKind; -use ty_python_core::scope::ScopeId; +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::symbol::ScopedSymbolId; use ty_python_core::{place_table, semantic_index}; use crate::Db; @@ -52,6 +53,10 @@ pub(crate) struct ContextSensitiveMember<'db> { enum_class: ClassLiteral<'db>, /// the type of `.` pub(crate) ty: Type<'db>, + /// the binding the qualifier is read through once the name is lowered to + /// `.`: the symbol, and the scope that binds it. set once the member is + /// known to be nameable + qualifier_binding: Option<(FileScopeId, ScopedSymbolId)>, } impl<'db> ContextSensitiveMember<'db> { @@ -65,6 +70,20 @@ impl<'db> ContextSensitiveMember<'db> { pub(crate) fn enum_class(self) -> ClassLiteral<'db> { self.enum_class } + + /// the binding the lowered `.` reads the qualifier through. the checker + /// records it, because the source spells no read of it + pub(crate) fn qualifier_binding(self) -> Option<(FileScopeId, ScopedSymbolId)> { + self.qualifier_binding + } + + /// this member, once the enum is known to be nameable here through `binding` + fn named_through(self, binding: (FileScopeId, ScopedSymbolId)) -> Self { + Self { + qualifier_binding: Some(binding), + ..self + } + } } /// what the expected type has to say about `name` before the resolution rules @@ -132,7 +151,8 @@ pub(crate) fn resolve_in_context<'db>( let Search::Found(member) = search(db, env, target, name) else { return None; }; - is_nameable(db, file, scope, member.enum_class).then_some(member) + nameable_through(db, file, scope, member.enum_class) + .map(|binding| member.named_through(binding)) } /// the member a bare `case A:` names, or `None` when it is the capture python @@ -157,7 +177,8 @@ pub(crate) fn resolve_case_name<'db>( let Search::Found(member) = search(db, env, subject_ty, name) else { return None; }; - is_nameable(db, file, scope, member.enum_class).then_some(member) + nameable_through(db, file, scope, member.enum_class) + .map(|binding| member.named_through(binding)) } /// the type a bare `case A:` matches, or `None` when the name is an ordinary @@ -200,7 +221,7 @@ pub(crate) fn explain_case_name_miss<'db>( { Some(Miss::Shadowed(member.enum_class)) } - Search::Found(member) if !is_nameable(db, file, scope, member.enum_class) => { + Search::Found(member) if nameable_through(db, file, scope, member.enum_class).is_none() => { Some(Miss::Unnameable(member.enum_class)) } Search::Found(_) | Search::Nothing => None, @@ -221,7 +242,7 @@ pub(crate) fn explain_miss<'db>( Search::Found(member) if claimed_by_lexical_scope(db, env, file, scope, name) => { Some(Miss::Shadowed(member.enum_class)) } - Search::Found(member) if !is_nameable(db, file, scope, member.enum_class) => { + Search::Found(member) if nameable_through(db, file, scope, member.enum_class).is_none() => { Some(Miss::Unnameable(member.enum_class)) } Search::Found(_) | Search::Nothing => None, @@ -272,6 +293,7 @@ fn member_of<'db>( return Some(ContextSensitiveMember { enum_class: literal, ty: Type::enum_literal(EnumLiteralType::new(db, enum_class, member.clone())), + qualifier_binding: None, }); } // a based enum's payload variant, whose class is declared in the enum body @@ -283,6 +305,7 @@ fn member_of<'db>( return Some(ContextSensitiveMember { enum_class: literal, ty, + qualifier_binding: None, }); } } @@ -296,28 +319,31 @@ fn is_variant_class<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { .is_some_and(|class| class.is_enum_variant(db)) } -/// whether the enum is reachable from `scope` under its own name. the transpiler -/// qualifies the resolved name with it (`Red` → `Color.Red`), so a spelling that -/// is not in scope has no lowering — and must therefore not resolve here either -fn is_nameable<'db>( +/// the binding the enum is reachable through from `scope` under its own name: the +/// symbol, and the scope that binds it. the transpiler qualifies the resolved name with +/// it (`Red` → `Color.Red`), so a spelling that is not in scope has no lowering — and +/// must therefore not resolve here either +fn nameable_through<'db>( db: &'db dyn Db, file: File, scope: ScopeId<'db>, enum_class: ClassLiteral<'db>, -) -> bool { +) -> Option<(FileScopeId, ScopedSymbolId)> { let name = enum_class.name(db); let index = semantic_index(db, db.program_file(file)); for (ancestor_id, _) in index.visible_ancestor_scopes(scope.file_scope_id(db)) { let ancestor_scope = ancestor_id.to_scope_id(db, db.program_file(file)); - let Some(place) = place_table(db, ancestor_scope).symbol_by_name(name) else { + let table = place_table(db, ancestor_scope); + let Some(symbol_id) = table.symbol_id(name) else { continue; }; + let place = table.symbol(symbol_id); if !(place.is_bound() || place.is_declared()) { continue; } // the first scope that gives the name a value decides it, exactly as a // name load would - return symbol( + let names_enum = symbol( db, ancestor_scope, name, @@ -327,8 +353,9 @@ fn is_nameable<'db>( .ignore_possibly_undefined() .and_then(Type::as_class_literal) == Some(enum_class); + return names_enum.then_some((ancestor_id, symbol_id)); } - false + None } /// shared checker/transpiler contract: the enum a name that resolved to `ty` @@ -381,7 +408,7 @@ pub(crate) fn enum_class_for_unbound_name<'db>( if !declares_member(db, env, enum_class, name) { return None; } - is_nameable(db, file, scope, enum_class).then_some(enum_class) + nameable_through(db, file, scope, enum_class).map(|_| enum_class) } /// the enum an already-resolved member type belongs to diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index c400048e45..9360948594 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -63,6 +63,8 @@ use ty_module_resolver::{KnownModule, Module, ModuleName, SearchPath, file_to_mo use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; use ty_python_core::place::{PlaceTable, ScopedPlaceId}; use ty_python_core::predicate::CaseNamePredicateKind; +use ty_python_core::scope::FileScopeId; +use ty_python_core::symbol::ScopedSymbolId; use ty_python_core::{ProgramFile, global_scope, place_table, use_def_map}; const RUNTIME_CHECKABLE_DOCS_URL: &str = @@ -4117,11 +4119,27 @@ declare_lint! { pub struct TypeCheckDiagnostics { diagnostics: Vec, used_suppressions: FxHashSet, - /// basedpython: the modules whose `extension` or conformance supplied something inference - /// relied on — a member, an operator's dunder, a conversion, a conformance. none of these - /// reads a name the module's import binds, so the import that makes the module's - /// extensions applicable is used only through this record (see `unused-import`) - used_extension_modules: FxHashSet, + /// basedpython: allocated by the first use recorded. every inference builder holds its + /// diagnostics by value, and inference recurses through builders on the stack, so the + /// record is kept out of line + uses: Option>, +} + +/// basedpython: what an inference relied on that the semantic index has no record of +#[derive(Default, Eq, PartialEq, get_size2::GetSize)] +struct InferenceUses { + /// the modules whose `extension` or conformance supplied something inference relied on — + /// a member, an operator's dunder, a conversion, a conformance. none of these reads a name + /// the module's import binds, so the import that makes the module's extensions applicable + /// is used only through this record (see `unused-import`) + extension_modules: FxHashSet, + /// the names inference read as every binding their scope makes, where the semantic index + /// records no read — a receiver callable read as a method (`x.fn()`), a name in an + /// annotation, which is evaluated lazily, a name read through `global`, the enum a + /// context-sensitive name or case name is lowered to qualify (`Color.Red`), and each + /// module-level name a runtime spelling the transpiled program writes reads. each is a + /// symbol of the scope that owns it + scope_reads: FxHashSet<(FileScopeId, ScopedSymbolId)>, } pub(crate) fn report_mismatched_type_name<'db>( @@ -4159,8 +4177,11 @@ impl TypeCheckDiagnostics { pub(super) fn extend(&mut self, other: &TypeCheckDiagnostics) { self.diagnostics.extend_from_slice(&other.diagnostics); self.used_suppressions.extend(&other.used_suppressions); - self.used_extension_modules - .extend(&other.used_extension_modules); + if let Some(other_uses) = &other.uses { + let uses = self.uses_mut(); + uses.extension_modules.extend(&other_uses.extension_modules); + uses.scope_reads.extend(&other_uses.scope_reads); + } } pub(super) fn extend_diagnostics(&mut self, diagnostics: impl IntoIterator) { @@ -4182,12 +4203,36 @@ impl TypeCheckDiagnostics { /// basedpython: records that an extension or conformance declared in `module` supplied /// something inference relied on pub(crate) fn mark_extension_module_used(&mut self, module: File) { - self.used_extension_modules.insert(module); + self.uses_mut().extension_modules.insert(module); + } + + pub(crate) fn is_extension_module_used(&self, module: File) -> bool { + self.uses + .as_ref() + .is_some_and(|uses| uses.extension_modules.contains(&module)) + } + + /// basedpython: records that inference read every binding `symbol` has in `scope` + pub(crate) fn mark_scope_read(&mut self, scope: FileScopeId, symbol: ScopedSymbolId) { + self.uses_mut().scope_reads.insert((scope, symbol)); + } + + pub(crate) fn scope_reads(&self) -> impl Iterator { + self.uses + .iter() + .flat_map(|uses| uses.scope_reads.iter().copied()) + } + + fn uses_mut(&mut self) -> &mut InferenceUses { + self.uses.get_or_insert_with(Box::default) } pub(crate) fn shrink_to_fit(&mut self) { self.used_suppressions.shrink_to_fit(); - self.used_extension_modules.shrink_to_fit(); + if let Some(uses) = &mut self.uses { + uses.extension_modules.shrink_to_fit(); + uses.scope_reads.shrink_to_fit(); + } self.diagnostics.shrink_to_fit(); } @@ -4196,7 +4241,7 @@ impl TypeCheckDiagnostics { } pub(crate) fn is_empty(&self) -> bool { - !self.has_reports() && self.used_extension_modules.is_empty() + !self.has_reports() && self.uses.is_none() } /// whether anything was reported, suppressed or not. a recorded extension use is not a diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 33bcf654bf..7ea16e3101 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -71,7 +71,7 @@ use crate::types::class::{ }; use crate::types::constraints::{ConstraintSetBuilder, PathBounds, Solutions}; use crate::types::context::InferContext; -use crate::types::context_sensitive::{self, case_name_pattern_type}; +use crate::types::context_sensitive; use crate::types::dedicated::{django, pydantic}; use crate::types::deferred::{is_integer_operand, is_symbolic_operand}; use crate::types::definition_resolution::AttributeRead; @@ -3438,8 +3438,13 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let Some(case_name) = self.index.case_name(NodeKey::from_node(identifier)) else { return; }; - if case_name_pattern_type(self.db(), self.program_environment(), case_name).is_none() { - captures.push((alternative, identifier, case_name)); + match context_sensitive::resolve_case_name( + self.db(), + self.program_environment(), + case_name, + ) { + Some(member) => self.record_context_sensitive_qualifier(member), + None => captures.push((alternative, identifier, case_name)), } }); for (alternative, identifier, case_name) in captures { @@ -11410,6 +11415,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // claim; the plain `cast` never reaches a check in the first place if cast_kind.verifies_at_runtime() { self.report_erased_cast_argument(type_arg, value_ty, target); + self.record_checked_cast_spellings(value_ty, target, type_arg); } // an unsound plain `cast` is already the sharper report — a // non-overlapping one would only repeat that the value is not the @@ -14055,6 +14061,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { symbol_name, ) { + self.record_context_sensitive_qualifier(member); Place::bound(member.ty).into() } else { Place::Undefined.into() @@ -14071,6 +14078,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { && let Some(case_name) = self.index.case_name(NodeKey::from_node(name_node)) && let Some(member) = context_sensitive::resolve_case_name(db, env, case_name) { + self.record_context_sensitive_qualifier(member); Place::bound(member.ty).into() } else { Place::Undefined.into() @@ -14251,6 +14259,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let env = self.program_environment(); let is_class_body_global_fallback = source.is_class_body_global_fallback(); + self.record_whole_scope_read(&source.kind); + let place = match source.kind { PlaceLoadSourceKind::Bindings(bindings) => { let mut place = place_from_bindings_with_reachability_cache( @@ -14320,6 +14330,38 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + /// basedpython: records a read the semantic index has no record of — one that reads every + /// binding a scope makes rather than the bindings live at the reading expression. a name in + /// an annotation is evaluated lazily, and a name declared `global` is read from the module. + /// a nested scope's read of an enclosing one is not among them: the index records that + /// itself + fn record_whole_scope_read(&self, source: &PlaceLoadSourceKind<'db>) { + if !self.is_basedpython_file() { + return; + } + let db = self.db(); + match source { + PlaceLoadSourceKind::Bindings(bindings) if self.is_deferred() => { + for binding in bindings.clone() { + if let Some(definition) = binding.binding.definition() + && let Some(symbol) = definition.place(db).as_symbol() + { + self.context + .record_scope_read(definition.file_scope(db), symbol); + } + } + } + PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::ExplicitGlobalSymbol { + file, + name, + }) if *file == self.program_file() => { + self.context + .record_scope_read_of_name(ty_python_core::global_scope(db, *file), name); + } + _ => {} + } + } + /// Applies ty's convenience fallback for an unimported `reveal_type`. fn infer_unimported_reveal_type_fallback( &self, @@ -15070,7 +15112,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // extensions, this never shadows a declared member if self.is_basedpython_file() && fallback_place.place.is_undefined() - && let Some(bound) = receivers::resolve_receiver_attribute( + && let Some((declaring_scope, bound)) = receivers::resolve_receiver_attribute_in_scope( db, env, self.file(), @@ -15079,6 +15121,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { &attr.id, ) { + // the access reads the name, which no name expression spells + self.context + .record_scope_read_of_name(declaring_scope, &attr.id); fallback_place = Place::bound(bound).into(); } @@ -15769,6 +15814,51 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Some(returned) } + /// basedpython: records the bindings a checked cast's runtime check reads. the lowering + /// builds its predicate from the type test the cast shares with `is`, and falls back to + /// the shallow target the soundness checks use, so both are spelled here as it spells them + fn record_checked_cast_spellings( + &self, + value_ty: Type<'db>, + target: Type<'db>, + type_arg: &ast::Expr, + ) { + if !self.is_basedpython_file() { + return; + } + let db = self.db(); + let env = self.program_environment(); + let plan = crate::types::reified_infer::type_test_plan( + db, + env, + self.file(), + value_ty, + target, + Some(type_arg), + ); + for spelling in plan.spellings() { + self.context.record_spelling(spelling); + } + if let Some(spelling) = + crate::types::soundness::cast_check_plan(db, env, self.file(), target) + .and_then(|check| check.spelling()) + { + self.context.record_spelling(&spelling); + } + } + + /// basedpython: records the binding a context-sensitive name's lowering reads its + /// qualifier through — `Red` is written `Color.Red`, which reads `Color` although the + /// source never spells it + fn record_context_sensitive_qualifier( + &self, + member: context_sensitive::ContextSensitiveMember<'db>, + ) { + if let Some((scope, symbol)) = member.qualifier_binding() { + self.context.record_scope_read(scope, symbol); + } + } + /// basedpython: records every extension an extension-member resolution relied on, so /// the imports that make them applicable count as used fn record_extension_resolution(&self, resolution: &extensions::ExtensionMemberResolution<'db>) { @@ -16595,6 +16685,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { for module in plan.conformance_modules() { self.context.record_extension_module_use(module); } + for spelling in plan.spellings() { + self.context.record_spelling(spelling); + } if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = plan { let source = ruff_db::source::source_text(db, self.file()); self.report_erased_type_check( diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index 4c57181333..e61d4142aa 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -338,6 +338,25 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { |call| self.call_solution(call), ); + // basedpython: the guard a `raises` function is lowered behind names the declared + // exceptions as the module reaches them, so it reads the bindings it names them by + if function.raises.is_some() && self.is_basedpython_file() { + let db = self.db(); + let function_ty = + crate::types::binding_type(db, self.index.expect_single_definition(function)); + if let Some(target) = crate::types::exceptions::declared_raises_runtime_target( + db, + self.program_environment(), + self.file(), + function_ty, + ) { + self.context.record_spelling(&target.ceiling); + if let Some(resolved) = &target.resolved { + self.context.record_spelling(&resolved.expression); + } + } + } + // basedpython: a trailing-lambda block in a loop that captures a loop // variable is a late-binding trap unless its callee confines it // (`local` / `once`) — the type-aware complement to ruff's `B023` diff --git a/crates/ty_python_semantic/src/types/receivers.rs b/crates/ty_python_semantic/src/types/receivers.rs index 833197b563..67e4d55b33 100644 --- a/crates/ty_python_semantic/src/types/receivers.rs +++ b/crates/ty_python_semantic/src/types/receivers.rs @@ -97,7 +97,7 @@ fn bind_receiver<'db>( /// annotation, and a declaration has one type wherever it is visible. A scope /// that binds the name to anything else shadows it, the same way it would shadow /// the name in an ordinary load -pub(crate) fn resolve_receiver_attribute<'db>( +fn resolve_receiver_attribute<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index aef65d7c07..63ebb00850 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -829,6 +829,43 @@ impl ParametricIsPlan { } } + /// every spelling the plan writes, in any arm + pub(crate) fn spellings(&self) -> Vec<&RuntimeSpelling> { + fn rebuilt(target: &TargetSpelling) -> Option<&RuntimeSpelling> { + match target { + TargetSpelling::Rebuilt(spelling) => Some(spelling), + TargetSpelling::Written => None, + } + } + match self { + Self::Probe { target, .. } + | Self::Isinstance(target) + | Self::Identity(target) + | Self::Subclass(target) + | Self::Conformance { target, .. } + | Self::Equality { value: target, .. } => rebuilt(target).into_iter().collect(), + Self::ProtocolStructural(checks) => checks + .iter() + .flat_map(|check| match check { + ProtocolMemberCheck::Attribute { expected, .. } => vec![expected], + ProtocolMemberCheck::Method { params, ret, .. } => params + .iter() + .chain(ret) + .map(|(spelling, _)| spelling) + .collect(), + }) + .collect(), + Self::Union(arms) => arms.iter().flat_map(Self::spellings).collect(), + Self::Fold(_) + | Self::TokenEq(_) + | Self::ErasedTarget(_) + | Self::IsCallable + | Self::IsNone + | Self::Pattern(_) + | Self::Unresolved => Vec::new(), + } + } + /// every module whose conformance this plan answers from, in any arm pub(crate) fn conformance_modules(&self) -> Vec { match self { diff --git a/crates/ty_python_semantic/src/types/runtime_spelling.rs b/crates/ty_python_semantic/src/types/runtime_spelling.rs index 3c2a0b99b3..1a2dc66c8f 100644 --- a/crates/ty_python_semantic/src/types/runtime_spelling.rs +++ b/crates/ty_python_semantic/src/types/runtime_spelling.rs @@ -8,6 +8,10 @@ //! emitted as `_Kept`, or under another name where the module already has that one, and //! only the transpiler decides which. so a spelling keeps the names of those symbols //! apart, and whoever writes the spelling out spells them +//! +//! every binding of the module's global scope a spelling reads is kept apart the same way, +//! so the checker, which spells the same thing while it checks, can record that the +//! transpiled program reads it — an import read only by a lowering is still used use ruff_db::files::File; use ruff_python_ast::name::Name; @@ -31,6 +35,8 @@ pub struct RuntimeSpelling { #[derive(Debug, Clone, PartialEq, Eq)] enum SpellingPart { Text(String), + /// a binding of the module's global scope, written under its own name + Global(Name), /// a module-level `private` symbol, by the name the source declares it under ModulePrivate(Name), } @@ -51,13 +57,21 @@ impl RuntimeSpelling { /// [`Self::text`] instead: the module may declare a `private` symbol of that name /// without binding it, and the builtin is not that symbol pub(crate) fn global(db: &dyn Db, file: File, name: &Name) -> Self { - if private_symbols(db, file).contains(name) { - Self { - parts: vec![SpellingPart::ModulePrivate(name.clone())], - } + let part = if private_symbols(db, file).contains(name) { + SpellingPart::ModulePrivate(name.clone()) } else { - Self::text(name.as_str()) - } + SpellingPart::Global(name.clone()) + }; + Self { parts: vec![part] } + } + + /// every binding of the module's global scope the spelling reads, by the name the + /// source binds it under + pub(crate) fn global_reads(&self) -> impl Iterator { + self.parts.iter().filter_map(|part| match part { + SpellingPart::Global(name) | SpellingPart::ModulePrivate(name) => Some(name), + SpellingPart::Text(_) => None, + }) } pub(crate) fn push_str(&mut self, text: &str) { @@ -75,7 +89,9 @@ impl RuntimeSpelling { for part in &other.parts { match part { SpellingPart::Text(text) => self.push_str(text), - SpellingPart::ModulePrivate(_) => self.parts.push(part.clone()), + SpellingPart::Global(_) | SpellingPart::ModulePrivate(_) => { + self.parts.push(part.clone()); + } } } } @@ -103,6 +119,7 @@ impl RuntimeSpelling { for part in &self.parts { match part { SpellingPart::Text(text) => spelled.push_str(text), + SpellingPart::Global(name) => spelled.push_str(name), SpellingPart::ModulePrivate(name) => spelled.push_str(&module_private(name)), } } diff --git a/crates/ty_python_semantic/src/types/soundness.rs b/crates/ty_python_semantic/src/types/soundness.rs index ce0784dbdd..6b3b52ce23 100644 --- a/crates/ty_python_semantic/src/types/soundness.rs +++ b/crates/ty_python_semantic/src/types/soundness.rs @@ -62,6 +62,26 @@ impl CheckTarget { } } +impl CheckKind { + /// the spelling the check writes its target with + pub(crate) fn spelling(&self) -> RuntimeSpelling { + match self { + Self::Isinstance(target) => target.spelling(), + Self::Parametric { alias, .. } => alias.clone(), + } + } +} + +impl CastCheck { + /// the spelling the check writes its target with, when it writes one + pub(crate) fn spelling(&self) -> Option { + match self { + Self::Kind(kind) => Some(kind.spelling()), + Self::Protocol | Self::Unchecked | Self::Members(_) => None, + } + } +} + /// How a runtime soundness check validates a value against a target type. #[derive(Debug, Clone, PartialEq, Eq)] pub enum CheckKind { From 8bc0bdd0d0edb0ee2c11603169e523f8e3439974 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:16:10 +1000 Subject: [PATCH 20/47] add an `unused-import` lint for `.by` files with F401's rules, where an import also counts as used when the module it names declares an extension or a conformance that inference relied on --- crates/ty_python_core/src/use_def.rs | 24 +- .../mdtest/basedpython_unused_import.md | 793 ++++++++++++++++++ crates/ty_python_semantic/src/types.rs | 7 + .../src/types/diagnostic.rs | 69 ++ .../src/types/unused_imports.rs | 573 +++++++++++++ crates/ty_test/src/db.rs | 5 + 6 files changed, 1467 insertions(+), 4 deletions(-) create mode 100644 crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md create mode 100644 crates/ty_python_semantic/src/types/unused_imports.rs diff --git a/crates/ty_python_core/src/use_def.rs b/crates/ty_python_core/src/use_def.rs index 77efdb903a..6523883f1a 100644 --- a/crates/ty_python_core/src/use_def.rs +++ b/crates/ty_python_core/src/use_def.rs @@ -723,7 +723,8 @@ enum DefinitionEntry<'db> { Unused(Definition<'db>), Used(Definition<'db>), Undefined, - Deleted, + /// the place a `del`, or the end of an `except … as` clause, unbound + Deleted(ScopedPlaceId), } impl<'db> DefinitionEntry<'db> { @@ -733,7 +734,7 @@ impl<'db> DefinitionEntry<'db> { | Self::Unused(definition) | Self::Used(definition) => DefinitionState::Defined(definition), Self::Undefined => DefinitionState::Undefined, - Self::Deleted => DefinitionState::Deleted, + Self::Deleted(_) => DefinitionState::Deleted, } } } @@ -991,7 +992,22 @@ impl<'db> UseDefMap<'db> { DefinitionEntry::Used(definition) => Some((id, definition, true)), DefinitionEntry::DeclarationPart(_) | DefinitionEntry::Undefined - | DefinitionEntry::Deleted => None, + | DefinitionEntry::Deleted(_) => None, + }) + } + + /// every place this scope unbinds, with the point in the scope's history it does: a + /// `del`, the end of an `except … as` clause, and the end of a block that declared a + /// name. ordered with [`Self::definitions_with_usage`] by the ids both hand out + pub fn deletions(&self) -> impl Iterator + '_ { + self.all_definitions + .iter_enumerated() + .filter_map(|(id, entry)| match entry { + DefinitionEntry::Deleted(place) => Some((id, place)), + DefinitionEntry::DeclarationPart(_) + | DefinitionEntry::Unused(_) + | DefinitionEntry::Used(_) + | DefinitionEntry::Undefined => None, }) } @@ -2647,7 +2663,7 @@ impl<'db> UseDefMapBuilder<'db> { } pub(super) fn delete_binding(&mut self, place: ScopedPlaceId) { - let def_id = self.push_definition(DefinitionEntry::Deleted); + let def_id = self.push_definition(DefinitionEntry::Deleted(place)); let pending = self.pending_reachability.current; let place_state = pending_place_state_mut(place, &mut self.symbol_states, &mut self.member_states); diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md new file mode 100644 index 0000000000..218253c073 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md @@ -0,0 +1,793 @@ +# unused imports + +`unused-import` reports an import in a `.by` file that nothing uses. an import is used when a name +it binds is read, and also when the module it names declares an `extension` or a conformance that +supplied something the file relies on — the extension's members are only applicable in files that +import its module, so the import is what makes them reachable even when its name is never read. + +the mdtest corpus turns this rule off, so the tests below turn it back on. + +```toml +[environment] +python-version = "3.12" + +[rules] +unused-import = "warn" +``` + +## an import nothing reads is reported + +```by +import json # error: [unused-import] "`json` imported but unused" +from os import path # error: [unused-import] "`os.path` imported but unused" +from collections import OrderedDict, deque # error: [unused-import] "`collections.deque` imported but unused" + +OrderedDict() +``` + +## an import whose name is read is not + +a read anywhere in the file counts, including from a nested function and from an annotation + +```by +import json +from collections import OrderedDict + +def dump(value: OrderedDict[str, int]) -> str: + return json.dumps(value) +``` + +## an import used only through an extension + +`colours` declares an extension of `Enum`, and `main` reads its member without ever reading the name +`colours` + +`colours.by`: + +```by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +`main.by`: + +```by +from enum import Enum +import colours + +class Colour(Enum): + RED = 1 + +def describe(colour: Colour) -> str: + return colour.lower_name + +describe(Colour.RED) +``` + +`colours` itself reads `Enum` only in the `extension Enum:` header, which names the type it extends + +## an extension module whose extension nothing uses is reported + +`colours.by`: + +```by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +`main.by`: + +```by +import colours # error: [unused-import] +``` + +## an import used only through a conformance + +`render` asks for a `Show`, and `"hi"` is one only because `conforming` declares that `str` +conforms. nothing in `main` names `conforming` + +`shows.by`: + +```by +protocol Show: + def show(self) -> str + +def render(value: Show) -> str: + return value.show() +``` + +`conforming.by`: + +```by +from shows import Show + +extension str(Show): + override def show(self) -> str: + return self +``` + +`main.by`: + +```by +from shows import render +import conforming + +render("hi") +``` + +## an import used only through a protocol extension reached by conformance + +`loud` extends the protocol itself, and a conforming `str` reaches the member through the +conformance — so both imports are used, and neither name is read + +`shows.by`: + +```by +protocol Show: + def show(self) -> str +``` + +`loud.by`: + +```by +from shows import Show + +extension Show: + def shout(self) -> str: + return self.show().upper() +``` + +`conforming.by`: + +```by +from shows import Show + +extension str(Show): + override def show(self) -> str: + return self +``` + +`main.by`: + +```by +import loud +import conforming + +def greet() -> str: + return "hi".shout() +``` + +## an import used only for a default a conformance takes from a protocol extension + +`str` answers `Show`'s requirement with the default `defaults` declares on the protocol, so the +conformance's witness table takes that default, and the import of `defaults` is what makes it +applicable + +`shows.by`: + +```by +protocol Show: + def show(self) -> str +``` + +`defaults.by`: + +```by +from shows import Show + +extension Show: + def show(self) -> str: + return "?" +``` + +`conforming.by`: + +```by +from shows import Show +import defaults + +extension str(Show): ... +``` + +## an import that brings a second conformance of the same pair into view + +the conflict is reported where the second conformance is declared, and the import that brings the +first into view is what causes it — so it is used until one of the two goes + +`shows.by`: + +```by +protocol Show: + def show(self) -> str +``` + +`first.by`: + +```by +from shows import Show + +extension str(Show): + override def show(self) -> str: + return self +``` + +`second.by`: + +```by +from shows import Show +import first + +# error: [invalid-conformance] +extension str(Show): + override def show(self) -> str: + return self +``` + +## an import used only by a type test answered from the conformance registry + +`value is Show` asks the registry, because a conformance to `Show` is in scope. without the import +the test is a plain `isinstance` against a protocol that does not allow one + +`shows.by`: + +```by +protocol Show: + def show(self) -> str +``` + +`conforming.by`: + +```by +from shows import Show + +extension str(Show): + override def show(self) -> str: + return self +``` + +`main.by`: + +```by +from shows import Show +import conforming + +def check(value: object) -> bool: + return value is Show +``` + +## an import used only through an operator an extension supplies + +`money.by`: + +```by +class Money: + def __init__(self, cents: int) -> None: + self.cents = cents +``` + +`arithmetic.by`: + +```by +from money import Money + +extension Money: + def __add__(self, other: Money) -> Money: + return Money(self.cents + other.cents) +``` + +`main.by`: + +```by +from money import Money +import arithmetic + +def total(a: Money, b: Money) -> Money: + return a + b +``` + +## an import used only through a unary operator or a comparison an extension supplies + +`money.by`: + +```by +class Money: + def __init__(self, cents: int) -> None: + self.cents = cents +``` + +`negation.by`: + +```by +from money import Money + +extension Money: + def __neg__(self) -> Money: + return Money(-self.cents) +``` + +`ordering.by`: + +```by +from money import Money + +extension Money: + def __lt__(self, other: Money) -> bool: + return self.cents < other.cents +``` + +`main.by`: + +```by +from money import Money +import negation +import ordering + +def refund(a: Money) -> Money: + return -a + +def cheaper(a: Money, b: Money) -> bool: + return a < b +``` + +## an import used only through a conversion an extension supplies + +`ids.by`: + +```by +class UserId: + pass + +def register(user: UserId) -> None: ... +``` + +`parsing.by`: + +```by +from ids import UserId + +extension UserId: + class def __from__(cls, value: int) -> UserId: + return UserId() +``` + +`main.by`: + +```by +from ids import register +import parsing + +register(3) +``` + +## any import of the module grants its extensions + +a `from` import of a name the file never reads still makes the module's extensions applicable, so it +is used when one of them is + +`colours.by`: + +```by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() + +def unrelated() -> None: ... +``` + +`main.by`: + +```by +from enum import Enum +from colours import unrelated + +class Colour(Enum): + RED = 1 + +def describe(colour: Colour) -> str: + return colour.lower_name +``` + +## an import inside a function grants the whole file + +an import makes a module's extensions applicable wherever in the file it is written, so one inside a +function that is never called is still used by an extension member read anywhere else + +`colours.by`: + +```by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +`main.by`: + +```by +from enum import Enum + +class Colour(Enum): + RED = 1 + +def setup() -> None: + import colours + +def describe(colour: Colour) -> str: + return colour.lower_name +``` + +## an extension member read in a receiver block + +a bare name inside a trailing lambda block whose callback declares a receiver reads the receiver's +members, and an extension's members with them + +`tags.by`: + +```by +class Tag: + def __init__(self) -> None: + self.children: list[str] = [] +``` + +`paragraphs.by`: + +```by +from tags import Tag + +extension Tag: + def p(self, text: str) -> None: + self.children.append(text) +``` + +`main.by`: + +```by +from tags import Tag +import paragraphs + +def page(fn: Tag.() -> None) -> Tag: + tag = Tag() + fn(tag) + return tag + +page: + p("hello") +``` + +## a receiver callable read as a method + +`x.fn()` reads `fn`, a receiver callable in scope, when `x` has no member of that name + +`render.by`: + +```by +def describe(value: int) -> str: + return str(value) + +renderer: int.() -> str = describe +``` + +`main.by`: + +```by +from render import renderer + +def show(x: int) -> str: + return x.renderer() +``` + +## explicit re-exports are not reported + +`import a as a`, `from a import b as b` and basedpython's `from a export b` say the name is part of +this module's interface + +```by +import json as json +from os import path as path +from collections export OrderedDict +``` + +## a name listed in `__all__` is exported + +```by +import json +from os import path + +__all__ = ["json", "path"] +``` + +## a name listed in an `__all__` the checker cannot read in full + +`__all__` built by concatenating something other than a list of names is too dynamic for the checker +to know every name in it, but a name spelled out in it is still exported + +```by +import json + +extra: list[str] = [] +__all__ = ["json"] + extra +``` + +## a file that does not parse is not checked + +a read the parser could not recover is invisible, so an import in a file with a syntax error is +never reported + +```by +import json + +x = 1 + # error: [invalid-syntax] +``` + +## a name another module's `__all__` adds to this one's + +the checker follows `__all__ += helpers.__all__` to the names `helpers` lists, so a name imported +from it is exported + +`helpers.by`: + +```by +def helper() -> None: ... + +__all__ = ["helper"] +``` + +`main.by`: + +```by +import helpers +from helpers import helper + +__all__ = ["helpers"] +__all__ += helpers.__all__ +``` + +## a `__future__` import is a directive + +```by +from __future__ import annotations +``` + +## a rebound name is reported only at its last binding + +only the binding a name is left holding at the end of its scope is reported. an import the name is +rebound after is `F811`'s concern + +```by +import json +import json as codec + +json = codec +``` + +## a read of an earlier binding counts for a later one + +```by +import os + +os.getcwd() + +import os +``` + +## the last of several unread bindings is reported + +```by +import json +import json # error: [unused-import] +``` + +## a `del` starts the name afresh + +a read before the `del` belongs to the binding it deletes, so it does not use an import made after + +```by +import os + +os.getcwd() +del os + +import os # error: [unused-import] +``` + +## a deleted import is not reported + +`del` reads the binding it deletes, and leaves the name unbound + +```by +import os + +del os +``` + +## a name declared `global` binds in the module + +an import in a function that declares the name `global` binds the module's name, and is not reported +there. the module's own import is reported when nothing reads the name + +```by +import json # error: [unused-import] + +def reload() -> None: + global json + import json +``` + +## a read through a `global` declaration uses the module's import + +```by +import json + +def dump() -> str: + global json + return json.dumps(1) +``` + +## an import in a class body is not reported + +```by +class Config: + import json +``` + +## a submodule import binds the top-level package + +`import os.path` binds `os`, so a read of `os` uses it + +```by +import os.path + +def cwd() -> str: + return os.getcwd() +``` + +## an unread submodule import is reported by its full name + +```by +import os.path # error: [unused-import] "`os.path` imported but unused" +``` + +## two submodule imports bind one name + +`import os.path` and `import email.mime` bind `os` and `email`; a second submodule of the same +package rebinds the same name, so a read through either uses both + +```by +import email.mime +import email.parser + +def parser() -> object: + return email.parser +``` + +## a read earlier in a loop body uses an import later in it + +the second time round the loop, the read sees the import the first pass made. the checker cannot +tell that the first pass always takes the other branch, so it reports the read too + +```by +def poll() -> None: + for attempt in range(2): + if attempt: + os.getcwd() # error: [possibly-unresolved-reference] + else: + import os +``` + +## a name in a `raises` clause is read + +```by +from json import JSONDecodeError + +def parse(text: str) -> int raises JSONDecodeError: + raise JSONDecodeError("bad", text, 0) +``` + +## an import read in an annotation in a `TYPE_CHECKING` block + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections import OrderedDict + +def first(value: OrderedDict[str, int]) -> int: + return 0 +``` + +## an unread import in a `TYPE_CHECKING` block is reported + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections import OrderedDict # error: [unused-import] +``` + +## an annotation may read an import written after it + +annotations are evaluated lazily, so a name in one resolves to every binding the scope makes + +```by +def first(value: OrderedDict[str, int]) -> int: + return 0 + +from collections import OrderedDict +``` + +## a string in a type position reads no name + +in a `.by` file a string in a type position is a literal type, never a forward reference, so it uses +nothing + +```by +from collections import OrderedDict # error: [unused-import] + +def name(value: "OrderedDict") -> None: ... +``` + +## an import that tests for a module's availability + +an import in the body of a `try` that handles `ImportError` is usually there to find out whether the +module is installed. it is still reported, and says how to ask without importing + +```by +try: + import tomllib # snapshot +except ImportError: + pass +``` + +```snapshot +warning[unused-import]: `tomllib` imported but unused + --> src/mdtest_snippet.by:2:12 + | +2 | import tomllib # snapshot + | ^^^^^^^ +help: To test whether a module is available, use `importlib.util.find_spec` instead of importing it +``` + +## an unused import in a package's `__init__` may be a re-export + +a package's `__init__.by` is what importing the package runs, so an import there may be how the +package exports the name. it is still reported, and one of the project's own modules says how to +make the re-export explicit + +`pkg/models.by`: + +```by +class Widget: + pass +``` + +`pkg/__init__.by`: + +```by +import json # error: [unused-import] +from .models import Widget # snapshot +``` + +```snapshot +warning[unused-import]: `.models.Widget` imported but unused + --> src/pkg/__init__.by:2:21 + | +2 | from .models import Widget # snapshot + | ^^^^^^ +help: If this is a re-export, write it as `from … export …`, as a redundant alias, or list it in `__all__` +``` + +## an import in a function is reported in its scope + +```by +def load() -> None: + import json # error: [unused-import] +``` diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 3bc25d653e..5e1b4512ed 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -259,6 +259,7 @@ mod typevar; pub(crate) mod underscore_names; mod unpacker; mod unsafe_union; +mod unused_imports; mod variance; pub(crate) mod visibility; mod visitor; @@ -305,6 +306,12 @@ pub(crate) fn check_types_with( } } + // basedpython: an import may be used only through an extension it makes applicable, + // which only the whole file's inference knows + let mut unused_imports = TypeCheckDiagnostics::default(); + unused_imports::report_unused_imports(db, file, &diagnostics, &mut unused_imports); + diagnostics.extend(&unused_imports); + diagnostics.extend_diagnostics( index .semantic_syntax_errors() diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 9360948594..edb3e5282b 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -213,6 +213,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&OPTIONAL_OBJECT_CONVERSION); registry.register_lint(&BOOL_AS_INT); registry.register_lint(&UNUSED_RETURN_VALUE); + registry.register_lint(&UNUSED_IMPORT); registry.register_lint(&MISSING_CONTEXT_ARGUMENT); registry.register_lint(&AMBIGUOUS_CONTEXT_ARGUMENT); registry.register_lint(&REIFIED_WITHOUT_RECEIVER); @@ -2862,6 +2863,74 @@ declare_lint! { } } +declare_lint! { + /// ## What it does + /// Checks a basedpython file for an import that nothing uses: no name it binds + /// is read, and no `extension` or conformance it brings into scope supplies + /// anything the file relies on. + /// + /// ## Why is this bad? + /// An unused import costs time when the module is imported, can create an + /// import cycle, and tells a reader the module depends on something it does + /// not. + /// + /// The extension members a module declares are only applicable in the files + /// that import it, so an import can be used without its name ever being read. + /// This rule counts that as a use, which a linter working one file at a time + /// cannot — so in a `.by` file it takes the place of buff's `F401`, which + /// leaves `.by` files alone. + /// + /// Otherwise the rule is `F401`'s: + /// + /// - a read of a binding counts for every later binding of that name in the + /// scope. An import nothing reads before another import rebinds the name + /// is reported too, as `F811` reports the redefinition, so one round of + /// fixes removes every one. An import inside a class body is never + /// reported + /// - `import a as a`, `from a import b as b` and `from a export b` are + /// re-exports, a name listed in `__all__` is exported, and + /// `from __future__ import …` is a directive — none of them is reported + /// - in a package's `__init__.by`, and inside a `try` that handles + /// `ImportError`, an unused import is reported without a fix: it may be a + /// re-export, or a test for whether the module is installed + /// + /// ## Examples + /// ```by + /// # colours.by + /// from enum import Enum + /// + /// extension Enum: + /// let lower_name: str + /// get() = self.name.lower() + /// ``` + /// + /// ```by + /// # main.by + /// import json # error: `json` imported but unused + /// import colours # ok — the extension it declares is used below + /// from enum import Enum + /// + /// class Colour(Enum): + /// RED = 1 + /// + /// def describe(colour: Colour) -> str: + /// return colour.lower_name + /// ``` + /// + /// Use instead: + /// + /// ```by + /// import colours + /// from enum import Enum + /// ``` + pub(crate) static UNUSED_IMPORT = { + summary: "detects an import that nothing uses", + status: LintStatus::stable("0.0.1-beta.2"), + default_level: Level::Warn, + ty_compat: TyCompat::BasedPython, + } +} + declare_lint! { /// ## What it does /// Checks for a call, written as a statement of its own, whose result is diff --git a/crates/ty_python_semantic/src/types/unused_imports.rs b/crates/ty_python_semantic/src/types/unused_imports.rs new file mode 100644 index 0000000000..e929c26720 --- /dev/null +++ b/crates/ty_python_semantic/src/types/unused_imports.rs @@ -0,0 +1,573 @@ +//! basedpython: `unused-import`, the checker's half of pyflakes' unused-import rule for +//! `.by` files +//! +//! whether an import in a `.by` file is used is a question about types as well as names. +//! `import colours` may be written only for the `extension Enum:` block `colours` declares, +//! and then nothing reads the name `colours` — the import is used because a member access +//! resolved through an extension it makes applicable. the linter answers per file and has no +//! types, so the rule lives here, where the file's inference has recorded every extension +//! and conformance it relied on, and buff's `F401` leaves `.by` files alone +//! +//! everything else is `F401`'s rule, read in the checker's model of the file: +//! +//! - a read of a binding counts for every later binding of the name in the scope. an import +//! nothing reads before another import rebinds the name is reported too, which is the +//! redefinition `F811` reports, so that one round of fixes removes all of them +//! - a `global` or `nonlocal` name binds in another scope, and is never reported +//! - `import a as a`, `from a import b as b` and `from a export b` re-export on purpose +//! - `from __future__ import …` is a compiler directive rather than a binding +//! - a module-scope name listed in `__all__` is exported, which is a use +//! - in a package's `__init__.by` an unused import may be an intended re-export, so it is +//! reported without a fix, and a first-party one says how to make the re-export explicit +//! - an import inside a `try` that handles `ImportError` probes for the module, so it is +//! reported without a fix, and says how to probe without importing + +use ruff_db::files::File; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_python_ast::identifier::Identifier; +use ruff_python_ast::name::Name; +use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; +use ruff_python_ast::{self as ast}; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::FxHashSet; +use ty_module_resolver::{ImportingFile, ModuleName, resolve_module}; +use ty_python_core::definition::{Definition, DefinitionCategory, DefinitionKind}; +use ty_python_core::place::ScopedPlaceId; +use ty_python_core::scope::{FileScopeId, ScopeKind}; +use ty_python_core::symbol::ScopedSymbolId; +use ty_python_core::{ProgramFile, ScopedDefinitionId, SemanticIndex, semantic_index}; + +use crate::dunder_all::dunder_all_names; +use crate::lint::LintId; +use crate::place::builtins_symbol; +use crate::reachability::is_reachable; +use crate::types::context::InferContext; +use crate::types::definition_resolution::user_visible_definitions; +use crate::types::diagnostic::{TypeCheckDiagnostics, UNUSED_IMPORT}; +use crate::types::extensions::{extension_granting_imports, extensions_in_module}; +use crate::types::{ProgramEnvironment, Type}; +use crate::{Db, FxIndexMap}; + +/// report every unused import in `file`. `inferred` holds the diagnostics of the whole file's +/// inference, whose record of the extensions it relied on decides the imports no name is +/// read through +pub(crate) fn report_unused_imports( + db: &dyn Db, + file: ProgramFile<'_>, + inferred: &TypeCheckDiagnostics, + diagnostics: &mut TypeCheckDiagnostics, +) { + let source_file = file.file(db); + if !source_file.source_type(db).is_basedpython() { + return; + } + if db + .rule_selection(source_file) + .get(LintId::of(&UNUSED_IMPORT)) + .is_none() + { + return; + } + let env = ProgramEnvironment::from_file(file); + let parsed = parsed_module(db, file.python_file(db)).load(db); + // a file that does not parse may have lost a read to the parser's recovery, and `F401` + // does not check one either + if parsed.has_syntax_errors() { + return; + } + + let index = semantic_index(db, file); + let uses = Uses::new(db, file, index, &parsed, inferred); + let context = ImportContext::new(db, file, &env, &parsed); + + for scope in index.scope_ids() { + let file_scope = scope.file_scope_id(db); + let kind = index.scope(file_scope).kind(); + if !matches!(kind, ScopeKind::Module | ScopeKind::Function) { + continue; + } + let unused = unused_in_scope(db, index, file_scope, &parsed, &uses, source_file); + if unused.is_empty() { + continue; + } + let infer_context = InferContext::new(db, &env, scope, source_file, file, &parsed); + for import in &unused { + import.report(&infer_context, &context, kind == ScopeKind::Module); + } + diagnostics.extend(&infer_context.finish()); + } +} + +/// every read of a binding the file makes, from both the semantic index and inference +struct Uses<'db> { + /// the definitions a name expression, a nested scope or a loop reads + definitions: FxHashSet>, + /// the imports whose module supplied an extension or a conformance inference relied on + granting: FxHashSet>, + /// names inference read as every binding their scope makes, rather than at a position + /// the semantic index records: a receiver callable read as a method, and a name in an + /// annotation, which is evaluated lazily + scope_reads: FxHashSet<(FileScopeId, ScopedSymbolId)>, + /// module-scope names read as a whole rather than at a position: an `__all__` entry, + /// and the type an `extension` header names + module_names: FxHashSet, +} + +impl<'db> Uses<'db> { + fn new( + db: &'db dyn Db, + file: ProgramFile<'db>, + index: &'db SemanticIndex<'db>, + parsed: &ParsedModuleRef, + inferred: &TypeCheckDiagnostics, + ) -> Self { + let source_file = file.file(db); + + let read = index.scope_ids().flat_map(|scope| { + index + .use_def_map(scope.file_scope_id(db)) + .definitions_with_usage() + .filter_map(|(_, definition, is_used)| is_used.then_some(definition)) + }); + // a read of a synthetic definition reads the definitions it stands for + let mut definitions: FxHashSet> = + user_visible_definitions(db, read).into_iter().collect(); + // a read of a loop's header reads every binding live at the top of the loop, which + // is how a use earlier in the body reaches a binding later in it + for scope in index.scope_ids() { + let use_def = index.use_def_map(scope.file_scope_id(db)); + for (_, definition, is_used) in use_def.definitions_with_usage() { + if !is_used { + continue; + } + let DefinitionKind::LoopHeader(header) = definition.kind(db) else { + continue; + }; + let loop_header = use_def.loop_header(header.loop_header_id()); + for live in loop_header.bindings_for_place(header.place()) { + if is_reachable(db, use_def, live.reachability_constraint()) + && let Some(binding) = use_def.definition(live.binding()).definition() + { + definitions.insert(binding); + } + } + } + } + + let granting = extension_granting_imports(db, source_file) + .iter() + .filter(|grant| inferred.is_extension_module_used(grant.module)) + .map(|grant| grant.import) + .collect(); + + let mut module_names: FxHashSet = dunder_all_names(db, file) + .into_iter() + .flatten() + .cloned() + .collect(); + let mut literal_all = DunderAllLiterals::default(); + literal_all.visit_body(parsed.suite()); + module_names.extend(literal_all.names); + // an extension resolves the type it names in its module's globals, as a whole — + // see `extended_class` + module_names.extend( + extensions_in_module(db, source_file) + .iter() + .map(|extension| extension.name(db).clone()), + ); + + Self { + definitions, + granting, + scope_reads: inferred.scope_reads().collect(), + module_names, + } + } + + fn reads(&self, definition: Definition<'db>) -> bool { + self.definitions.contains(&definition) || self.granting.contains(&definition) + } +} + +/// an import binding that nothing reads +struct UnusedImport { + /// the statement the binding is one alias of + statement: TextRange, + /// the name the import makes available, as F401 spells it: `a.b`, `a.b.c` for + /// `from a.b import c`, `.c` for `from . import c` + qualified_name: String, + /// the module the import names, for deciding whether it is first party + module: Option, + /// the bound name, the range the diagnostic is reported on + binding: TextRange, +} + +fn unused_in_scope<'db>( + db: &'db dyn Db, + index: &'db SemanticIndex<'db>, + scope: FileScopeId, + parsed: &ParsedModuleRef, + uses: &Uses<'db>, + file: File, +) -> Vec { + let is_stub = file.is_stub(db); + let place_table = index.place_table(scope); + let use_def = index.use_def_map(scope); + + // what happens to each name over the scope, in the order the scope does it: a binding, + // or a `del` (or the end of an `except … as` clause) that unbinds it + let mut history: FxIndexMap< + ScopedSymbolId, + Vec<(ScopedDefinitionId, Option>)>, + > = FxIndexMap::default(); + for (id, definition, _) in use_def.definitions_with_usage() { + let ScopedPlaceId::Symbol(symbol) = definition.place(db) else { + continue; + }; + let kind = definition.kind(db); + // a definition the source does not spell — a loop header, the submodule binding + // `from .x import y` makes in a package — stands for something written elsewhere + if !kind.is_user_visible() || matches!(kind, DefinitionKind::ImportFromSubmodule(_)) { + continue; + } + // a bare annotation declares a type and binds nothing + if matches!( + kind.category(is_stub, parsed), + DefinitionCategory::Declaration + ) { + continue; + } + history + .entry(symbol) + .or_default() + .push((id, Some(definition))); + } + for (id, place) in use_def.deletions() { + if let ScopedPlaceId::Symbol(symbol) = place + && let Some(events) = history.get_mut(&symbol) + { + events.push((id, None)); + } + } + + let mut unused = Vec::new(); + for (symbol_id, mut events) in history { + let symbol = place_table.symbol(symbol_id); + if symbol.is_global() || symbol.is_nonlocal() { + continue; + } + events.sort_unstable_by_key(|(id, _)| *id); + // the bindings since the name was last unbound: the one it is left holding, and the + // ones it shadows. a read of any of them counts for the last, as it does for `F401` + let since_unbound = match events.iter().rposition(|(_, event)| event.is_none()) { + Some(deletion) => &events[deletion + 1..], + None => &events[..], + }; + // a name left unbound at the end of the scope holds no import + let Some(&(_, Some(live))) = since_unbound.last() else { + continue; + }; + let Some(import) = reportable_import(db, live, parsed) else { + continue; + }; + if since_unbound + .iter() + .filter_map(|(_, definition)| *definition) + .any(|definition| uses.reads(definition)) + { + continue; + } + if uses.scope_reads.contains(&(scope, symbol_id)) { + continue; + } + if scope.is_global() && uses.module_names.contains(symbol.name()) { + continue; + } + unused.push(import); + } + unused.sort_unstable_by_key(|import| import.binding.start()); + unused +} + +/// `definition` as an import F401 reports when nothing reads it: not a `__future__` +/// directive, and not an explicit re-export +fn reportable_import<'db>( + db: &'db dyn Db, + definition: Definition<'db>, + parsed: &ParsedModuleRef, +) -> Option { + match definition.kind(db) { + DefinitionKind::Import(import) => { + let statement = import.import(parsed); + let alias = import.alias(parsed); + if alias + .asname + .as_ref() + .is_some_and(|asname| asname.id == alias.name.id) + { + return None; + } + Some(UnusedImport { + statement: statement.range(), + qualified_name: alias.name.to_string(), + module: (!alias.is_resource) + .then(|| ModuleName::new(&alias.name)) + .flatten(), + binding: alias.identifier(), + }) + } + DefinitionKind::ImportFrom(import) => { + let statement = import.import(parsed); + let alias = import.alias(parsed); + if statement.module.as_deref() == Some("__future__") + && statement.level == 0 + && !statement.is_lazy + { + return None; + } + if statement.is_export + || alias + .asname + .as_ref() + .is_some_and(|asname| asname.id == alias.name.id) + { + return None; + } + let mut qualified_name = ".".repeat(statement.level as usize); + if let Some(module) = &statement.module { + qualified_name.push_str(module); + qualified_name.push('.'); + } + qualified_name.push_str(&alias.name); + Some(UnusedImport { + statement: statement.range(), + qualified_name, + module: statement + .module + .as_deref() + .and_then(ModuleName::new) + .filter(|_| statement.level == 0), + binding: alias.identifier(), + }) + } + _ => None, + } +} + +/// what the diagnostics of one file share: whether it is a package's `__init__`, and which +/// import statements probe for their module +struct ImportContext<'db> { + db: &'db dyn Db, + file: ProgramFile<'db>, + in_init: bool, + /// the start of every import statement written in the body of a `try` that handles + /// `ImportError` or `ModuleNotFoundError` + probing: FxHashSet, +} + +impl<'db> ImportContext<'db> { + fn new( + db: &'db dyn Db, + file: ProgramFile<'db>, + env: &ProgramEnvironment<'db>, + parsed: &ParsedModuleRef, + ) -> Self { + let source_file = file.file(db); + // a package's `__init__.by` is the module importing the package runs; its stub + // re-exports only what it spells as one, as any stub does + let in_init = !source_file.is_stub(db) + && source_file + .path(db) + .as_system_path() + .and_then(|path| path.file_stem()) + .is_some_and(|stem| stem == "__init__"); + let import_errors: Vec> = ["ImportError", "ModuleNotFoundError"] + .into_iter() + .filter_map(|name| { + builtins_symbol(db, env, name) + .place + .ignore_possibly_undefined() + }) + .collect(); + let model = crate::SemanticModel::new(db, file); + let mut probes = ImportProbes { + model: &model, + import_errors: &import_errors, + depth: 0, + probing: FxHashSet::default(), + }; + probes.visit_body(parsed.suite()); + Self { + db, + file, + in_init, + probing: probes.probing, + } + } + + /// whether `module` is the project's own code + fn is_first_party(&self, import: &UnusedImport) -> bool { + let db = self.db; + let Some(module) = &import.module else { + // a relative import names the package it is written in + return import.qualified_name.starts_with('.'); + }; + let file = self.file.file(db); + resolve_module( + db, + ImportingFile::File(file, self.file.resolver_environment(db)), + module, + ) + .and_then(|module| module.search_path(db)) + .is_some_and(ty_module_resolver::SearchPath::is_first_party) + } +} + +impl UnusedImport { + fn report(&self, context: &InferContext<'_, '_>, imports: &ImportContext<'_>, at_module: bool) { + let Some(builder) = context.report_lint(&UNUSED_IMPORT, self.binding) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{}` imported but unused", + self.qualified_name + )); + if let Some(annotation) = diagnostic.primary_annotation_mut() { + annotation.push_tag(ruff_db::diagnostic::DiagnosticTag::Unnecessary); + } + if imports.probing.contains(&self.statement.start()) { + diagnostic.help( + "To test whether a module is available, use `importlib.util.find_spec` \ + instead of importing it", + ); + } else if imports.in_init && at_module && imports.is_first_party(self) { + diagnostic.help( + "If this is a re-export, write it as `from … export …`, as a redundant alias, \ + or list it in `__all__`", + ); + } + } +} + +/// finds the import statements written in the body of a `try` that handles `ImportError` or +/// `ModuleNotFoundError` — an import made to find out whether the module is there +struct ImportProbes<'a, 'db> { + model: &'a crate::SemanticModel<'db>, + import_errors: &'a [Type<'db>], + /// how many enclosing `try` bodies handle an import error + depth: usize, + probing: FxHashSet, +} + +impl ImportProbes<'_, '_> { + fn handles_import_error(&self, handlers: &[ast::ExceptHandler]) -> bool { + handlers.iter().any(|handler| { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + let Some(caught) = handler.type_.as_deref() else { + return false; + }; + let caught: &[ast::Expr] = match caught { + ast::Expr::Tuple(tuple) => &tuple.elts, + single => std::slice::from_ref(single), + }; + caught.iter().any(|element| { + crate::HasType::inferred_type(element, self.model) + .is_some_and(|ty| self.import_errors.contains(&ty)) + }) + }) + } +} + +impl<'ast> StatementVisitor<'ast> for ImportProbes<'_, '_> { + fn visit_stmt(&mut self, stmt: &'ast ast::Stmt) { + match stmt { + ast::Stmt::Import(_) | ast::Stmt::ImportFrom(_) => { + if self.depth > 0 { + self.probing.insert(stmt.start()); + } + } + ast::Stmt::Try(try_stmt) => { + let probes = self.handles_import_error(&try_stmt.handlers); + self.depth += usize::from(probes); + self.visit_body(&try_stmt.body); + self.depth -= usize::from(probes); + for handler in &try_stmt.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + self.visit_body(&handler.body); + } + self.visit_body(&try_stmt.orelse); + self.visit_body(&try_stmt.finalbody); + } + _ => walk_stmt(self, stmt), + } + } +} + +/// the string literals listed in the module's `__all__`, wherever its definition is too +/// dynamic for the checker's own reading of it to give a set of names +#[derive(Default)] +struct DunderAllLiterals { + names: Vec, +} + +impl DunderAllLiterals { + fn extend_from(&mut self, value: &ast::Expr) { + match value { + ast::Expr::List(ast::ExprList { elts, .. }) + | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { + for element in elts { + if let ast::Expr::StringLiteral(literal) = element { + self.names.push(Name::new(literal.value.to_str())); + } + } + } + ast::Expr::BinOp(ast::ExprBinOp { left, right, .. }) => { + self.extend_from(left); + self.extend_from(right); + } + _ => {} + } + } +} + +fn is_dunder_all(target: &ast::Expr) -> bool { + matches!(target, ast::Expr::Name(name) if name.id == "__all__") +} + +impl<'ast> StatementVisitor<'ast> for DunderAllLiterals { + fn visit_stmt(&mut self, stmt: &'ast ast::Stmt) { + match stmt { + ast::Stmt::Assign(assign) if assign.targets.iter().any(is_dunder_all) => { + self.extend_from(&assign.value); + } + ast::Stmt::AnnAssign(assign) if is_dunder_all(&assign.target) => { + if let Some(value) = &assign.value { + self.extend_from(value); + } + } + ast::Stmt::AugAssign(assign) if is_dunder_all(&assign.target) => { + self.extend_from(&assign.value); + } + ast::Stmt::Expr(ast::StmtExpr { value, .. }) => { + if let ast::Expr::Call(call) = value.as_ref() + && let ast::Expr::Attribute(attribute) = call.func.as_ref() + && is_dunder_all(&attribute.value) + && let [argument] = &*call.arguments.args + { + match attribute.attr.as_str() { + "extend" => self.extend_from(argument), + "append" => { + if let ast::Expr::StringLiteral(literal) = argument { + self.names.push(Name::new(literal.value.to_str())); + } + } + _ => {} + } + } + } + // a nested scope's `__all__` is not the module's + ast::Stmt::FunctionDef(_) | ast::Stmt::ClassDef(_) => {} + _ => walk_stmt(self, stmt), + } + } +} diff --git a/crates/ty_test/src/db.rs b/crates/ty_test/src/db.rs index 93326a54ae..57ed39baf0 100644 --- a/crates/ty_test/src/db.rs +++ b/crates/ty_test/src/db.rs @@ -536,6 +536,11 @@ fn mdtest_rule_selection( // `.by` test to write `let` or `var` on assignments whose subject is // something else entirely. "implicit-declaration", + // `unused-import` too: a `.by` test imports what its subject needs, and often + // more than one block reads, so reporting every unused import would bury the + // tests that are about something else. `basedpython_unused_import.md` turns it + // back on for the tests that are about it. + "unused-import", ]; let registry = default_lint_registry(); From 358e3cb60ec630ce2ad2a4153ca840d1b820877e Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:18:07 +1000 Subject: [PATCH 21/47] move ruff's `delete_stmt` fix helper into `ruff_python_edits` so ty can delete a statement the same way --- Cargo.lock | 4 + crates/basedpython/Cargo.lock | 3 + crates/ruff_linter/src/fix/edits.rs | 221 +----------------------- crates/ruff_python_edits/Cargo.toml | 4 + crates/ruff_python_edits/src/delete.rs | 225 +++++++++++++++++++++++++ crates/ruff_python_edits/src/lib.rs | 2 + 6 files changed, 245 insertions(+), 214 deletions(-) create mode 100644 crates/ruff_python_edits/src/delete.rs diff --git a/Cargo.lock b/Cargo.lock index ad177ae667..eed67ea3e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3710,8 +3710,12 @@ dependencies = [ name = "ruff_python_edits" version = "0.0.12" dependencies = [ + "anyhow", + "ruff_diagnostics", "ruff_python_ast", + "ruff_python_index", "ruff_python_parser", + "ruff_python_trivia", "ruff_source_file", "ruff_text_size", ] diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index f492e74c20..f2c0951a2f 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -2728,7 +2728,10 @@ dependencies = [ name = "ruff_python_edits" version = "0.0.12" dependencies = [ + "ruff_diagnostics", "ruff_python_ast", + "ruff_python_index", + "ruff_python_trivia", "ruff_source_file", "ruff_text_size", ] diff --git a/crates/ruff_linter/src/fix/edits.rs b/crates/ruff_linter/src/fix/edits.rs index d8ee983fce..90ac7c5a9c 100644 --- a/crates/ruff_linter/src/fix/edits.rs +++ b/crates/ruff_linter/src/fix/edits.rs @@ -5,15 +5,13 @@ use anyhow::{Context, Result}; use ruff_python_ast::AnyNodeRef; use ruff_python_ast::name::Name; use ruff_python_ast::token::{self, Tokens, parenthesized_range}; -use ruff_python_ast::{self as ast, Arguments, ExceptHandler, Expr, ExprList, Parameters, Stmt}; +use ruff_python_ast::{self as ast, Arguments, Expr, ExprList, Parameters, Stmt}; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; use ruff_python_semantic::SemanticModel; use ruff_python_trivia::textwrap::dedent_to; -use ruff_python_trivia::{ - PythonWhitespace, SimpleTokenKind, SimpleTokenizer, has_leading_content, is_python_whitespace, -}; -use ruff_source_file::{LineRanges, NewlineWithTrailingNewline, UniversalNewlines}; +use ruff_python_trivia::{PythonWhitespace, SimpleTokenKind, SimpleTokenizer}; +use ruff_source_file::{LineRanges, UniversalNewlines}; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use crate::Edit; @@ -23,44 +21,14 @@ use crate::fix::codemods; use crate::fix::codemods::CodegenStylist; use crate::line_width::{IndentWidth, LineLength, LineWidthBuilder}; -/// Return the [`Edit`] to use when deleting a [`Stmt`]. -/// -/// In some cases, this is as simple as deleting the [`TextRange`] of the [`Stmt`] -/// itself. However, there are a few exceptions: -/// - If the [`Stmt`] is _not_ the terminal statement in a multi-statement line, -/// we need to delete up to the start of the next statement (and avoid -/// deleting any content that precedes the statement). -/// - If the [`Stmt`] is the terminal statement in a multi-statement line, we need -/// to avoid deleting any content that precedes the statement. -/// - If the [`Stmt`] has no trailing and leading content, then it's convenient to -/// remove the entire start and end lines. -/// - If the [`Stmt`] is the last statement in its parent body, replace it with a -/// `pass` instead. +/// Return the [`Edit`] to use when deleting a [`Stmt`]. See [`ruff_python_edits::delete_stmt`]. pub(crate) fn delete_stmt( stmt: &Stmt, parent: Option<&Stmt>, locator: &Locator, indexer: &Indexer, ) -> Edit { - if parent.is_some_and(|parent| is_lone_child(stmt, parent)) { - // If removing this node would lead to an invalid syntax tree, replace - // it with a `pass`. - Edit::range_replacement("pass".to_string(), stmt.range()) - } else { - if let Some(semicolon) = trailing_semicolon(stmt.end(), locator) { - let next = next_stmt_break(semicolon, locator); - Edit::deletion(stmt.start(), next) - } else if has_leading_content(stmt.start(), locator.contents()) { - Edit::range_deletion(stmt.range()) - } else if let Some(start) = - indexer.preceded_by_continuations(stmt.start(), locator.contents()) - { - Edit::deletion(start, stmt.end()) - } else { - let range = locator.full_lines_range(stmt.range()); - Edit::range_deletion(range) - } - } + ruff_python_edits::delete_stmt(stmt, parent, locator.contents(), indexer) } /// Generate a [`Edit`] to delete a comment (for example: a `noqa` directive). @@ -475,113 +443,6 @@ pub(crate) fn adjust_indentation( Ok(module_text) } -/// Determine if a vector contains only one, specific element. -fn is_only(vec: &[T], value: &T) -> bool { - vec.len() == 1 && vec[0] == *value -} - -/// Determine if a child is the only statement in its body. -fn is_lone_child(child: &Stmt, parent: &Stmt) -> bool { - match parent { - Stmt::FunctionDef(ast::StmtFunctionDef { body, .. }) - | Stmt::ClassDef(ast::StmtClassDef { body, .. }) - | Stmt::With(ast::StmtWith { body, .. }) - if is_only(body, child) => - { - return true; - } - Stmt::For(ast::StmtFor { body, orelse, .. }) - | Stmt::While(ast::StmtWhile { body, orelse, .. }) - if (is_only(body, child) || is_only(orelse, child)) => - { - return true; - } - Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) if (is_only(body, child) - || elif_else_clauses - .iter() - .any(|ast::ElifElseClause { body, .. }| is_only(body, child))) => - { - return true; - } - Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) if (is_only(body, child) - || is_only(orelse, child) - || is_only(finalbody, child) - || handlers.iter().any(|handler| match handler { - ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) => { - is_only(body, child) - } - })) => - { - return true; - } - Stmt::Match(ast::StmtMatch { cases, .. }) - if cases.iter().any(|case| is_only(&case.body, child)) => - { - return true; - } - _ => {} - } - false -} - -/// Return the location of a trailing semicolon following a `Stmt`, if it's part -/// of a multi-statement line. -fn trailing_semicolon(offset: TextSize, locator: &Locator) -> Option { - let contents = locator.after(offset); - - for line in NewlineWithTrailingNewline::from(contents) { - let trimmed = line.trim_whitespace_start(); - - if trimmed.starts_with(';') { - let colon_offset = line.text_len() - trimmed.text_len(); - return Some(offset + line.start() + colon_offset); - } - - if !trimmed.starts_with('\\') { - break; - } - } - None -} - -/// Find the next valid break for a `Stmt` after a semicolon. -fn next_stmt_break(semicolon: TextSize, locator: &Locator) -> TextSize { - let start_location = semicolon + TextSize::from(1); - - for line in - NewlineWithTrailingNewline::with_offset(locator.after(start_location), start_location) - { - let trimmed = line.trim_whitespace(); - // Skip past any continuations. - if trimmed.starts_with('\\') { - continue; - } - - return if trimmed.is_empty() { - // If the line is empty, then despite the previous statement ending in a - // semicolon, we know that it's not a multi-statement line. - line.start() - } else { - // Otherwise, find the start of the next statement. (Or, anything that isn't - // whitespace.) - let relative_offset = line.find(|c: char| !is_python_whitespace(c)).unwrap(); - line.start() + TextSize::try_from(relative_offset).unwrap() - }; - } - - locator.line_end(start_location) -} - /// Add leading whitespace to a snippet, if it's immediately preceded an identifier or keyword. pub(crate) fn pad_start(mut content: String, start: TextSize, locator: &Locator) -> String { // Ex) When converting `except(ValueError,)` from a tuple to a single argument, we need to @@ -675,12 +536,10 @@ mod tests { use ruff_python_ast::Stmt; use ruff_python_codegen::Stylist; use ruff_python_parser::{parse_expression, parse_module}; - use ruff_text_size::{Ranged, TextRange, TextSize}; + use ruff_text_size::{TextRange, TextSize}; use crate::fix::apply_fixes; - use crate::fix::edits::{ - add_to_dunder_all, make_redundant_alias, next_stmt_break, trailing_semicolon, - }; + use crate::fix::edits::{add_to_dunder_all, make_redundant_alias}; use crate::{Edit, Fix, Locator, Violation}; /// Parse the given source using [`Mode::Module`] and return the first statement. @@ -689,72 +548,6 @@ mod tests { Ok(suite.into_iter().next().unwrap()) } - #[test] - fn find_semicolon() -> Result<()> { - let contents = "x = 1"; - let stmt = parse_first_stmt(contents)?; - let locator = Locator::new(contents); - assert_eq!(trailing_semicolon(stmt.end(), &locator), None); - - let contents = "x = 1; y = 1"; - let stmt = parse_first_stmt(contents)?; - let locator = Locator::new(contents); - assert_eq!( - trailing_semicolon(stmt.end(), &locator), - Some(TextSize::from(5)) - ); - - let contents = "x = 1 ; y = 1"; - let stmt = parse_first_stmt(contents)?; - let locator = Locator::new(contents); - assert_eq!( - trailing_semicolon(stmt.end(), &locator), - Some(TextSize::from(6)) - ); - - let contents = r" -x = 1 \ - ; y = 1 -" - .trim(); - let stmt = parse_first_stmt(contents)?; - let locator = Locator::new(contents); - assert_eq!( - trailing_semicolon(stmt.end(), &locator), - Some(TextSize::from(10)) - ); - - Ok(()) - } - - #[test] - fn find_next_stmt_break() { - let contents = "x = 1; y = 1"; - let locator = Locator::new(contents); - assert_eq!( - next_stmt_break(TextSize::from(4), &locator), - TextSize::from(5) - ); - - let contents = "x = 1 ; y = 1"; - let locator = Locator::new(contents); - assert_eq!( - next_stmt_break(TextSize::from(5), &locator), - TextSize::from(6) - ); - - let contents = r" -x = 1 \ - ; y = 1 -" - .trim(); - let locator = Locator::new(contents); - assert_eq!( - next_stmt_break(TextSize::from(10), &locator), - TextSize::from(12) - ); - } - #[test] fn redundant_alias() -> Result<()> { let contents = "import x, y as y, z as bees"; diff --git a/crates/ruff_python_edits/Cargo.toml b/crates/ruff_python_edits/Cargo.toml index 046045452f..da700dc768 100644 --- a/crates/ruff_python_edits/Cargo.toml +++ b/crates/ruff_python_edits/Cargo.toml @@ -11,11 +11,15 @@ repository = { workspace = true } license = { workspace = true } [dependencies] +ruff_diagnostics = { workspace = true } ruff_python_ast = { workspace = true } +ruff_python_index = { workspace = true } +ruff_python_trivia = { workspace = true } ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } ruff_python_parser = { workspace = true } [lints] diff --git a/crates/ruff_python_edits/src/delete.rs b/crates/ruff_python_edits/src/delete.rs new file mode 100644 index 0000000000..038bad57e6 --- /dev/null +++ b/crates/ruff_python_edits/src/delete.rs @@ -0,0 +1,225 @@ +//! deleting a statement without leaving the source around it invalid + +use ruff_diagnostics::Edit; +use ruff_python_ast::{self as ast, ExceptHandler, Stmt}; +use ruff_python_index::Indexer; +use ruff_python_trivia::{PythonWhitespace, has_leading_content, is_python_whitespace}; +use ruff_source_file::{LineRanges, NewlineWithTrailingNewline}; +use ruff_text_size::{Ranged, TextLen, TextSize}; + +/// Return the [`Edit`] to use when deleting a [`Stmt`]. +/// +/// In some cases, this is as simple as deleting the +/// [`TextRange`](ruff_text_size::TextRange) of the [`Stmt`] itself. However, there are a +/// few exceptions: +/// - If the [`Stmt`] is _not_ the terminal statement in a multi-statement line, +/// we need to delete up to the start of the next statement (and avoid +/// deleting any content that precedes the statement). +/// - If the [`Stmt`] is the terminal statement in a multi-statement line, we need +/// to avoid deleting any content that precedes the statement. +/// - If the [`Stmt`] has no trailing and leading content, then it's convenient to +/// remove the entire start and end lines. +/// - If the [`Stmt`] is the last statement in its parent body, replace it with a +/// `pass` instead. +pub fn delete_stmt(stmt: &Stmt, parent: Option<&Stmt>, source: &str, indexer: &Indexer) -> Edit { + if parent.is_some_and(|parent| is_lone_child(stmt, parent)) { + // If removing this node would lead to an invalid syntax tree, replace + // it with a `pass`. + Edit::range_replacement("pass".to_string(), stmt.range()) + } else { + if let Some(semicolon) = trailing_semicolon(stmt.end(), source) { + let next = next_stmt_break(semicolon, source); + Edit::deletion(stmt.start(), next) + } else if has_leading_content(stmt.start(), source) { + Edit::range_deletion(stmt.range()) + } else if let Some(start) = indexer.preceded_by_continuations(stmt.start(), source) { + Edit::deletion(start, stmt.end()) + } else { + let range = source.full_lines_range(stmt.range()); + Edit::range_deletion(range) + } + } +} + +/// Determine if a vector contains only one, specific element. +fn is_only(vec: &[T], value: &T) -> bool { + vec.len() == 1 && vec[0] == *value +} + +/// Determine if a child is the only statement in its body. +fn is_lone_child(child: &Stmt, parent: &Stmt) -> bool { + match parent { + Stmt::FunctionDef(ast::StmtFunctionDef { body, .. }) + | Stmt::ClassDef(ast::StmtClassDef { body, .. }) + | Stmt::With(ast::StmtWith { body, .. }) + if is_only(body, child) => + { + return true; + } + Stmt::For(ast::StmtFor { body, orelse, .. }) + | Stmt::While(ast::StmtWhile { body, orelse, .. }) + if (is_only(body, child) || is_only(orelse, child)) => + { + return true; + } + Stmt::If(ast::StmtIf { + body, + elif_else_clauses, + .. + }) if (is_only(body, child) + || elif_else_clauses + .iter() + .any(|ast::ElifElseClause { body, .. }| is_only(body, child))) => + { + return true; + } + Stmt::Try(ast::StmtTry { + body, + handlers, + orelse, + finalbody, + .. + }) if (is_only(body, child) + || is_only(orelse, child) + || is_only(finalbody, child) + || handlers.iter().any(|handler| match handler { + ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) => { + is_only(body, child) + } + })) => + { + return true; + } + Stmt::Match(ast::StmtMatch { cases, .. }) + if cases.iter().any(|case| is_only(&case.body, child)) => + { + return true; + } + _ => {} + } + false +} + +/// Return the location of a trailing semicolon following a `Stmt`, if it's part +/// of a multi-statement line. +fn trailing_semicolon(offset: TextSize, source: &str) -> Option { + let contents = &source[offset.to_usize()..]; + + for line in NewlineWithTrailingNewline::from(contents) { + let trimmed = line.trim_whitespace_start(); + + if trimmed.starts_with(';') { + let colon_offset = line.text_len() - trimmed.text_len(); + return Some(offset + line.start() + colon_offset); + } + + if !trimmed.starts_with('\\') { + break; + } + } + None +} + +/// Find the next valid break for a `Stmt` after a semicolon. +fn next_stmt_break(semicolon: TextSize, source: &str) -> TextSize { + let start_location = semicolon + TextSize::from(1); + + for line in NewlineWithTrailingNewline::with_offset( + &source[start_location.to_usize()..], + start_location, + ) { + let trimmed = line.trim_whitespace(); + // Skip past any continuations. + if trimmed.starts_with('\\') { + continue; + } + + return if trimmed.is_empty() { + // If the line is empty, then despite the previous statement ending in a + // semicolon, we know that it's not a multi-statement line. + line.start() + } else { + // Otherwise, find the start of the next statement. (Or, anything that isn't + // whitespace.) + let relative_offset = line.find(|c: char| !is_python_whitespace(c)).unwrap(); + line.start() + TextSize::try_from(relative_offset).unwrap() + }; + } + + source.line_end(start_location) +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use ruff_python_ast::Stmt; + use ruff_python_parser::parse_module; + use ruff_text_size::{Ranged, TextSize}; + + use super::{next_stmt_break, trailing_semicolon}; + + /// Parse the given source using [`Mode::Module`] and return the first statement. + fn parse_first_stmt(source: &str) -> Result { + let suite = parse_module(source)?.into_suite(); + Ok(suite.into_iter().next().unwrap()) + } + + #[test] + fn find_semicolon() -> Result<()> { + let contents = "x = 1"; + let stmt = parse_first_stmt(contents)?; + assert_eq!(trailing_semicolon(stmt.end(), contents), None); + + let contents = "x = 1; y = 1"; + let stmt = parse_first_stmt(contents)?; + assert_eq!( + trailing_semicolon(stmt.end(), contents), + Some(TextSize::from(5)) + ); + + let contents = "x = 1 ; y = 1"; + let stmt = parse_first_stmt(contents)?; + assert_eq!( + trailing_semicolon(stmt.end(), contents), + Some(TextSize::from(6)) + ); + + let contents = r" +x = 1 \ + ; y = 1 +" + .trim(); + let stmt = parse_first_stmt(contents)?; + assert_eq!( + trailing_semicolon(stmt.end(), contents), + Some(TextSize::from(10)) + ); + + Ok(()) + } + + #[test] + fn find_next_stmt_break() { + let contents = "x = 1; y = 1"; + assert_eq!( + next_stmt_break(TextSize::from(4), contents), + TextSize::from(5) + ); + + let contents = "x = 1 ; y = 1"; + assert_eq!( + next_stmt_break(TextSize::from(5), contents), + TextSize::from(6) + ); + + let contents = r" +x = 1 \ + ; y = 1 +" + .trim(); + assert_eq!( + next_stmt_break(TextSize::from(10), contents), + TextSize::from(12) + ); + } +} diff --git a/crates/ruff_python_edits/src/lib.rs b/crates/ruff_python_edits/src/lib.rs index 26b50a9e5a..19fc06358e 100644 --- a/crates/ruff_python_edits/src/lib.rs +++ b/crates/ruff_python_edits/src/lib.rs @@ -3,5 +3,7 @@ Utilities for constructing edits to Python source code. */ pub use self::call::unwrapped_call_argument; +pub use self::delete::delete_stmt; mod call; +mod delete; From d2b522120f9a13bd60b37e6cc6b35131085331db Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:22:22 +1000 Subject: [PATCH 22/47] fix an unused import with `by check --fix`: remove the statement, or only its unused aliases, writing `pass` where a block would be left empty; an import in a package's `__init__.by` or one testing for a module's availability gets no fix --- Cargo.lock | 1 + crates/basedpython/Cargo.lock | 1 + crates/ty/tests/cli/fixes.rs | 94 ++++ crates/ty_python_core/src/definition.rs | 10 + crates/ty_python_semantic/Cargo.toml | 1 + .../mdtest/basedpython_unused_import.md | 376 ++++++++++++++- .../src/types/diagnostic.rs | 6 + .../src/types/unused_imports.rs | 434 ++++++++++++++++-- 8 files changed, 880 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eed67ea3e9..289a8d0292 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5119,6 +5119,7 @@ dependencies = [ "ruff_memory_usage", "ruff_python_ast", "ruff_python_edits", + "ruff_python_index", "ruff_python_literal", "ruff_python_parser", "ruff_python_stdlib", diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index f2c0951a2f..fbbbff736d 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -3865,6 +3865,7 @@ dependencies = [ "ruff_memory_usage", "ruff_python_ast", "ruff_python_edits", + "ruff_python_index", "ruff_python_literal", "ruff_python_parser", "ruff_python_stdlib", diff --git a/crates/ty/tests/cli/fixes.rs b/crates/ty/tests/cli/fixes.rs index 21424a58e0..5b50d94dfd 100644 --- a/crates/ty/tests/cli/fixes.rs +++ b/crates/ty/tests/cli/fixes.rs @@ -202,6 +202,100 @@ fn fix() -> anyhow::Result<()> { Ok(()) } +/// `--fix` removes the imports nothing in a `.by` file uses, and keeps the one used only +/// through the extension its module declares +#[test] +fn fix_unused_imports() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ( + "colours.by", + r#" + from enum import Enum + + extension Enum: + let lower_name: str + get() = self.name.lower() + "#, + ), + ( + "main.by", + r#" + import json + from enum import Enum + import colours + from collections import OrderedDict, deque + + class Colour(Enum): + RED = 1 + + def describe(colour: Colour, seen: OrderedDict[str, int]) -> str: + return colour.lower_name + "#, + ), + ])?; + + assert_cmd_snapshot!(case.command().arg("--fix"), @" + success: true + exit_code: 0 + ----- stdout ----- + Found 2 diagnostics (2 fixed, 0 remaining). + + ----- stderr ----- + "); + + assert_snapshot!(fs::read_to_string(case.root().join("main.by"))?, @" + + from enum import Enum + import colours + from collections import OrderedDict + + class Colour(Enum): + RED = 1 + + def describe(colour: Colour, seen: OrderedDict[str, int]) -> str: + return colour.lower_name + "); + + // the fixed file checks clean, the extension still resolving + assert_cmd_snapshot!(case.command(), @" + success: true + exit_code: 0 + ----- stdout ----- + All checks passed! + + ----- stderr ----- + "); + + Ok(()) +} + +/// a name imported many times over is fixed in one round, however many times: every unread +/// import of it is reported at once, so the fixes do not converge one import per round. a +/// block left with nothing in it is left with `pass` and nothing else +#[test] +fn fix_unused_imports_in_one_round() -> anyhow::Result<()> { + let repeated = "import os\n".repeat(12); + let source = format!("{repeated}\nif True:\n import json; import sys\n"); + let case = CliTest::with_file("main.by", &source)?; + + assert_cmd_snapshot!(case.command().arg("--fix"), @" + success: true + exit_code: 0 + ----- stdout ----- + Found 14 diagnostics (14 fixed, 0 remaining). + + ----- stderr ----- + "); + + assert_snapshot!(fs::read_to_string(case.root().join("main.by"))?, @" + + if True: + pass + "); + + Ok(()) +} + #[test] fn fix_unfixable() -> anyhow::Result<()> { let case = CliTest::with_files([ diff --git a/crates/ty_python_core/src/definition.rs b/crates/ty_python_core/src/definition.rs index 60904b433e..3a8776141c 100644 --- a/crates/ty_python_core/src/definition.rs +++ b/crates/ty_python_core/src/definition.rs @@ -1463,6 +1463,11 @@ pub struct ImportDefinitionKind { } impl ImportDefinitionKind { + /// which of the statement's aliases this definition is the binding of + pub fn alias_index(&self) -> usize { + self.alias_index as usize + } + pub fn import<'ast>(&self, module: &'ast ParsedModuleRef) -> &'ast ast::StmtImport { self.node.node(module) } @@ -1484,6 +1489,11 @@ pub struct ImportFromDefinitionKind { } impl ImportFromDefinitionKind { + /// which of the statement's aliases this definition is the binding of + pub fn alias_index(&self) -> usize { + self.alias_index as usize + } + pub fn import<'ast>(&self, module: &'ast ParsedModuleRef) -> &'ast ast::StmtImportFrom { self.node.node(module) } diff --git a/crates/ty_python_semantic/Cargo.toml b/crates/ty_python_semantic/Cargo.toml index 756b33fea8..fc0504d550 100644 --- a/crates/ty_python_semantic/Cargo.toml +++ b/crates/ty_python_semantic/Cargo.toml @@ -19,6 +19,7 @@ ruff_macros = { workspace = true } ruff_memory_usage = { workspace = true } ruff_python_ast = { workspace = true, features = ["salsa"] } ruff_python_edits = { workspace = true } +ruff_python_index = { workspace = true } ruff_python_literal = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md index 218253c073..935c2ceb83 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md @@ -365,6 +365,98 @@ import parsing register(3) ``` +## every element of a collection literal that converts uses its extension + +each element of the list converts through a different module's extension, so both imports are used — +the second element's as much as the first's + +`ids.by`: + +```by +class UserId: + pass + +class Tag: + pass +``` + +`numbers.by`: + +```by +from ids import UserId + +extension UserId: + class def __from__(cls, value: int) -> UserId: + return UserId() +``` + +`labels.by`: + +```by +from ids import Tag + +extension Tag: + class def __from__(cls, value: str) -> Tag: + return Tag() +``` + +`main.by`: + +```by +from ids import UserId, Tag +import numbers +import labels + +xs: list[UserId | Tag] = [1, "s"] +``` + +## every element of a collection literal assigned to an attribute uses its extension + +`ids.by`: + +```by +class UserId: + pass + +class Tag: + pass +``` + +`numbers.by`: + +```by +from ids import UserId + +extension UserId: + class def __from__(cls, value: int) -> UserId: + return UserId() +``` + +`labels.by`: + +```by +from ids import Tag + +extension Tag: + class def __from__(cls, value: str) -> Tag: + return Tag() +``` + +`main.by`: + +```by +from ids import UserId, Tag +import numbers +import labels + +class Holder: + def __init__(self) -> None: + self.xs: list[UserId | Tag] = [] + +holder = Holder() +holder.xs = [1, "s"] +``` + ## any import of the module grants its extensions a `from` import of a name the file never reads still makes the module's extensions applicable, so it @@ -485,6 +577,168 @@ def show(x: int) -> str: return x.renderer() ``` +## an import read only as a context-sensitive name's qualifier + +`Red` where a `Color` is expected is lowered to `Color.Red`, so the import of `Color` is read +although the source never spells the name + +`palette.by`: + +```by +enum class Color: + case Red, Green + +def paint(c: Color) -> str: + return str(c) +``` + +`main.by`: + +```by +from palette import Color, paint + +paint(Red) +``` + +## an import read only as a case name's qualifier + +`case Red:` is lowered to `case Color.Red:`, and `case Circle(r):` to `case Shape.Circle(r):` + +`palette.by`: + +```by +enum class Color: + case Red, Green + +enum class Shape: + case Circle(radius: float) + case Dot + +def current() -> Color: + return Color.Green + +def outline() -> Shape: + return Shape.Dot +``` + +`main.by`: + +```by +from palette import Color, Shape, current, outline + +def describe() -> str: + match current(): + case Red: + return "red" + case _: + return "other" + +def radius() -> float: + match outline(): + case Circle(r): + return r + case _: + return 0.0 +``` + +## an import read only by a type test's runtime check + +`x is AB` is lowered to a test of each arm of the alias, `isinstance(x, A) or isinstance(x, B)`, so +it reads `A` and `B` although the source spells neither + +`kinds.by`: + +```by +class A: + pass + +class B: + pass + +type AB = A | B +``` + +`main.by`: + +```by +from kinds import AB, A, B + +def check(x: object) -> bool: + return x is AB +``` + +## an import read only by a type test against a module attribute + +`kinds.by`: + +```by +class A: + pass + +class B: + pass + +type AB = A | B +``` + +`main.by`: + +```by +import kinds +from kinds import A, B + +def check(x: object) -> bool: + return x is kinds.AB +``` + +## an import read only by a checked cast's runtime check + +`kinds.by`: + +```by +class A: + pass + +class B: + pass + +type AB = A | B +``` + +`main.by`: + +```by +from kinds import AB, A, B + +def narrow(x: object) -> AB: + return x cast! AB +``` + +## an import read only by a `raises` guard + +the guard a `raises` function is lowered behind names the exceptions it may raise + +`errors.by`: + +```by +class Oops(Exception): + pass + +class Bad(Exception): + pass + +type Errs = Oops | Bad +``` + +`main.by`: + +```by +from errors import Errs, Oops, Bad + +def run() -> None raises Errs: + raise Oops() +``` + ## explicit re-exports are not reported `import a as a`, `from a import b as b` and basedpython's `from a export b` say the name is part of @@ -557,10 +811,10 @@ __all__ += helpers.__all__ from __future__ import annotations ``` -## a rebound name is reported only at its last binding +## an import rebound by something other than an import is not reported -only the binding a name is left holding at the end of its scope is reported. an import the name is -rebound after is `F811`'s concern +an assignment that rebinds the name makes the import a redefinition rather than an unused import, +which is `F811`'s concern ```by import json @@ -579,10 +833,13 @@ os.getcwd() import os ``` -## the last of several unread bindings is reported +## every one of several unread imports of a name is reported + +an import rebound by another import before anything reads it is reported too, so one round of fixes +removes them all ```by -import json +import json # error: [unused-import] import json # error: [unused-import] ``` @@ -669,6 +926,18 @@ def parser() -> object: return email.parser ``` +## an import of the package does not redefine a submodule import + +`import os` after `import os.path` binds `os` again, but `os.path` stays reachable through it only +because of the first import, so the first is not an unused redefinition + +```by +import os.path +import os + +os.getcwd() +``` + ## a read earlier in a loop body uses an import later in it the second time round the loop, the read sees the import the first pass made. the checker cannot @@ -756,6 +1025,35 @@ warning[unused-import]: `tomllib` imported but unused help: To test whether a module is available, use `importlib.util.find_spec` instead of importing it ``` +## an import in a function defined inside such a `try` is not a test + +the function's body runs when it is called, outside the `try`, so its import gets the ordinary fix + +```by +try: + import tomllib # error: [unused-import] + + def load() -> None: + import json # snapshot +except ImportError: + pass +``` + +```snapshot +warning[unused-import]: `json` imported but unused + --> src/mdtest_snippet.by:5:16 + | +5 | import json # snapshot + | ^^^^ +help: Remove the unused import `json` + | +4 | def load() -> None: + - import json # snapshot +5 + pass # snapshot +6 | except ImportError: + | +``` + ## an unused import in a package's `__init__` may be a re-export a package's `__init__.by` is what importing the package runs, so an import there may be how the @@ -791,3 +1089,71 @@ help: If this is a re-export, write it as `from … export …`, as a redundant def load() -> None: import json # error: [unused-import] ``` + +## the fix removes an unused alias and keeps the rest of the statement + +```by +from collections import OrderedDict, deque # snapshot + +OrderedDict() +``` + +```snapshot +warning[unused-import]: `collections.deque` imported but unused + --> src/mdtest_snippet.by:1:38 + | +1 | from collections import OrderedDict, deque # snapshot + | ^^^^^ +help: Remove the unused import `collections.deque` + | + - from collections import OrderedDict, deque # snapshot +1 + from collections import OrderedDict # snapshot +2 | + | +``` + +## the fix removes an alias with its own line inside parentheses + +```by +from collections import ( + OrderedDict, # read below + deque, # snapshot +) + +OrderedDict() +``` + +```snapshot +warning[unused-import]: `collections.deque` imported but unused + --> src/mdtest_snippet.by:3:5 + | +3 | deque, # snapshot + | ^^^^^ +help: Remove the unused import `collections.deque` + | +2 | OrderedDict, # read below + - deque, # snapshot +3 | ) + | +``` + +## the fix writes `pass` where removing the import would leave a block empty + +```by +def load() -> None: + import json # snapshot +``` + +```snapshot +warning[unused-import]: `json` imported but unused + --> src/mdtest_snippet.by:2:12 + | +2 | import json # snapshot + | ^^^^ +help: Remove the unused import `json` + | +1 | def load() -> None: + - import json # snapshot +2 + pass # snapshot + | +``` diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index edb3e5282b..2c562be014 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -2923,6 +2923,12 @@ declare_lint! { /// import colours /// from enum import Enum /// ``` + /// + /// ## Fix safety + /// The fix removes the import, and is safe: nothing in the file reads it. + /// Removing an import also stops the imported module from being run, which + /// matters only if the module's import has a side effect the program relies + /// on. pub(crate) static UNUSED_IMPORT = { summary: "detects an import that nothing uses", status: LintStatus::stable("0.0.1-beta.2"), diff --git a/crates/ty_python_semantic/src/types/unused_imports.rs b/crates/ty_python_semantic/src/types/unused_imports.rs index e929c26720..10a5c0bb16 100644 --- a/crates/ty_python_semantic/src/types/unused_imports.rs +++ b/crates/ty_python_semantic/src/types/unused_imports.rs @@ -24,12 +24,17 @@ use ruff_db::files::File; use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::identifier::Identifier; use ruff_python_ast::name::Name; use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; use ruff_python_ast::{self as ast}; +use ruff_python_index::Indexer; +use ruff_python_trivia::{PythonWhitespace, is_python_whitespace}; +use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; -use rustc_hash::FxHashSet; +use rustc_hash::{FxHashMap, FxHashSet}; use ty_module_resolver::{ImportingFile, ModuleName, resolve_module}; use ty_python_core::definition::{Definition, DefinitionCategory, DefinitionKind}; use ty_python_core::place::ScopedPlaceId; @@ -77,22 +82,64 @@ pub(crate) fn report_unused_imports( } let index = semantic_index(db, file); + let checked_scopes = || { + index.scope_ids().filter(|scope| { + matches!( + index.scope(scope.file_scope_id(db)).kind(), + ScopeKind::Module | ScopeKind::Function + ) + }) + }; + // most files read every import they make, which the semantic index alone can tell: an + // import is only ever unused when the index records no read of it + let unread_import = checked_scopes().any(|scope| { + index + .use_def_map(scope.file_scope_id(db)) + .definitions_with_usage() + .any(|(_, definition, is_used)| !is_used && definition.kind(db).is_import()) + }); + if !unread_import { + return; + } + let uses = Uses::new(db, file, index, &parsed, inferred); + let unused_by_scope: Vec<_> = checked_scopes() + .filter_map(|scope| { + let file_scope = scope.file_scope_id(db); + let unused = unused_in_scope(db, index, file_scope, &parsed, &uses, source_file); + (!unused.is_empty()).then_some((scope, unused)) + }) + .collect(); + if unused_by_scope.is_empty() { + return; + } + let context = ImportContext::new(db, file, &env, &parsed); + let scopes: Vec<_> = unused_by_scope + .into_iter() + .map(|(scope, unused)| { + let at_module = scope.file_scope_id(db).is_global(); + let infer_context = InferContext::new(db, &env, scope, source_file, file, &parsed); + (infer_context, unused, at_module) + }) + .collect(); - for scope in index.scope_ids() { - let file_scope = scope.file_scope_id(db); - let kind = index.scope(file_scope).kind(); - if !matches!(kind, ScopeKind::Module | ScopeKind::Function) { - continue; - } - let unused = unused_in_scope(db, index, file_scope, &parsed, &uses, source_file); - if unused.is_empty() { - continue; - } - let infer_context = InferContext::new(db, &env, scope, source_file, file, &parsed); + // an import the lint is not reporting — a suppressed one, or one in unreachable code — is + // not removed, so the fix for its statement leaves it in place + let removed = scopes.iter().flat_map(|(infer_context, unused, _)| { + unused.iter().filter(|import| { + context.fixable(import) + && infer_context + .report_lint(&UNUSED_IMPORT, import.binding) + .is_some() + }) + }); + let source = source_text(db, source_file); + let removals = Removals::new(&parsed, source.as_str(), removed); + + for (infer_context, unused, at_module) in scopes { for import in &unused { - import.report(&infer_context, &context, kind == ScopeKind::Module); + import.report(&infer_context, &context, at_module, &removals); } diagnostics.extend(&infer_context.finish()); } @@ -193,6 +240,8 @@ impl<'db> Uses<'db> { struct UnusedImport { /// the statement the binding is one alias of statement: TextRange, + /// which of the statement's aliases makes the binding + alias_index: usize, /// the name the import makes available, as F401 spells it: `a.b`, `a.b.c` for /// `from a.b import c`, `.c` for `from . import c` qualified_name: String, @@ -256,39 +305,96 @@ fn unused_in_scope<'db>( if symbol.is_global() || symbol.is_nonlocal() { continue; } - events.sort_unstable_by_key(|(id, _)| *id); - // the bindings since the name was last unbound: the one it is left holding, and the - // ones it shadows. a read of any of them counts for the last, as it does for `F401` - let since_unbound = match events.iter().rposition(|(_, event)| event.is_none()) { - Some(deletion) => &events[deletion + 1..], - None => &events[..], - }; - // a name left unbound at the end of the scope holds no import - let Some(&(_, Some(live))) = since_unbound.last() else { - continue; - }; - let Some(import) = reportable_import(db, live, parsed) else { - continue; - }; - if since_unbound - .iter() - .filter_map(|(_, definition)| *definition) - .any(|definition| uses.reads(definition)) + // a name read as a whole reads whichever binding it is left holding, and every + // binding before it carries that read forward + if uses.scope_reads.contains(&(scope, symbol_id)) + || (scope.is_global() && uses.module_names.contains(symbol.name())) { continue; } - if uses.scope_reads.contains(&(scope, symbol_id)) { - continue; - } - if scope.is_global() && uses.module_names.contains(symbol.name()) { - continue; + events.sort_unstable_by_key(|(id, _)| *id); + // the bindings since the name was last unbound, in order. a read of one counts for + // every binding after it, as `F401` carries a shadowed binding's reads to the one that + // shadows it, so the bindings that count as unread are a prefix of these + let since_unbound: Vec> = + match events.iter().rposition(|(_, event)| event.is_none()) { + Some(deletion) => &events[deletion + 1..], + None => &events[..], + } + .iter() + .filter_map(|(_, definition)| *definition) + .collect(); + for (position, &definition) in since_unbound.iter().enumerate() { + if uses.reads(definition) { + break; + } + // a binding the name goes on to be rebound by is reported only when the import + // rebinding it redefines it — the unused import `F811` reports as redefined. + // reporting every one at once is what lets the fix remove them all in one round + if let Some(&next) = since_unbound.get(position + 1) + && !redefines(db, parsed, next, definition) + { + continue; + } + if let Some(import) = reportable_import(db, definition, parsed) { + unused.push(import); + } } - unused.push(import); } unused.sort_unstable_by_key(|import| import.binding.start()); unused } +/// what an import binds, for deciding whether a later import redefines it: the name it +/// imports, as `F401` spells it, and whether it is a submodule import (`import a.b`, which +/// binds `a` but makes `a.b` reachable through it) +fn imported_name<'db>( + db: &'db dyn Db, + definition: Definition<'db>, + parsed: &ParsedModuleRef, +) -> Option<(String, bool)> { + match definition.kind(db) { + DefinitionKind::Import(import) => { + let alias = import.alias(parsed); + let submodule = alias.asname.is_none() && alias.name.contains('.'); + Some((alias.name.to_string(), submodule)) + } + DefinitionKind::ImportFrom(import) => { + let statement = import.import(parsed); + let mut name = ".".repeat(statement.level as usize); + if let Some(module) = &statement.module { + name.push_str(module); + name.push('.'); + } + name.push_str(&import.alias(parsed).name); + Some((name, false)) + } + _ => None, + } +} + +/// whether the import `later` redefines the import `earlier`, by ruff's rule: a submodule +/// import redefines, and is redefined by, only an import of the same submodule — `import a.b` +/// then `import a.c` binds `a` twice, and the second leaves `a.b` reachable through it +fn redefines<'db>( + db: &'db dyn Db, + parsed: &ParsedModuleRef, + later: Definition<'db>, + earlier: Definition<'db>, +) -> bool { + let (Some((later_name, later_submodule)), Some((earlier_name, earlier_submodule))) = ( + imported_name(db, later, parsed), + imported_name(db, earlier, parsed), + ) else { + return false; + }; + if later_submodule || earlier_submodule { + later_name == earlier_name + } else { + true + } +} + /// `definition` as an import F401 reports when nothing reads it: not a `__future__` /// directive, and not an explicit re-export fn reportable_import<'db>( @@ -309,6 +415,7 @@ fn reportable_import<'db>( } Some(UnusedImport { statement: statement.range(), + alias_index: import.alias_index(), qualified_name: alias.name.to_string(), module: (!alias.is_resource) .then(|| ModuleName::new(&alias.name)) @@ -341,6 +448,7 @@ fn reportable_import<'db>( qualified_name.push_str(&alias.name); Some(UnusedImport { statement: statement.range(), + alias_index: import.alias_index(), qualified_name, module: statement .module @@ -405,6 +513,12 @@ impl<'db> ImportContext<'db> { } } + /// whether the import is removed by a fix: not one that may be a re-export from a + /// package's `__init__`, and not one that tests for its module's availability + fn fixable(&self, import: &UnusedImport) -> bool { + !self.in_init && !self.probing.contains(&import.statement.start()) + } + /// whether `module` is the project's own code fn is_first_party(&self, import: &UnusedImport) -> bool { let db = self.db; @@ -424,7 +538,13 @@ impl<'db> ImportContext<'db> { } impl UnusedImport { - fn report(&self, context: &InferContext<'_, '_>, imports: &ImportContext<'_>, at_module: bool) { + fn report( + &self, + context: &InferContext<'_, '_>, + imports: &ImportContext<'_>, + at_module: bool, + removals: &Removals<'_>, + ) { let Some(builder) = context.report_lint(&UNUSED_IMPORT, self.binding) else { return; }; @@ -446,9 +566,240 @@ impl UnusedImport { or list it in `__all__`", ); } + if imports.fixable(self) + && let Some(edits) = removals.edits(self.statement) + { + let mut edits = edits.into_iter(); + if let Some(first) = edits.next() { + diagnostic.set_fix(Fix::safe_edits(first, edits)); + diagnostic.help(format_args!( + "Remove the unused import `{}`", + self.qualified_name + )); + } + } + } +} + +/// the edits that remove the unused imports the lint reports. every diagnostic in a statement +/// carries the same fix, so applying any of them removes all of the statement's unused +/// aliases at once, as `F401`'s fix does. a statement that loses every alias is deleted, and +/// the statements deleted next to one another in a suite share one fix too: the deletion of +/// one ends where the next begins, and two fixes that touch are applied in separate rounds +struct Removals<'a> { + source: &'a str, + indexer: Indexer, + /// each statement with an alias to remove, by its start, with the aliases to remove + statements: FxHashMap)>, + /// each deleted statement, by its start, with the run of adjacent deleted statements in + /// its suite it belongs to + deleted: FxHashMap, + runs: Vec>, +} + +/// statements next to one another in a suite, all deleted +struct Run<'a> { + statements: Vec<&'a ast::Stmt>, + /// whether the run is the whole of a suite that may not be left empty, so its last + /// statement is written as `pass`. deleting each one before it takes the separator after + /// it too, so a suite written on one line (`import os; import sys`) is left as `pass` + leaves_pass: bool, +} + +impl<'a> Removals<'a> { + fn new<'i>( + parsed: &'a ParsedModuleRef, + source: &'a str, + removed: impl Iterator, + ) -> Self { + let mut wanted: FxHashMap> = FxHashMap::default(); + for import in removed { + wanted + .entry(import.statement.start()) + .or_default() + .push(import.alias_index); + } + + let mut suites = Suites { + wanted: &wanted, + statements: FxHashMap::default(), + deleted: FxHashMap::default(), + runs: Vec::new(), + in_module: true, + }; + suites.visit_body(parsed.suite()); + + Self { + source, + indexer: Indexer::from_tokens(parsed.tokens(), source), + statements: suites.statements, + deleted: suites.deleted, + runs: suites.runs, + } + } + + /// the edits removing the unused aliases of the statement starting at `statement` + fn edits(&self, statement: TextRange) -> Option> { + let start = statement.start(); + if let Some(&run) = self.deleted.get(&start) { + let run = &self.runs[run]; + let last = run.statements.len() - 1; + return Some( + run.statements + .iter() + .enumerate() + .map(|(position, stmt)| { + if run.leaves_pass && position == last { + Edit::range_replacement("pass".to_owned(), stmt.range()) + } else { + ruff_python_edits::delete_stmt(stmt, None, self.source, &self.indexer) + } + }) + .collect(), + ); + } + let (stmt, aliases) = self.statements.get(&start)?; + let names = import_aliases(stmt)?; + Some(remove_aliases(self.source, stmt, names, aliases)) + } +} + +/// the aliases of an import statement +fn import_aliases(stmt: &ast::Stmt) -> Option<&[ast::Alias]> { + match stmt { + ast::Stmt::Import(import) => Some(&import.names), + ast::Stmt::ImportFrom(import) => Some(&import.names), + _ => None, + } +} + +/// finds each import statement with an alias to remove, and groups the ones that lose every +/// alias, and with it the whole statement, into runs of neighbours +struct Suites<'a, 'w> { + wanted: &'w FxHashMap>, + statements: FxHashMap)>, + deleted: FxHashMap, + runs: Vec>, + /// whether the suite being visited is the module's own, which may be left empty + in_module: bool, +} + +impl<'a> StatementVisitor<'a> for Suites<'a, '_> { + fn visit_body(&mut self, body: &'a [ast::Stmt]) { + let in_module = std::mem::replace(&mut self.in_module, false); + let mut run: Vec<&'a ast::Stmt> = Vec::new(); + let mut runs: Vec> = Vec::new(); + for stmt in body { + let start = stmt.start(); + let mut deleted = false; + if let (Some(aliases), Some(names)) = (self.wanted.get(&start), import_aliases(stmt)) { + let mut aliases = aliases.clone(); + aliases.sort_unstable(); + aliases.dedup(); + deleted = aliases.len() == names.len(); + if !deleted { + self.statements.insert(start, (stmt, aliases)); + } + } + if deleted { + run.push(stmt); + } else if !run.is_empty() { + runs.push(std::mem::take(&mut run)); + } + self.visit_stmt(stmt); + } + if !run.is_empty() { + runs.push(run); + } + let whole_suite = !in_module && runs.len() == 1 && runs[0].len() == body.len(); + for statements in runs { + let id = self.runs.len(); + for stmt in &statements { + self.deleted.insert(stmt.start(), id); + } + self.runs.push(Run { + statements, + leaves_pass: whole_suite, + }); + } + self.in_module = in_module; } } +/// the edits removing the aliases at `remove` (sorted, and not every one) from an import +/// statement whose aliases are `aliases`. +/// +/// a run of aliases removed together goes with the separator after it, up to the next alias +/// kept, or — at the end of the statement — with the separator before it. a run that has +/// lines of its own inside parentheses goes with those lines, so the comments the formatter +/// leaves on an alias's line go with it and no other line changes +fn remove_aliases( + source: &str, + statement: &ast::Stmt, + aliases: &[ast::Alias], + remove: &[usize], +) -> Vec { + let mut edits = Vec::new(); + let mut runs: Vec<(usize, usize)> = Vec::new(); + for &index in remove { + match runs.last_mut() { + Some((_, last)) if *last + 1 == index => *last = index, + _ => runs.push((index, index)), + } + } + for (first, last) in runs { + let start = aliases[first].start(); + let end = aliases[last].end(); + if let Some(lines) = own_lines(source, statement, aliases, first, last) { + edits.push(Edit::range_deletion(lines)); + } else if let Some(next) = aliases.get(last + 1) { + edits.push(Edit::deletion(start, next.start())); + } else { + // the run ends the statement, and something before it is kept + edits.push(Edit::deletion(aliases[first - 1].end(), end)); + } + } + edits +} + +/// the whole lines the aliases `first..=last` occupy, when they occupy lines of their own +/// inside the statement's parentheses: nothing but indentation before the first, and nothing +/// but its comma and a comment after the last +fn own_lines( + source: &str, + statement: &ast::Stmt, + aliases: &[ast::Alias], + first: usize, + last: usize, +) -> Option { + // only a `from` import is parenthesized, and nothing before its first alias but the + // parenthesis can be one + let parenthesized = matches!(statement, ast::Stmt::ImportFrom(_)) + && source[TextRange::new(statement.start(), aliases.first()?.start())].contains('('); + if !parenthesized { + return None; + } + let start = aliases[first].start(); + let end = aliases[last].end(); + let line_start = source.line_start(start); + if !source[TextRange::new(line_start, start)] + .chars() + .all(is_python_whitespace) + { + return None; + } + let line_end = source.line_end(end); + let rest = source[TextRange::new(end, line_end)].trim_whitespace_start(); + let rest = rest + .strip_prefix(',') + .unwrap_or(rest) + .trim_whitespace_start(); + if !(rest.is_empty() || rest.starts_with('#')) { + return None; + } + Some(TextRange::new(line_start, source.full_line_end(end))) +} + /// finds the import statements written in the body of a `try` that handles `ImportError` or /// `ModuleNotFoundError` — an import made to find out whether the module is there struct ImportProbes<'a, 'db> { @@ -486,6 +837,13 @@ impl<'ast> StatementVisitor<'ast> for ImportProbes<'_, '_> { self.probing.insert(stmt.start()); } } + // a function's body runs when it is called, not inside the `try` it is defined + // in, and the `try` protects none of its imports — `F401` reads it the same way + ast::Stmt::FunctionDef(function) => { + let depth = std::mem::take(&mut self.depth); + self.visit_body(&function.body); + self.depth = depth; + } ast::Stmt::Try(try_stmt) => { let probes = self.handles_import_error(&try_stmt.handlers); self.depth += usize::from(probes); From a5e916d4294425d2c10d8e4fe05b599ac94fbcaa Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:25:49 +1000 Subject: [PATCH 23/47] stop buff's F401 checking `.by` and `.byi` files, whose unused imports `by check` reports as `unused-import` --- ...li__lint__output_format_preview_sarif.snap | 3 +- .../cli__lint__output_format_sarif.snap | 4 +-- .../integration_test__rule_f401.snap | 11 ++++++- ...tegration_test__rule_f401_output_json.snap | 7 ++-- ...tegration_test__rule_f401_output_text.snap | 11 ++++++- .../fixtures/pyflakes/F401_basedpython.by | 32 +++---------------- ...inter__message__sarif__tests__results.snap | 2 +- .../src/rules/pyflakes/rules/unused_import.rs | 15 +++++++++ ...ts__unused-import_F401_basedpython.by.snap | 16 +--------- 9 files changed, 50 insertions(+), 51 deletions(-) diff --git a/crates/ruff/tests/cli/snapshots/cli__lint__output_format_preview_sarif.snap b/crates/ruff/tests/cli/snapshots/cli__lint__output_format_preview_sarif.snap index f246ff25e2..dc14b90e0e 100644 --- a/crates/ruff/tests/cli/snapshots/cli__lint__output_format_preview_sarif.snap +++ b/crates/ruff/tests/cli/snapshots/cli__lint__output_format_preview_sarif.snap @@ -1,5 +1,6 @@ --- source: crates/ruff/tests/cli/lint.rs +assertion_line: 4424 info: program: buff args: @@ -139,7 +140,7 @@ exit_code: 1 }, { "fullDescription": { - "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" + "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## basedpython\na `.by` or `.byi` file is not checked. an import there can be used without its\nname ever being read: the `extension`s and conformances a module declares are\nonly applicable in the files that import it, and whether one of them supplies\nsomething a file relies on is a question about types, which a linter checking one\nfile at a time cannot answer. `by check` reports unused imports in `.by` files\ninstead, with this rule's semantics, as `unused-import`\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" }, "help": { "text": "`{name}` imported but unused; consider using `importlib.util.find_spec` to test for availability" diff --git a/crates/ruff/tests/cli/snapshots/cli__lint__output_format_sarif.snap b/crates/ruff/tests/cli/snapshots/cli__lint__output_format_sarif.snap index 2db1001143..3381523f0a 100644 --- a/crates/ruff/tests/cli/snapshots/cli__lint__output_format_sarif.snap +++ b/crates/ruff/tests/cli/snapshots/cli__lint__output_format_sarif.snap @@ -1,6 +1,6 @@ --- source: crates/ruff/tests/cli/lint.rs -assertion_line: 3663 +assertion_line: 4375 info: program: buff args: @@ -120,7 +120,7 @@ exit_code: 1 "rules": [ { "fullDescription": { - "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" + "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## basedpython\na `.by` or `.byi` file is not checked. an import there can be used without its\nname ever being read: the `extension`s and conformances a module declares are\nonly applicable in the files that import it, and whether one of them supplies\nsomething a file relies on is a question about types, which a linter checking one\nfile at a time cannot answer. `by check` reports unused imports in `.by` files\ninstead, with this rule's semantics, as `unused-import`\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" }, "help": { "text": "`{name}` imported but unused; consider using `importlib.util.find_spec` to test for availability" diff --git a/crates/ruff/tests/snapshots/integration_test__rule_f401.snap b/crates/ruff/tests/snapshots/integration_test__rule_f401.snap index 7eba551c4a..bdd939cb8c 100644 --- a/crates/ruff/tests/snapshots/integration_test__rule_f401.snap +++ b/crates/ruff/tests/snapshots/integration_test__rule_f401.snap @@ -1,7 +1,8 @@ --- source: crates/ruff/tests/integration_test.rs +assertion_line: 1009 info: - program: ruff + program: buff args: - rule - F401 @@ -123,6 +124,14 @@ else: print("numpy is not installed") ``` +## basedpython +a `.by` or `.byi` file is not checked. an import there can be used without its +name ever being read: the `extension`s and conformances a module declares are +only applicable in the files that import it, and whether one of them supplies +something a file relies on is a question about types, which a linter checking one +file at a time cannot answer. `by check` reports unused imports in `.by` files +instead, with this rule's semantics, as `unused-import` + ## Options - `lint.ignore-init-module-imports` - `lint.pyflakes.allowed-unused-imports` diff --git a/crates/ruff/tests/snapshots/integration_test__rule_f401_output_json.snap b/crates/ruff/tests/snapshots/integration_test__rule_f401_output_json.snap index 9795a7a608..5832be996d 100644 --- a/crates/ruff/tests/snapshots/integration_test__rule_f401_output_json.snap +++ b/crates/ruff/tests/snapshots/integration_test__rule_f401_output_json.snap @@ -1,7 +1,8 @@ --- source: crates/ruff/tests/integration_test.rs +assertion_line: 1040 info: - program: ruff + program: buff args: - rule - F401 @@ -23,7 +24,7 @@ exit_code: 0 ], "fix": "Fix is sometimes available.", "fix_availability": "Sometimes", - "explanation": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n", + "explanation": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## basedpython\na `.by` or `.byi` file is not checked. an import there can be used without its\nname ever being read: the `extension`s and conformances a module declares are\nonly applicable in the files that import it, and whether one of them supplies\nsomething a file relies on is a question about types, which a linter checking one\nfile at a time cannot answer. `by check` reports unused imports in `.by` files\ninstead, with this rule's semantics, as `unused-import`\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n", "preview": false, "status": { "Stable": { @@ -33,7 +34,7 @@ exit_code: 0 "category": "suspicious", "source_location": { "file": "", - "line": 145 + "line": 153 } } ----- stderr ----- diff --git a/crates/ruff/tests/snapshots/integration_test__rule_f401_output_text.snap b/crates/ruff/tests/snapshots/integration_test__rule_f401_output_text.snap index fa85f29d1e..e10cd64515 100644 --- a/crates/ruff/tests/snapshots/integration_test__rule_f401_output_text.snap +++ b/crates/ruff/tests/snapshots/integration_test__rule_f401_output_text.snap @@ -1,7 +1,8 @@ --- source: crates/ruff/tests/integration_test.rs +assertion_line: 1046 info: - program: ruff + program: buff args: - rule - F401 @@ -125,6 +126,14 @@ else: print("numpy is not installed") ``` +## basedpython +a `.by` or `.byi` file is not checked. an import there can be used without its +name ever being read: the `extension`s and conformances a module declares are +only applicable in the files that import it, and whether one of them supplies +something a file relies on is a question about types, which a linter checking one +file at a time cannot answer. `by check` reports unused imports in `.by` files +instead, with this rule's semantics, as `unused-import` + ## Options - `lint.ignore-init-module-imports` - `lint.pyflakes.allowed-unused-imports` diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F401_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F401_basedpython.by index 3ae6e5258e..3dc883cf95 100644 --- a/crates/ruff_linter/resources/test/fixtures/pyflakes/F401_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F401_basedpython.by @@ -1,32 +1,10 @@ -# `from x export y` is basedpython's explicit re-export spelling, so an -# exported name is deliberately public and never "imported but unused" — -# the same exemption `from x import y as y` gets -from a export exported_name -from b import y as y - -# a plain import of an unreferenced name is still reported +# F401 does not check a `.by` file. whether an import there is used can depend on the +# `extension`s and conformances its module declares, which only types tell, so +# `by check` reports an unused import instead, as `unused-import` +import json from c import unused_name - -# and an exported name may of course also be used locally -from d export used_name - -print(used_name) - -# an `extension` header references both the type it names and the interfaces it -# declares conformance to, so neither import is unused -from e import extended_type, implemented_type, interface_type +from e import extended_type extension extended_type: def extra(self) -> int: return 0 - -extension implemented_type(interface_type): - def m(self) -> int: - return 1 - -# a `raises` clause names exception types, so the import that supplies them is -# used — deleting it would leave a name the type checker cannot resolve -from f import RaisedError - -def may_fail() raises RaisedError: - raise RaisedError diff --git a/crates/ruff_linter/src/message/snapshots/ruff_linter__message__sarif__tests__results.snap b/crates/ruff_linter/src/message/snapshots/ruff_linter__message__sarif__tests__results.snap index c7365b7af2..34acd5e898 100644 --- a/crates/ruff_linter/src/message/snapshots/ruff_linter__message__sarif__tests__results.snap +++ b/crates/ruff_linter/src/message/snapshots/ruff_linter__message__sarif__tests__results.snap @@ -129,7 +129,7 @@ expression: value "rules": [ { "fullDescription": { - "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" + "text": "## What it does\nChecks for unused imports.\n\n## Why is this bad?\nUnused imports add a performance overhead at runtime, and risk creating\nimport cycles. They also increase the cognitive load of reading the code.\n\nIf an import statement is used to check for the availability or existence\nof a module, consider using `importlib.util.find_spec` instead.\n\nIf an import statement is used to re-export a symbol as part of a module's\npublic interface, consider using a \"redundant\" import alias, which\ninstructs Ruff (and other tools) to respect the re-export, and avoid\nmarking it as unused, as in:\n\n```python\nfrom module import member as member\n```\n\nAlternatively, you can use `__all__` to declare a symbol as part of the module's\ninterface, as in:\n\n```python\n# __init__.py\nimport some_module\n\n__all__ = [\"some_module\"]\n```\n\n## Preview\nWhen [preview] is enabled (and certain simplifying assumptions\nare met), we analyze all import statements for a given module\nwhen determining whether an import is used, rather than simply\nthe last of these statements. This can result in both different and\nmore import statements being marked as unused.\n\nFor example, if a module consists of\n\n```python\nimport a\nimport a.b\n```\n\nthen both statements are marked as unused under [preview], whereas\nonly the second is marked as unused under stable behavior.\n\nAs another example, if a module consists of\n\n```python\nimport a.b\nimport a\n\na.b.foo()\n```\n\nthen a diagnostic will be emitted for the second line under [preview],\nwhereas no diagnostic is emitted under stable behavior.\n\nNote that this behavior is somewhat subjective and is designed\nto conform to the developer's intuition rather than Python's actual\nexecution. To wit, the statement `import a.b` automatically executes\n`import a`, so in some sense `import a` is _always_ redundant\nin the presence of `import a.b`.\n\n\n## Fix safety\n\nFixes to remove unused imports are safe, except in `__init__.py` files.\n\nApplying fixes to `__init__.py` files is currently in preview. The fix offered depends on the\ntype of the unused import. Ruff will suggest a safe fix to export first-party imports with\neither a redundant alias or, if already present in the file, an `__all__` entry. If multiple\n`__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix\nto remove third-party and standard library imports -- the fix is unsafe because the module's\ninterface changes.\n\nSee [this FAQ section](https://docs.astral.sh/ruff/faq/#how-does-ruff-determine-which-of-my-imports-are-first-party-third-party-etc)\nfor more details on how Ruff\ndetermines whether an import is first or third-party.\n\n## Example\n\n```python\nimport numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n```\n\nUse instead:\n\n```python\ndef area(radius):\n return 3.14 * radius**2\n```\n\nTo check the availability of a module, use `importlib.util.find_spec`:\n\n```python\nfrom importlib.util import find_spec\n\nif find_spec(\"numpy\") is not None:\n print(\"numpy is installed\")\nelse:\n print(\"numpy is not installed\")\n```\n\n## basedpython\na `.by` or `.byi` file is not checked. an import there can be used without its\nname ever being read: the `extension`s and conformances a module declares are\nonly applicable in the files that import it, and whether one of them supplies\nsomething a file relies on is a question about types, which a linter checking one\nfile at a time cannot answer. `by check` reports unused imports in `.by` files\ninstead, with this rule's semantics, as `unused-import`\n\n## Options\n- `lint.ignore-init-module-imports`\n- `lint.pyflakes.allowed-unused-imports`\n\n## References\n- [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement)\n- [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec)\n- [Typing documentation: interface conventions](https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols)\n\n[preview]: https://docs.astral.sh/ruff/preview/\n" }, "help": { "text": "`{name}` imported but unused; consider using `importlib.util.find_spec` to test for availability" diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/unused_import.rs b/crates/ruff_linter/src/rules/pyflakes/rules/unused_import.rs index cd03b14cc2..8696ceac28 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/unused_import.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/unused_import.rs @@ -132,6 +132,14 @@ use crate::{Applicability, Fix, FixAvailability, Violation}; /// print("numpy is not installed") /// ``` /// +/// ## basedpython +/// a `.by` or `.byi` file is not checked. an import there can be used without its +/// name ever being read: the `extension`s and conformances a module declares are +/// only applicable in the files that import it, and whether one of them supplies +/// something a file relies on is a question about types, which a linter checking one +/// file at a time cannot answer. `by check` reports unused imports in `.by` files +/// instead, with this rule's semantics, as `unused-import` +/// /// ## Options /// - `lint.ignore-init-module-imports` /// - `lint.pyflakes.allowed-unused-imports` @@ -326,6 +334,13 @@ fn find_dunder_all_exprs<'a>(semantic: &'a SemanticModel) -> Vec<&'a ast::Expr> /// ¬__init__.py ∧ 3rdpty → safe, remove /// pub(crate) fn unused_import(checker: &Checker, scope: &Scope) { + // basedpython: the checker's `unused-import` owns this rule for `.by` files, because an + // import used only through the extensions its module declares reads no name this + // per-file, typeless analysis could see + if checker.source_type.is_basedpython() { + return; + } + // Collect all unused imports by statement. let mut unused: BTreeMap<(NodeId, Exceptions), Vec> = BTreeMap::default(); let mut ignored: BTreeMap<(NodeId, Exceptions), Vec> = BTreeMap::default(); diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-import_F401_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-import_F401_basedpython.by.snap index 57f888e5fc..d0b409f39e 100644 --- a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-import_F401_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-import_F401_basedpython.by.snap @@ -1,18 +1,4 @@ --- source: crates/ruff_linter/src/rules/pyflakes/mod.rs --- -F401 [*] `c.unused_name` imported but unused - --> F401_basedpython.by:8:15 - | - 7 | # a plain import of an unreferenced name is still reported - 8 | from c import unused_name - | ^^^^^^^^^^^ - 9 | -10 | # and an exported name may of course also be used locally - | -help: Remove unused import: `c.unused_name` - | -7 | # a plain import of an unreferenced name is still reported - - from c import unused_name -8 | - | + From 2b3ea61ebf41366ec3779fea95f59b4c91e5be17 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:29:29 +1000 Subject: [PATCH 24/47] document `unused-import`, F401 leaving `.by` files to it, and that an import kept for its module's extensions is used; regenerate the rule docs and schema --- crates/ty/docs/rules.md | 486 ++++++++++++-------- docs/basedpython/features/extensions.md | 12 +- docs/basedpython/features/index.md | 2 + docs/basedpython/features/linter.md | 8 + docs/basedpython/features/unused-imports.md | 110 +++++ ty.schema.json | 10 + zensical.toml | 1 + 7 files changed, 421 insertions(+), 208 deletions(-) create mode 100644 docs/basedpython/features/unused-imports.md diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index 384ee2a04f..bac390e807 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -8,7 +8,7 @@ Default level: error · Added in 0.0.64 · Related issues · -View source +View source @@ -44,7 +44,7 @@ class Base(ABC): Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -90,7 +90,7 @@ class Derived(Base): # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -123,7 +123,7 @@ f(1, b=s1) # ok — explicit Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -159,7 +159,7 @@ report(Celsius()) # error: two conversions apply Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -194,7 +194,7 @@ extension list: Default level: warn · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -258,7 +258,7 @@ class SubProto(BaseProto, Protocol): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -340,7 +340,7 @@ value = unknown # ty: ignore[unresolved-reference] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -383,7 +383,7 @@ a4 = True + 1 # ok — a boolean used as a boolean Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -435,7 +435,7 @@ Foo.method() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -463,7 +463,7 @@ Calling a non-callable object will raise a `TypeError` at runtime. Default level: error · Added in 0.0.7 · Related issues · -View source +View source @@ -498,7 +498,7 @@ def f(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -546,7 +546,7 @@ def main(): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -593,7 +593,7 @@ def Fixed(show: bool): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -626,7 +626,7 @@ a = 1 # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -660,7 +660,7 @@ class C(A, B): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -706,7 +706,7 @@ def App(done: bool) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -740,7 +740,7 @@ class B(A): ... # error Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -785,7 +785,7 @@ type Tree = int | list[Tree] # valid recursive alias Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -821,7 +821,7 @@ class Example: Default level: warn · Added in 0.0.1-alpha.16 · Related issues · -View source +View source @@ -860,7 +860,7 @@ old_func() # error: [deprecated] Default level: ignore · Added in 0.0.78 · Related issues · -View source +View source @@ -1032,7 +1032,7 @@ soundness checks from their type checker, and it may have false positives in som Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1064,7 +1064,7 @@ This rule is currently disabled by default because of the number of false positi Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1095,7 +1095,7 @@ class B(A, A): ... # error Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -1135,7 +1135,7 @@ class A: # error Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -1246,7 +1246,7 @@ Python code. Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -1295,7 +1295,7 @@ def bar() -> str: # error: [empty-body] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -1356,7 +1356,7 @@ def h(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -1458,7 +1458,7 @@ def foo() -> "intt\b": ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1497,7 +1497,7 @@ def f(local fn: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1538,7 +1538,7 @@ for x in [1, 2, 3]: Default level: warn · Added in 0.0.50 · Related issues · -View source +View source @@ -1578,7 +1578,7 @@ def g(value: ~A) -> None: ... # error: [experimental-syntax] Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -1612,7 +1612,7 @@ def my_function() -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.40 · Related issues · -View source +View source @@ -1647,7 +1647,7 @@ let a = 1 Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -1764,7 +1764,7 @@ def test() -> "Literal[5]": Default level: ignore · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -1817,7 +1817,7 @@ unpacking — is not a declaration, and is never reported. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -1892,7 +1892,7 @@ print(Labelled) # warning: prints `` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -1931,7 +1931,7 @@ def audit(account: Account) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1967,7 +1967,7 @@ class C(A, B): ... # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1996,7 +1996,7 @@ t[3] # error Default level: warn · Added in 0.0.1-alpha.33 · Related issues · -View source +View source @@ -2033,7 +2033,7 @@ class MyClass: ... Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2069,7 +2069,7 @@ class Point: Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -2164,7 +2164,7 @@ will produce instances with an atypical memory layout. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2196,7 +2196,7 @@ func("foo") # error: [invalid-argument-type] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2226,7 +2226,7 @@ a: int = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2307,7 +2307,7 @@ box.value = 1 # okay Default level: error · Added in 0.0.33 · Related issues · -View source +View source @@ -2352,7 +2352,7 @@ class Sub(Base): Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -2394,7 +2394,7 @@ asyncio.run(main()) Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2421,7 +2421,7 @@ class A(42): ... # error: [invalid-base] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2459,7 +2459,7 @@ build: # error: `build` is an experimental feature, and is off Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.5 · Related issues · -View source +View source @@ -2496,7 +2496,7 @@ extension str(A): # error: `str` does not answer every member of `A` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2525,7 +2525,7 @@ with 1: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -2558,7 +2558,7 @@ class Fahrenheit: Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -2611,7 +2611,7 @@ See: Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -2647,7 +2647,7 @@ class A: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2679,7 +2679,7 @@ a: str # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.82 · Related issues · -View source +View source @@ -2750,7 +2750,7 @@ decorator def traced(fn: (...) -> object, label: str = "") -> int: Default level: warn · Added in 0.0.20 · Related issues · -View source +View source @@ -2806,7 +2806,7 @@ class Pet(Enum): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2871,7 +2871,7 @@ This rule corresponds to Ruff's Default level: error · Added in 0.0.1-alpha.28 · Related issues · -View source +View source @@ -2925,7 +2925,7 @@ class D(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -2958,7 +2958,7 @@ extension list[T: int]: # error: `list` declares no type parameter `T` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -2987,7 +2987,7 @@ Author.objects.filter(name__startswith=1) # error: lookup wants `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -3023,7 +3023,7 @@ def test_user(user: int) -> None: # error: fixture provides `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -3071,7 +3071,7 @@ f"{'name':>10}" # ok Default level: error · Added in 0.0.1-alpha.35 · Related issues · -View source +View source @@ -3121,7 +3121,7 @@ class NonFrozenChild(FrozenBase): # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3176,7 +3176,7 @@ class E(Generic[V]): Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -3271,7 +3271,7 @@ a = 20 / 0 # type: ignore Default level: error · Added in 0.0.1-alpha.17 · Related issues · -View source +View source @@ -3318,7 +3318,7 @@ carol = Person(name="Carol", aeg=25) # typo! Default level: warn · Added in 0.0.15 · Related issues · -View source +View source @@ -3379,7 +3379,7 @@ def f(x, y, /): # Python 3.8+ syntax Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3419,7 +3419,7 @@ def f(t: TypeVar("U")): ... # ty: ignore[invalid-type-form] Default level: error · Added in 0.0.18 · Related issues · -View source +View source @@ -3470,7 +3470,7 @@ match object(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3504,7 +3504,7 @@ class B(metaclass=42): ... # error Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -3621,7 +3621,7 @@ Correct use of `@override` is enforced by ty's [`invalid-explicit-override`](#in Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -3664,7 +3664,7 @@ implements Backend # error: `Backend` is not a protocol Default level: error · Added in 0.0.72 · Related issues · -View source +View source @@ -3702,7 +3702,7 @@ from module import missing # error Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -3767,7 +3767,7 @@ TypeError: typing.ClassVar[int] is not valid as type argument Default level: warn · Added in 0.0.31 · Related issues · -View source +View source @@ -3813,7 +3813,7 @@ admin[0] # "Alice" Default level: error · Added in 0.0.1-alpha.27 · Related issues · -View source +View source @@ -3851,7 +3851,7 @@ Baz = NewType("Baz", int | str) # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3928,7 +3928,7 @@ def foo(x: object = "a", y: int = 0) -> object: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -3966,7 +3966,7 @@ B().f() # 1, not 2 Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4021,7 +4021,7 @@ def h(a: S = 1) -> S: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4054,7 +4054,7 @@ def test_add(a: int, b: int) -> None: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4090,7 +4090,7 @@ P2 = ParamSpec() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4142,7 +4142,7 @@ Declare the type variable with `TypeVar("T", covariant=True)` instead. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4213,7 +4213,7 @@ def g(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -4248,7 +4248,7 @@ def g[T](value: T) raises T: # error: `T@g` is not always an exception Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4281,7 +4281,7 @@ if m := re.match("(a)(b)", "ab"): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -4313,7 +4313,7 @@ type Alias[reified T] = list[T] # error: an alias's parameters are erased Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.83 · Related issues · -View source +View source @@ -4384,7 +4384,7 @@ class Override(Base): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4530,7 +4530,7 @@ def detail(request, pk: int): ... # ok Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -4569,7 +4569,7 @@ import "data/missing.json" as missing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4677,7 +4677,7 @@ class C: ... Default level: error · Added in 0.0.10 · Related issues · -View source +View source @@ -4728,7 +4728,7 @@ class MyClass: Default level: error · Added in 0.0.1-alpha.6 · Related issues · -View source +View source @@ -4774,7 +4774,7 @@ InvalidAlias = TypeAliasType("InvalidAlias", list[T], type_params=(list[T],)) # Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -4840,7 +4840,7 @@ Bar[int] # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4873,7 +4873,7 @@ TYPE_CHECKING = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4908,7 +4908,7 @@ b: Annotated[int] # error Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -4965,7 +4965,7 @@ class C: Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -5031,7 +5031,7 @@ class Owner[U]: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5088,7 +5088,7 @@ V = TypeVar("V", list[int], int) # valid constrained Type Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -5130,7 +5130,7 @@ U = TypeVar("U", int, str, default=bytes) # error: [invalid-type-variable-defau Default level: error · Added in 0.0.28 · Related issues · -View source +View source @@ -5166,7 +5166,7 @@ class Child(Base): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5207,7 +5207,7 @@ def f(options: dict[str, object]): Default level: error · Added in 0.0.9 · Related issues · -View source +View source @@ -5241,7 +5241,7 @@ class Foo(TypedDict): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5282,7 +5282,7 @@ type Alias[out T] = list[T] # error: `list` is invariant Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -5322,7 +5322,7 @@ class Point: Default level: error · Added in 0.0.25 · Related issues · -View source +View source @@ -5356,7 +5356,7 @@ def gen() -> Iterator[int]: Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5422,7 +5422,7 @@ def h(arg2: type): Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -5471,7 +5471,7 @@ def g(arg: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -5502,7 +5502,7 @@ def f(s: str): Default level: warn · Added in 0.0.30 · Related issues · -View source +View source @@ -5544,7 +5544,7 @@ Movie = TypedDict("Film", {"title": str}) # error: [mismatched-type-name] Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -5612,7 +5612,7 @@ and nothing is reported. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5643,7 +5643,7 @@ func() # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5682,7 +5682,7 @@ def g(): ... Default level: ignore · Preview (since 0.0.76) · Related issues · -View source +View source @@ -5771,7 +5771,7 @@ Add `urllib3` to `project.dependencies` if your code imports it directly. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -5799,7 +5799,7 @@ from django.db import models # warning: install `django-stubs` for precise type Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.81 · Related issues · -View source +View source @@ -5851,7 +5851,7 @@ def ignore(event: str): ... Default level: error · Level under ty-compatible: ignore · Added in 0.0.41 · Related issues · -View source +View source @@ -5912,7 +5912,7 @@ class ExplicitChild(Parent): Default level: error · Added in 0.0.75 · Related issues · -View source +View source @@ -5999,7 +5999,7 @@ class Item: Default level: error · Level under ty-compatible: ignore · Added in 0.0.45 · Related issues · -View source +View source @@ -6037,7 +6037,7 @@ def handle(m: re.Match[str]) -> str: Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -6076,7 +6076,7 @@ alice["age"] # KeyError Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -6123,7 +6123,7 @@ def App(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6156,7 +6156,7 @@ def f(a: int | None): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6194,7 +6194,7 @@ func("string") # error: [no-matching-overload] Default level: error · Added in 0.0.30 · Related issues · -View source +View source @@ -6231,7 +6231,7 @@ class Sub(Super): ... # error: [non-callable-init-subclass] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -6262,7 +6262,7 @@ def f(x: int | str) -> int: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6291,7 +6291,7 @@ def f(a: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6335,7 +6335,7 @@ def g(o: object, shape: Shape): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6364,7 +6364,7 @@ for i in 34: # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6392,7 +6392,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6421,7 +6421,7 @@ def f(once done: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6449,7 +6449,7 @@ def f(once done: () -> None): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6491,7 +6491,7 @@ def f(x: int?): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6546,7 +6546,7 @@ def g(name: str | None): Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -6583,7 +6583,7 @@ class B(A): Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -6620,7 +6620,7 @@ class B(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.38 · Related issues · -View source +View source @@ -6663,7 +6663,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6694,7 +6694,7 @@ f(1, x=2) # error Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6725,7 +6725,7 @@ f(x=1) # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6763,7 +6763,7 @@ A.c # error Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6801,7 +6801,7 @@ A()[0] # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6845,7 +6845,7 @@ from module import a # error Default level: warn · Added in 0.0.23 · Related issues · -View source +View source @@ -6877,7 +6877,7 @@ html.parser # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6913,7 +6913,7 @@ print(x) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -6952,7 +6952,7 @@ Id("x") # error: `Id`'s constructor is private Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -6983,7 +6983,7 @@ __all__ = ["helper"] # error: `helper` is private Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7015,7 +7015,7 @@ from helpers import Key # error: `Key` is private to `helpers` Default level: warn · Added in 0.0.60 · Related issues · -View source +View source @@ -7090,7 +7090,7 @@ def test() -> "int": Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7132,7 +7132,7 @@ def g(a: bool | None): Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7167,7 +7167,7 @@ cast(int, f()) # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7221,7 +7221,7 @@ if sys.version_info >= (3, 12): # ok — artificially constant Default level: warn · Added in 0.0.18 · Related issues · -View source +View source @@ -7259,7 +7259,7 @@ class C: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7314,7 +7314,7 @@ class Sub(Base): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7363,7 +7363,7 @@ def f(value: int | str) -> int: Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -7427,7 +7427,7 @@ def g(values: tuple[int, ...]) -> None: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -7460,7 +7460,7 @@ class C: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -7496,7 +7496,7 @@ class C[T]: Default level: warn · Added in 0.0.71 · Related issues · -View source +View source @@ -7539,7 +7539,7 @@ def build(t: Tag) -> None: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -7583,7 +7583,7 @@ class Outer[T]: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7632,7 +7632,7 @@ def Observed(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7676,7 +7676,7 @@ def Counter(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7710,7 +7710,7 @@ static_assert(int(2.0 * 3.0) == 6) # error Default level: warn · Added in 0.0.39 · Related issues · -View source +View source @@ -7762,7 +7762,7 @@ limitation. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7796,7 +7796,7 @@ class B(A): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7828,7 +7828,7 @@ class Circle(Shape): ... # error: `Shape` is sealed in another workspace Default level: error · Added in 0.0.1-alpha.30 · Related issues · -View source +View source @@ -7937,7 +7937,7 @@ class Book: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7967,7 +7967,7 @@ f("foo") # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8006,7 +8006,7 @@ def find(items: list[int]) -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8053,7 +8053,7 @@ g: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8088,7 +8088,7 @@ f: # error: the block returns `None`, not `str` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8127,7 +8127,7 @@ def _(x: int): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -8158,7 +8158,7 @@ class User(BaseModel): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8217,7 +8217,7 @@ class A: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -8290,7 +8290,7 @@ the project registers with `@register.simple_block_tag`. Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -8356,7 +8356,7 @@ what the projects depending on it read. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8385,7 +8385,7 @@ def f() raises TypeError: Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8414,7 +8414,7 @@ reveal_type(1) # revealed: Literal[1] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8452,7 +8452,7 @@ def check(x: int | None) -> asserts x: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8482,7 +8482,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8513,7 +8513,7 @@ f(x=1, y=2) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -8724,7 +8724,7 @@ page does not render at all. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -8758,7 +8758,7 @@ implements Backend # error: this module does not answer `Backend` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8817,7 +8817,7 @@ def Held(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8850,7 +8850,7 @@ A().foo # error Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -8925,7 +8925,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8953,7 +8953,7 @@ import foo # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8982,7 +8982,7 @@ def check(value: int | None) -> asserts values: # error: `values` is nothing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9100,7 +9100,7 @@ is one whose template set cannot be established. Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -9228,7 +9228,7 @@ Python code. Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -9269,7 +9269,7 @@ def f(a: object, b: int, c: Any): Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9414,7 +9414,7 @@ Python code. Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9561,7 +9561,7 @@ generator boundaries. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -9611,7 +9611,7 @@ A() # error: nothing says which specialization this is Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -9658,7 +9658,7 @@ def Observed(items: StateList[int]): ... # ok: an observable handle Default level: warn · Added in 0.0.1-alpha.7 · Related issues · -View source +View source @@ -9704,7 +9704,7 @@ class D(C): ... # error: [unsupported-base] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9753,7 +9753,7 @@ b1 < b2 < b1 # error Default level: warn · Level under ty-compatible: ignore · Added in 0.0.12 · Related issues · -View source +View source @@ -9798,7 +9798,7 @@ def factory(base: type[Base]) -> type: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9830,7 +9830,7 @@ A() + A() # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -9873,7 +9873,7 @@ reveal_type(project.root) # revealed: "." Default level: warn · Added in 0.0.21 · Related issues · -View source +View source @@ -9946,13 +9946,93 @@ Set [`analysis.respect-type-ignore-comments`](https://docs.astral.sh/ty/reference/configuration/#respect-type-ignore-comments) to `false` to prevent this rule from reporting unused `type: ignore` comments. +## `unused-import` + + +Default level: warn · basedpython only, so absent under ty-compatible · +Added in 0.0.1-beta.2 · +Related issues · +View source + + + +**What it does** + +Checks a basedpython file for an import that nothing uses: no name it binds +is read, and no `extension` or conformance it brings into scope supplies +anything the file relies on. + +**Why is this bad?** + +An unused import costs time when the module is imported, can create an +import cycle, and tells a reader the module depends on something it does +not. + +The extension members a module declares are only applicable in the files +that import it, so an import can be used without its name ever being read. +This rule counts that as a use, which a linter working one file at a time +cannot — so in a `.by` file it takes the place of buff's `F401`, which +leaves `.by` files alone. + +Otherwise the rule is `F401`'s: + +- a read of a binding counts for every later binding of that name in the + scope. An import nothing reads before another import rebinds the name + is reported too, as `F811` reports the redefinition, so one round of + fixes removes every one. An import inside a class body is never + reported +- `import a as a`, `from a import b as b` and `from a export b` are + re-exports, a name listed in `__all__` is exported, and + `from __future__ import …` is a directive — none of them is reported +- in a package's `__init__.by`, and inside a `try` that handles + `ImportError`, an unused import is reported without a fix: it may be a + re-export, or a test for whether the module is installed + +**Examples** + +```by +# colours.by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +```by +# main.by +import json # error: `json` imported but unused +import colours # ok — the extension it declares is used below +from enum import Enum + +class Colour(Enum): + RED = 1 + +def describe(colour: Colour) -> str: + return colour.lower_name +``` + +Use instead: + +```by +import colours +from enum import Enum +``` + +**Fix safety** + +The fix removes the import, and is safe: nothing in the file reads it. +Removing an import also stops the imported module from being run, which +matters only if the module's import has a side effect the program relies +on. + ## `unused-return-value` Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.71 · Related issues · -View source +View source @@ -10038,7 +10118,7 @@ to `false`. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.81 · Related issues · -View source +View source @@ -10083,7 +10163,7 @@ parse("1") Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -10162,7 +10242,7 @@ def foo(x: int | str) -> int | str: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/docs/basedpython/features/extensions.md b/docs/basedpython/features/extensions.md index 60fddbfad8..6924408864 100644 --- a/docs/basedpython/features/extensions.md +++ b/docs/basedpython/features/extensions.md @@ -309,9 +309,13 @@ greeting.dedented() naming what a module declares is the usual way a file depends on it — a conformance is written against an interface imported by name — so -`from mod import Show` carries `mod`'s extensions too. requiring a separate -`import mod` whose symbols were never used would leave an import that reads as -removable to anyone tidying the file, silently withdrawing conformance +`from mod import Show` carries `mod`'s extensions too. an import of `mod` +anywhere in the file is enough, inside a function included + +a bare `import mod` whose name is never read is not unused: `by check` counts an +import as used when an extension or conformance it makes applicable supplies +something the file relies on, so [`unused-import`](unused-imports.md) does not +report it and `by check --fix` keeps it nothing is registered globally and nothing is monkeypatched, so two dependencies cannot fight over the same pair: their conformances are simply not visible to @@ -514,5 +518,3 @@ header already carries is not reported yet, and the last module imported wins at runtime - a requirement cannot be reached through an optional chain (`value?.show()`) or assigned through the interface; both are reported rather than lowered -- an unused `import` that carries a conformance is still reported by `F401`, - whose autofix would remove it and silently withdraw the conformance diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index e25d7a63c8..7dab9b8871 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -45,6 +45,8 @@ features that apply to a project rather than to a file language is written inside it - [linting](linter.md) — the `BY` rules, and how ruff's own rules read `.by` source +- [unused imports](unused-imports.md) — an import nothing uses, counting the extensions it brings + into scope - [the project server](project-server.md) — `by check` answered out of the language server's warm state diff --git a/docs/basedpython/features/linter.md b/docs/basedpython/features/linter.md index 3e8550f015..8e17f34eb5 100644 --- a/docs/basedpython/features/linter.md +++ b/docs/basedpython/features/linter.md @@ -66,6 +66,14 @@ here, and why `t[0]` is not reported as a [tuple member access](tuple-index.md) — whether `t` is a tuple is a type question +`F401` is the one upstream rule that follows from that split. whether an +import is used can be a type question in a `.by` file — an import may be there +only for the [extensions](extensions.md) its module declares — so `F401` does +not check `.by` or `.byi` files, and `by check` reports them as +[`unused-import`](unused-imports.md), with `F401`'s rules. a `# noqa: F401` in a +`.by` file suppresses nothing any more, and `RUF100` reports it where it is +enabled; write `# ty: ignore[unused-import]` instead + ## upstream rules on `.by` source ruff's own rules are written for python source, and two things follow from that. diff --git a/docs/basedpython/features/unused-imports.md b/docs/basedpython/features/unused-imports.md new file mode 100644 index 0000000000..f522a26fc5 --- /dev/null +++ b/docs/basedpython/features/unused-imports.md @@ -0,0 +1,110 @@ +# unused imports + +`by check` reports an import in a `.by` file that nothing uses, as `unused-import`, and +`by check --fix` removes it + +```by +import json # warning: `json` imported but unused +from enum import Enum + +class Colour(Enum): + RED = 1 +``` + +an import is used when a name it binds is read — and also when the module it names declares an +[extension](extensions.md) or a conformance that supplies something the file relies on. an +extension's members are only applicable in the files that import its module, so an import can be +the reason `colour.lower_name` resolves without the module's name ever being written again + +```by +# colours.by +from enum import Enum + +extension Enum: + let lower_name: str + get() = self.name.lower() +``` + +```by +# main.by +import colours # ok: the extension it declares is used below +from enum import Enum + +class Colour(Enum): + RED = 1 + +def describe(colour: Colour) -> str: + return colour.lower_name +``` + +each of these counts as a use of the module that declares the extension or conformance: + +- an extension member, read off a value or as a bare name in a + [receiver block](implicit-receivers.md) +- an [operator](extensions.md#operators) or a [conversion](extensions.md#conversions) an extension + supplies +- a conformance that makes a value acceptable where its protocol is asked for, or carries a + protocol extension's members to it +- a [type test](extensions.md#the-type-test) answered from the conformance registry + +a name the emitted python reads although the source never spells it is a use too, of the import +that binds it: the enum a bare `Red` or `case Red:` is lowered to qualify (`Color.Red`), and the +classes a type is written out with — the runtime check of a +[type test](parametric-type-tests.md), a [checked cast](checked-cast.md), a +[`raises`](exceptions.md) guard or a [soundness check](soundness.md), the specialization a +[reified](reified-generics.md) call or construction is given, and the bound of the type parameter +a union parameter is given. `x is AB`, with `type AB = A | B`, checks `A` and `B`. a soundness +check counts at every position, because which ones run is the build's choice + +any import of the module makes its extensions applicable — `import colours`, `import colours.extra` +(for `colours` and `colours.extra`), and `from colours import something` — so each of them is used +when one of the module's extensions is, whether or not anything reads the name it binds. an import +makes them applicable anywhere in the file, so one written inside a function counts too + +## the rules + +apart from extensions, the rule is ruff's `F401`, which in turn is pyflakes': + +- a read of a binding counts for every binding of that name after it in the scope, and a `del` + starts the name afresh. an import nothing reads before the name is rebound is reported when + another import rebinds it — what `F811` reports as a redefinition — so one round of fixes + removes every one; an import a submodule import follows, or the other way round, is only + redefined by an import of the same module. an import rebound by anything else is left to that + rebinding +- an import in a class body is never reported, nor one that binds a `global` or `nonlocal` name +- `import a as a`, `from a import b as b` and [`from a export b`](export-imports.md) are re-exports, + a name listed in `__all__` is exported, and `from __future__ import …` is a directive — none is + reported +- a read inside a `TYPE_CHECKING` block counts, and an annotation may read an import written after + it. a string in a type position is a [literal type](literal-types.md), so it reads nothing +- a file with a syntax error is not checked: the parser's recovery may have dropped a read + +two kinds of unused import are reported without a fix: + +- one in a package's `__init__.by`, which may be how the package exports the name. for one of the + project's own modules the report says how to make the re-export explicit +- one in the body of a `try` that handles `ImportError`, which is usually a test for whether the + module is installed. the report suggests `importlib.util.find_spec` + +## buff + +buff's `F401` does not check `.by` or `.byi` files. a linter reads one file at a time and has no +types, so it cannot see that an extension supplied something, and its fix would remove the import +that makes the extension applicable. `unused-import` is the rule there instead + +that changes how a suppression is written. `# noqa: F401` in a `.by` file no longer suppresses +anything, and `RUF100`, where it is enabled, reports it as unused and removes it. an import that +needs to stay although nothing uses it is suppressed for `by check` instead: + +```by +import side_effects # ty: ignore[unused-import] +``` + +`by check --add-ignore` writes that comment on every unused import still reported. +`lint.pyflakes.allowed-unused-imports` is buff's setting and does not apply to `by check`; turn the +rule off for a set of files with a [per-path override](../configuration.md#per-path-overrides) instead + +## related + +- [extensions](extensions.md) — what an extension is, and how its import makes it applicable +- [linting](linter.md) — how ruff's other rules read `.by` source diff --git a/ty.schema.json b/ty.schema.json index da4e523aef..020c4c8f73 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -2911,6 +2911,16 @@ } ] }, + "unused-import": { + "title": "detects an import that nothing uses", + "description": "## What it does\nChecks a basedpython file for an import that nothing uses: no name it binds\nis read, and no `extension` or conformance it brings into scope supplies\nanything the file relies on.\n\n## Why is this bad?\nAn unused import costs time when the module is imported, can create an\nimport cycle, and tells a reader the module depends on something it does\nnot.\n\nThe extension members a module declares are only applicable in the files\nthat import it, so an import can be used without its name ever being read.\nThis rule counts that as a use, which a linter working one file at a time\ncannot — so in a `.by` file it takes the place of buff's `F401`, which\nleaves `.by` files alone.\n\nOtherwise the rule is `F401`'s:\n\n- a read of a binding counts for every later binding of that name in the\n scope. An import nothing reads before another import rebinds the name\n is reported too, as `F811` reports the redefinition, so one round of\n fixes removes every one. An import inside a class body is never\n reported\n- `import a as a`, `from a import b as b` and `from a export b` are\n re-exports, a name listed in `__all__` is exported, and\n `from __future__ import …` is a directive — none of them is reported\n- in a package's `__init__.by`, and inside a `try` that handles\n `ImportError`, an unused import is reported without a fix: it may be a\n re-export, or a test for whether the module is installed\n\n## Examples\n```by\n# colours.by\nfrom enum import Enum\n\nextension Enum:\n let lower_name: str\n get() = self.name.lower()\n```\n\n```by\n# main.by\nimport json # error: `json` imported but unused\nimport colours # ok — the extension it declares is used below\nfrom enum import Enum\n\nclass Colour(Enum):\n RED = 1\n\ndef describe(colour: Colour) -> str:\n return colour.lower_name\n```\n\nUse instead:\n\n```by\nimport colours\nfrom enum import Enum\n```\n\n## Fix safety\nThe fix removes the import, and is safe: nothing in the file reads it.\nRemoving an import also stops the imported module from being run, which\nmatters only if the module's import has a side effect the program relies\non.", + "default": "warn", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "unused-return-value": { "title": "detects a call whose result is thrown away", "description": "## What it does\nChecks for a call, written as a statement of its own, whose result is\nthrown away.\n\n## Why is this bad?\nA call on a line of its own keeps whatever the call did on the way to its\nanswer, and discards the answer. For most functions that is a mistake:\n`text.strip()` on its own line reads as if it strips `text`, and strips\nnothing — `str` is immutable, and the stripped string was the result.\n\nA function whose result really is optional says so with\n`@ignorable_return_value`, and a member of a class that says so says\notherwise with `@must_use_return_value`. A function returning `None` has\nno result to discard and is never reported.\n\n## Examples\n```python\ndef parse(text: str) -> int: ...\ndef log(message: str) -> None: ...\n\nparse(\"1\") # warning: the parsed number goes nowhere\nlog(\"started\") # ok — nothing was returned\n```\n\n```by\n@ignorable_return_value\ndef prime_cache(key: str) -> bytes: ...\n\nprime_cache(\"k\") # ok — the call was for the caching\n```", diff --git a/zensical.toml b/zensical.toml index 2db307f10f..d852eaee25 100644 --- a/zensical.toml +++ b/zensical.toml @@ -93,6 +93,7 @@ features = [ { "editor support" = "features/editor.md" }, { "language injection" = "features/language-injection.md" }, { "linting" = "features/linter.md" }, + { "unused imports" = "features/unused-imports.md" }, { "the project server" = "features/project-server.md" }, ] }, { "standard library" = [ From b25fc640811c66d9cd503b7265ba034fc9d6cd5a Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:47:04 +1000 Subject: [PATCH 25/47] move the choice of where runtime soundness checks go into the checker, as one query the transpiler and the native backend both ask; the emitted python is unchanged --- crates/by_transforms/src/config.rs | 85 +--- .../by_transforms/src/transforms/soundness.rs | 360 ++----------- crates/by_transforms/src/type_info.rs | 129 +---- crates/ty_python_semantic/src/types.rs | 1 + .../ty_python_semantic/src/types/soundness.rs | 14 +- .../src/types/soundness_sites.rs | 472 ++++++++++++++++++ 6 files changed, 545 insertions(+), 516 deletions(-) create mode 100644 crates/ty_python_semantic/src/types/soundness_sites.rs diff --git a/crates/by_transforms/src/config.rs b/crates/by_transforms/src/config.rs index c3fa6db20e..562612cfdd 100644 --- a/crates/by_transforms/src/config.rs +++ b/crates/by_transforms/src/config.rs @@ -1,5 +1,7 @@ use std::collections::BTreeMap; +pub use ty_python_semantic::types::soundness_sites::SoundnessPositions; + pub use ruff_python_ast::PythonVersion; #[derive(Debug, Clone)] @@ -101,89 +103,6 @@ pub enum FloatLiteralLowering { Literal, } -/// Per-position toggles for the runtime type-soundness checks. Each field -/// enables validation at one syntactic position where ty accepts a value on -/// an annotation-level claim it can't verify at runtime. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[expect( - clippy::struct_excessive_bools, - reason = "independent per-position toggles, not a state machine" -)] -pub struct SoundnessPositions { - /// results of calls whose type is typevar-derived — a generic function's - /// return (`t[T]() -> T`) or a method bound to a specialized generic - /// instance (`dict[str, int].get`) - pub generic_calls: bool, - /// element reads out of a specialized container (`a[0]` on `list[str]`) - pub projections: bool, - /// loop / comprehension elements drawn from a specialized iterable - pub iterations: bool, - /// explicit `Any` (or a context-solved generic result) flowing into an - /// annotated assignment target (`a: str = any_val`) - pub assignments: bool, - /// a returned value validated against the enclosing function's declared - /// return type (`def g() -> str: return any_val`) - pub returns: bool, - /// a call argument validated against its matched parameter's annotation - /// (`takes(any_val)` where `takes(s: str)`) - pub arguments: bool, - /// a function's own parameters validated against their annotations at - /// entry, defending its contract against callers the checker never saw - /// (untyped or third-party code). off in the default set — it runs on - /// every call, so it's opt-in — but included in [`Self::all`] - pub parameters: bool, -} - -impl SoundnessPositions { - /// every position enabled, including the opt-in defensive `parameters` - /// entry checks (the CLI `--soundness all`) - pub fn all() -> Self { - Self { - parameters: true, - ..Self::defaults() - } - } - - /// the positions enabled without an explicit request: every inference-gap - /// check, but *not* the defensive `parameters` entry checks (those add a - /// check on every call, so they stay opt-in) - pub fn defaults() -> Self { - Self { - generic_calls: true, - projections: true, - iterations: true, - assignments: true, - returns: true, - arguments: true, - parameters: false, - } - } - - /// no position enabled — the pass is a no-op - pub fn none() -> Self { - Self { - generic_calls: false, - projections: false, - iterations: false, - assignments: false, - returns: false, - arguments: false, - parameters: false, - } - } - - /// whether any position is enabled - pub(crate) fn any(self) -> bool { - self.generic_calls - || self.projections - || self.iterations - || self.assignments - || self.returns - || self.arguments - || self.parameters - } -} - impl Default for Config { fn default() -> Self { Self { diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index 1b804bf43b..e4c9330990 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -69,13 +69,12 @@ use std::collections::HashMap; use ruff_python_ast::name::Name; use ruff_python_ast::visitor::transformer::{self, Transformer}; -use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{ - AtomicNodeIndex, Comprehension, Expr, ExprCall, ExprName, HasNodeIndex, Parameter, Stmt, - StmtFunctionDef, UnaryOp, + AtomicNodeIndex, Expr, ExprCall, ExprName, HasNodeIndex, Stmt, StmtFunctionDef, }; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::types::soundness::CheckTarget; +use ty_python_semantic::types::soundness_sites::SoundnessSite; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::parametric_is::variance_tuple; @@ -92,13 +91,9 @@ use crate::type_info::{SoundnessCheck, TypeInfo}; // leaving the base `isinstance` as the guarantee. reuses `_parametric_is` // (and its `_parametric_is_sub`) from `PARAMETRIC_IS_RUNTIME` -/// which parameter an argument binds to, for the `arguments` gate -enum ArgSlot<'a> { - Positional(usize), - Keyword(&'a str), -} - -/// one check the pass decided on, before anything is written for it +/// one check the pass makes, before anything is written for it: a site the checker's +/// [`soundness_sites`](ty_python_semantic::types::soundness_sites::soundness_sites) +/// decided on, with each checked parameter named as python binds it enum Site<'ast> { /// the value `expr` produces is checked as it is produced Value { @@ -120,298 +115,42 @@ enum Site<'ast> { }, } -/// where the checks go, decided once over a syntax tree, and what is not a place -/// for one -struct Soundness<'a, 'ast> { - types: &'a dyn TypeInfo, +/// the checks `suite` makes under `positions`, as the checker decides them. `written` is +/// the module as written, which a repeated `_` parameter's name is numbered around +fn sites<'ast>( + types: &dyn TypeInfo, + written: WrittenNames, + suite: &'ast [Stmt], positions: SoundnessPositions, - /// the module as written, which a repeated `_` parameter's name is numbered around - written: WrittenNames<'a>, - sites: Vec>, - /// gated expressions already covered by a wrap on an enclosing `!` - /// force-unwrap — wrapping them directly would splice the check's second - /// argument into `_force_unwrap`'s call parens - consumed: Vec, - /// declared-return-type check plans of the enclosing functions (innermost - /// last); `None` when a function has no annotation or an uncheckable one - return_targets: Vec>, -} - -impl<'a, 'ast> Soundness<'a, 'ast> { - fn new( - types: &'a dyn TypeInfo, - positions: SoundnessPositions, - written: WrittenNames<'a>, - ) -> Self { - Self { - types, - positions, - written, - sites: Vec::new(), - consumed: Vec::new(), - return_targets: Vec::new(), - } - } - - /// whether `expr` is a value this pass classifies as resting on an - /// unverified claim: a call whose result type came from a typevar - /// solution, or an element read out of a specialized container. this is - /// the position-agnostic *classification*; the direct-wrap sites also - /// require the corresponding position to be enabled ([`Self::gated_enabled`]) - fn gated(&self, expr: &Expr) -> bool { - match expr { - Expr::Call(call) => self.types.call_result_is_typevar_derived(&call.func), - Expr::Subscript(subscript) => { - subscript.ctx.is_load() - && self.types.is_specialized_generic_instance(&subscript.value) - } - _ => false, - } - } - - /// [`Self::gated`] restricted to the enabled positions: a `Call` needs - /// `generic_calls`, a `Subscript` needs `projections` - fn gated_enabled(&self, expr: &Expr) -> bool { - match expr { - Expr::Call(_) => self.positions.generic_calls && self.gated(expr), - Expr::Subscript(_) => self.positions.projections && self.gated(expr), - _ => false, - } - } - - /// whether `value` is "unhelpful" — its own inferred type doesn't back a - /// standalone check, so a surrounding annotation (assignment / return / - /// parameter) must supply the target: a plain `Any`, or a gated value - /// whose type is unresolved (a typevar solved only by that context) - fn value_needs_context_target(&self, value: &Expr) -> bool { - self.types.is_any(value) || (self.gated(value) && self.check_plan(value).is_none()) - } - - /// the check plan for `expr`'s inferred type, with the pure-`None` - /// isinstance noise case dropped (validating a `None` result guards nothing) - fn check_plan(&self, expr: &Expr) -> Option { - self.types - .soundness_check_plan(expr) - .filter(|plan| !matches!(plan, SoundnessCheck::Isinstance(CheckTarget::NoneType))) - } - - /// [`Self::check_plan`] for a parameter, read off the parameter rather than off - /// its annotation — which is the only way to see a type the source stated with a - /// default instead of with an annotation - fn parameter_plan(&self, parameter: &Parameter) -> Option { - self.types - .parameter_check_plan(parameter) - .filter(|plan| !matches!(plan, SoundnessCheck::Isinstance(CheckTarget::NoneType))) - } - - /// check the value `expr` produces against `plan` - fn check_value(&mut self, expr: &'ast Expr, plan: SoundnessCheck) { - self.sites.push(Site::Value { expr, plan }); - } - - /// check each element drawn from the iterable of a `for` / comprehension - /// clause when the iterable carries a generic specialization and the element - /// (loop target) type is checkable - fn check_iteration(&mut self, iterable: &'ast Expr, target_expr: &Expr, is_async: bool) { - if !self.types.is_specialized_generic_instance(iterable) { - return; - } - let Some(plan) = self.check_plan(target_expr) else { - return; - }; - self.sites.push(Site::Elements { - iterable, - is_async, - plan, - }); - } - - /// check each argument of `call` whose own type is unhelpful against its - /// matched parameter's annotation. positional mapping stops at the first - /// starred spread (positions past it are unknown); `**kwargs` spreads are - /// skipped - fn check_call_arguments(&mut self, call: &'ast ExprCall) { - let callee = call.func.as_ref(); - for (index, arg) in call.arguments.args.iter().enumerate() { - if arg.is_starred_expr() { - break; - } - self.maybe_check_argument(callee, arg, &ArgSlot::Positional(index)); - } - for keyword in &call.arguments.keywords { - if let Some(name) = &keyword.arg { - self.maybe_check_argument(callee, &keyword.value, &ArgSlot::Keyword(name.as_str())); - } - } - } - - fn maybe_check_argument(&mut self, callee: &Expr, arg: &'ast Expr, slot: &ArgSlot<'_>) { - if self.consumed.contains(&arg.range()) || !self.value_needs_context_target(arg) { - return; - } - let plan = match *slot { - ArgSlot::Positional(index) => self.types.call_positional_param_plan(callee, index), - ArgSlot::Keyword(name) => self.types.call_keyword_param_plan(callee, name), - }; - if let Some(plan) = plan { - self.check_value(arg, plan); - } - } - - /// the plan to validate a returned `value` against — the enclosing - /// function's declared return plan — but only when `value`'s own type - /// is unhelpful (else `generic_calls`/`projections` already covers it) - fn return_plan(&self, value: &Expr) -> Option { - let plan = self.return_targets.last()?.clone()?; - self.value_needs_context_target(value).then_some(plan) - } - - /// check each checkable parameter of `func` where its body begins — the - /// `parameters` position, defending the contract against callers the checker - /// never saw. variadic (`*args` / `**kwargs`) parameters are skipped, and so is - /// any parameter whose source states no type: an unannotated one with no - /// default states nothing, and `x=None` says the argument may be left out rather - /// than that `None` belongs there - /// - /// the plan is read off the *parameter*, not off its annotation, because a - /// default is a written type too. `def f(safe='/')` says `safe` is a `str` - /// — the native backend lays the parameter out at that bound and checks it - /// at the boundary, and asking the annotation node left the interpreted twin - /// silently more permissive than its own compiled form - fn check_parameters(&mut self, func: &'ast StmtFunctionDef) { - let params = &func.parameters; - let mut guards = Vec::new(); - for pwd in params - .posonlyargs - .iter() - .chain(¶ms.args) - .chain(¶ms.kwonlyargs) - { - let parameter = &pwd.parameter; - if let Some(plan) = self.parameter_plan(parameter) { - guards.push(( - crate::python_parameter_name(params, parameter, self.written), - plan, - )); - } - } - if !guards.is_empty() { - self.sites.push(Site::Parameters { - function: func, - guards, - }); - } - } -} - -impl<'ast> Visitor<'ast> for Soundness<'_, 'ast> { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - match stmt { - Stmt::FunctionDef(func) => { - if self.positions.parameters { - self.check_parameters(func); - } - // track the declared return plan so nested `return`s validate - // against the right function's annotation - let plan = func - .returns - .as_deref() - .and_then(|returns| self.check_plan(returns)); - self.return_targets.push(plan); - walk_stmt(self, stmt); - self.return_targets.pop(); - return; - } - Stmt::For(for_stmt) => { - if self.positions.iterations { - self.check_iteration(&for_stmt.iter, &for_stmt.target, for_stmt.is_async); - } - } - Stmt::AnnAssign(ann) => { - // the annotation node's stored type is the declared type; - // it backs two checks the value's own type can't: - // - an explicit `Any` flowing into the declared binding - // - a gated value whose typevar solution came from this very - // declaration (`a: str = t()` stores `Unknown` for `t()` — - // the context, not the expression, carries the claim) - if self.positions.assignments - && let Some(value) = &ann.value - && self.value_needs_context_target(value) - // a field-specifier RHS (`a: int = Field()`) is modelled as - // the field type but is a descriptor object at runtime, so a - // soundness check against the annotation would always fail - && !self.types.is_field_specifier(value) - && let Some(plan) = self.check_plan(&ann.annotation) - { - self.check_value(value, plan); - } - } - Stmt::Return(ret) => { - if self.positions.returns - && let Some(value) = &ret.value - && let Some(plan) = self.return_plan(value) - { - self.check_value(value, plan); - } - } - // a type-alias value is a type expression; nothing in it executes - Stmt::TypeAlias(_) => return, - _ => {} - } - walk_stmt(self, stmt); - } - - // annotations are type positions — never wrap inside them - fn visit_annotation(&mut self, _expr: &'ast Expr) {} - - fn visit_comprehension(&mut self, comprehension: &'ast Comprehension) { - if self.positions.iterations { - self.check_iteration( - &comprehension.iter, - &comprehension.target, - comprehension.is_async, - ); - } - ruff_python_ast::visitor::walk_comprehension(self, comprehension); - } - - fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::UnaryOp(unary) = expr - && unary.op == UnaryOp::Force - { - // the force lowering inserts `_force_unwrap(` at the operand's - // first byte; a check wrapped around the bare operand would land - // its second argument inside that call. wrap the whole `expr!` - // instead (the force edits materialize inside the passthrough) - // and consume the operand — and any inner `!` layers — so they - // aren't wrapped again - let already_consumed = self.consumed.contains(&expr.range()); - let mut operand = unary.operand.as_ref(); - while let Expr::UnaryOp(inner) = operand - && inner.op == UnaryOp::Force - { - self.consumed.push(operand.range()); - operand = &inner.operand; - } - if self.gated_enabled(operand) { - self.consumed.push(operand.range()); - if !already_consumed && let Some(plan) = self.check_plan(expr) { - self.check_value(expr, plan); - } - } - } else if self.gated_enabled(expr) - && !self.consumed.contains(&expr.range()) - && let Some(plan) = self.check_plan(expr) - { - self.check_value(expr, plan); - } - if self.positions.arguments - && let Expr::Call(call) = expr - { - self.check_call_arguments(call); - } - walk_expr(self, expr); - } +) -> Vec> { + types + .soundness_sites(suite, positions) + .into_iter() + .map(|site| match site { + SoundnessSite::Value { expr, plan } => Site::Value { expr, plan }, + SoundnessSite::Elements { + iterable, + is_async, + plan, + } => Site::Elements { + iterable, + is_async, + plan, + }, + SoundnessSite::Parameters { function, guards } => Site::Parameters { + function, + guards: guards + .into_iter() + .map(|(parameter, plan)| { + ( + crate::python_parameter_name(&function.parameters, parameter, written), + plan, + ) + }) + .collect(), + }, + }) + .collect() } /// a syntax node, as the checks a module makes are looked up by @@ -477,12 +216,7 @@ pub fn soundness_sites( positions: SoundnessPositions, ) -> SoundnessSites { let mut sites = SoundnessSites::default(); - if !positions.any() { - return sites; - } - let mut walker = Soundness::new(model, positions, written); - walker.visit_body(suite); - for site in walker.sites { + for site in self::sites(model, written, suite, positions) { match site { Site::Value { expr, plan } => { if let Some(key) = SiteKey::of(expr) { @@ -715,14 +449,18 @@ impl TypeAwarePass for SoundnessPass<'_> { let mut inner = Rendered::new(self.written); let mut rerendered = Vec::new(); for (idx, stmt) in stmts.iter().enumerate() { - let mut walker = Soundness::new(types, self.positions, self.written); - walker.visit_stmt(stmt); + let sites = sites( + types, + self.written, + std::slice::from_ref(stmt), + self.positions, + ); // an AST-mutation pass rewrote this statement, and an edit at a node it // replaced would be dropped: its checks go into the tree it is printed from if ctx.changed.contains(&idx) { let mut rendered = Rendered::new(self.written); let mut checks = Vec::new(); - for site in &walker.sites { + for site in &sites { match site { Site::Value { expr, plan } => rendered.wrap_check(expr, plan), Site::Elements { @@ -755,7 +493,7 @@ impl TypeAwarePass for SoundnessPass<'_> { } continue; } - for site in &walker.sites { + for site in &sites { match site { Site::Value { expr, plan } => inner.wrap_check(expr, plan), Site::Elements { diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 2177a73b56..f154e75fcd 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -5,8 +5,7 @@ use crate::transforms::trailing_lambda::RECEIVER_PARAMETER; use ruff_db::parsed::parsed_module; use ruff_python_ast::helpers::is_dotted_name; use ruff_python_ast::{ - Expr, ExprCall, ExprName, ExprRef, Parameter, ParameterWithDefault, Stmt, StmtClassDef, - StmtFunctionDef, + Expr, ExprCall, ExprName, ExprRef, ParameterWithDefault, Stmt, StmtClassDef, StmtFunctionDef, }; use ruff_python_parser::parse_expression; use ruff_text_size::{Ranged, TextRange}; @@ -638,44 +637,13 @@ pub(crate) trait TypeInfo { /// stands for the enclosing class and can never bind to an optional fn optional_wraps(&self, expr: &Expr) -> bool; - /// whether a call through `callee` yields a result whose type was - /// derived by substituting typevars — a generic function's return, or a - /// method bound to a specialized generic instance. such results rest on - /// an assumption ty cannot verify, so the soundness pass validates them - fn call_result_is_typevar_derived(&self, callee: &Expr) -> bool; - - /// whether `expr`'s inferred type is an instance carrying a generic - /// specialization (`list[str]`, a `TypedDict`) — element projections out - /// of it consume an annotation-level claim - fn is_specialized_generic_instance(&self, expr: &Expr) -> bool; - - /// the runtime soundness check for `expr`'s inferred type — a shallow - /// `isinstance` target (`str`, `(int, type(None))`) or a deep parametric - /// check for a user-generic specialization (`A[int]` + variances). `None` - /// when the type has no faithful runtime test or its name doesn't resolve - /// at module scope - fn soundness_check_plan(&self, expr: &Expr) -> Option; - - /// the runtime soundness check for a *parameter*, planned against the type its - /// source states rather than the one its annotation node spells - /// - /// the two differ exactly where there is no annotation to spell. `def f(safe='/')` - /// opens a hole bounded by everything the function requires of `safe`, and a hole - /// has no runtime test — so asking about the annotation answered nothing, and the - /// interpreted leg wrote no check where the native backend was already enforcing - /// the bound at its boundary - fn parameter_check_plan(&self, parameter: &Parameter) -> Option; - - /// the soundness check for the parameter that the positional argument at - /// `index` binds to in a call through `callee` (see - /// [`soundness_check_plan`](TypeInfo::soundness_check_plan)). `None` when - /// the callee isn't a single-overload function/method, the parameter is - /// variadic/unannotated, or its type has no runtime test - fn call_positional_param_plan(&self, callee: &Expr, index: usize) -> Option; - - /// like [`call_positional_param_plan`](TypeInfo::call_positional_param_plan) - /// but for a keyword argument matched by `name` - fn call_keyword_param_plan(&self, callee: &Expr, name: &str) -> Option; + /// every runtime soundness check `suite` makes under `positions` — the checker's one + /// answer, which the native backend reads as well + fn soundness_sites<'ast>( + &self, + suite: &'ast [ruff_python_ast::Stmt], + positions: crate::SoundnessPositions, + ) -> Vec>; /// the runtime check for a type expression used as a checked cast target, /// or `None` when the type has no faithful runtime test. a user generic @@ -826,14 +794,6 @@ pub(crate) trait TypeInfo { /// written fn decides_receiver_binding(&self, decorator: &ruff_python_ast::Decorator) -> bool; - /// whether `expr` is a field-specifier call — `pydantic.Field(...)`, - /// `dataclasses.field(...)`, `attrs.field(...)`, and the like. the checker - /// models these as assignable to the field's declared type, but their - /// runtime value is a field-descriptor object (`FieldInfo`, `Field`), so a - /// soundness check against the field annotation would always fail at - /// runtime — the transpiler must not emit one - fn is_field_specifier(&self, expr: &Expr) -> bool; - /// basedpython: the python a static resource import lowers to, bound to /// `binding` — the document at `path` written out as classes, tuples and /// `Final` literals — or why the document cannot be read. @@ -1425,34 +1385,12 @@ impl TypeInfo for SemanticModel<'_> { ) } - fn call_result_is_typevar_derived(&self, callee: &Expr) -> bool { - callee.inferred_type(self).is_some_and(|ty| { - ty_python_semantic::types::soundness::call_result_is_typevar_derived( - self.db(), - &self.program_environment(), - ty, - ) - }) - } - - fn is_specialized_generic_instance(&self, expr: &Expr) -> bool { - expr.inferred_type(self).is_some_and(|ty| { - ty_python_semantic::types::soundness::is_specialized_generic_instance( - self.db(), - &self.program_environment(), - ty, - ) - }) - } - - fn soundness_check_plan(&self, expr: &Expr) -> Option { - let ty = expr.inferred_type(self)?; - ty_python_semantic::types::soundness::runtime_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ) + fn soundness_sites<'ast>( + &self, + suite: &'ast [ruff_python_ast::Stmt], + positions: crate::SoundnessPositions, + ) -> Vec> { + ty_python_semantic::types::soundness_sites::soundness_sites(self, suite, positions) } fn inherited_parameter_default(&self, parameter: &ParameterWithDefault) -> Option { @@ -1466,38 +1404,6 @@ impl TypeInfo for SemanticModel<'_> { ty_python_semantic::types::ide_support::repeated_underscore_lowering(self, function) } - fn parameter_check_plan(&self, parameter: &Parameter) -> Option { - let ty = parameter.inferred_type(self)?; - ty_python_semantic::types::soundness::parameter_runtime_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ) - } - - fn call_positional_param_plan(&self, callee: &Expr, index: usize) -> Option { - let ty = callee.inferred_type(self)?; - ty_python_semantic::types::soundness::parameter_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ty_python_semantic::types::soundness::ArgSelector::Positional(index), - ) - } - - fn call_keyword_param_plan(&self, callee: &Expr, name: &str) -> Option { - let ty = callee.inferred_type(self)?; - ty_python_semantic::types::soundness::parameter_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - ty_python_semantic::types::soundness::ArgSelector::Keyword(name), - ) - } - fn cast_check_plan(&self, type_expr: &Expr) -> Option { let ty = type_expr.inferred_type(self)?; ty_python_semantic::types::soundness::cast_check_plan( @@ -1715,13 +1621,6 @@ impl TypeInfo for SemanticModel<'_> { .map(|rendered| rendered.source) .map_err(|error| error.to_string()) } - - fn is_field_specifier(&self, expr: &Expr) -> bool { - matches!( - expr.inferred_type(self), - Some(Type::KnownInstance(KnownInstanceType::Field(_))) - ) - } } /// Render a type as python source text. ty's own display is written for the reader of a diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 5e1b4512ed..e22f1ff720 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -239,6 +239,7 @@ mod safe_variance; mod set_theoretic; mod signatures; pub mod soundness; +pub mod soundness_sites; mod special_form; pub(crate) mod state_invalidations; pub(crate) mod state_reads; diff --git a/crates/ty_python_semantic/src/types/soundness.rs b/crates/ty_python_semantic/src/types/soundness.rs index 6b3b52ce23..7e2fac204e 100644 --- a/crates/ty_python_semantic/src/types/soundness.rs +++ b/crates/ty_python_semantic/src/types/soundness.rs @@ -7,7 +7,7 @@ //! these queries answer the two questions that decide an insertion: //! //! - does this expression's type rest on such an assumption? (the gates: -//! [`call_result_is_typevar_derived`], [`is_specialized_generic_instance`]) +//! `call_result_is_typevar_derived`, `is_specialized_generic_instance`) //! - can the inferred type be validated with `isinstance` at runtime, and //! what second argument does that check take? (`runtime_check_target`) @@ -246,7 +246,7 @@ pub fn cast_target_is_unverifiable_protocol<'db>( /// (`def t[T]() -> T`), or the method is bound to a specialized generic /// instance (`dict[str, int].get`), where the specialization itself is an /// unverified annotation-level claim -pub fn call_result_is_typevar_derived<'db>( +pub(crate) fn call_result_is_typevar_derived<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, callee: Type<'db>, @@ -283,7 +283,7 @@ fn mentions_type_var<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Ty /// specialization — the runtime-unverifiable part of an annotation like /// `list[str]`. element projections out of such values (subscripts, method /// results, iteration) are where the specialization's claim is consumed -pub fn is_specialized_generic_instance<'db>( +pub(crate) fn is_specialized_generic_instance<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>, @@ -332,7 +332,7 @@ pub(crate) fn runtime_check_target<'db>( /// arguments are checkable at runtime); otherwise falls back to the shallow /// [`CheckKind::Isinstance`] of `runtime_check_target`. `None` when neither /// applies (no faithful runtime test). -pub fn runtime_check_plan<'db>( +pub(crate) fn runtime_check_plan<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, @@ -361,7 +361,7 @@ pub fn runtime_check_plan<'db>( /// of the bound is recovered from how the body happens to use the parameter, and a /// recovered requirement is not something to raise a `TypeError` over — it is a /// sample, and it can be withdrawn. -pub fn parameter_runtime_check_plan<'db>( +pub(crate) fn parameter_runtime_check_plan<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, @@ -527,7 +527,7 @@ fn builtin_target(db: &dyn Db, file: File, name: &str) -> Option { } /// selects which parameter of a call an argument binds to, for -/// [`parameter_check_plan`] +/// `parameter_check_plan` #[derive(Clone, Copy)] pub enum ArgSelector<'a> { /// a positional argument at the given 0-based index (after any bound @@ -544,7 +544,7 @@ pub enum ArgSelector<'a> { /// misleading), a variadic / unmatched / unannotated parameter, or a /// parameter type with no runtime test. deliberately conservative — /// a missed check is a no-op, a wrong one changes semantics -pub fn parameter_check_plan<'db>( +pub(crate) fn parameter_check_plan<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, diff --git a/crates/ty_python_semantic/src/types/soundness_sites.rs b/crates/ty_python_semantic/src/types/soundness_sites.rs new file mode 100644 index 0000000000..20cfaabd8c --- /dev/null +++ b/crates/ty_python_semantic/src/types/soundness_sites.rs @@ -0,0 +1,472 @@ +//! basedpython: where the transpiled program checks a value at runtime against the type +//! the checker gave it +//! +//! the transpiler writes a check at each site this decides on, and a native build of the +//! same module makes the same checks, so neither decides for itself. the checker asks too: +//! a check names its type's classes as the module binds them, and the import binding one is +//! used by the check although the source never spells the read + +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{Comprehension, Expr, ExprCall, Parameter, Stmt, StmtFunctionDef, UnaryOp}; +use ruff_text_size::{Ranged, TextRange}; + +use crate::Db; +use crate::semantic_model::{HasType, SemanticModel}; +use crate::types::soundness::{ + ArgSelector, CheckKind, CheckTarget, call_result_is_typevar_derived, + is_specialized_generic_instance, parameter_check_plan, parameter_runtime_check_plan, + runtime_check_plan, +}; +use crate::types::{DynamicType, KnownInstanceType, ProgramEnvironment, Type}; + +/// every runtime soundness check `suite` makes under `positions`, in the order the +/// checks are found +pub fn soundness_sites<'ast>( + model: &SemanticModel<'_>, + suite: &'ast [Stmt], + positions: SoundnessPositions, +) -> Vec> { + if !positions.any() { + return Vec::new(); + } + let mut walker = Soundness::new(model, positions); + walker.visit_body(suite); + walker.sites +} + +/// Per-position toggles for the runtime type-soundness checks. Each field +/// enables validation at one syntactic position where ty accepts a value on +/// an annotation-level claim it can't verify at runtime. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[expect( + clippy::struct_excessive_bools, + reason = "independent per-position toggles, not a state machine" +)] +pub struct SoundnessPositions { + /// results of calls whose type is typevar-derived — a generic function's + /// return (`t[T]() -> T`) or a method bound to a specialized generic + /// instance (`dict[str, int].get`) + pub generic_calls: bool, + /// element reads out of a specialized container (`a[0]` on `list[str]`) + pub projections: bool, + /// loop / comprehension elements drawn from a specialized iterable + pub iterations: bool, + /// explicit `Any` (or a context-solved generic result) flowing into an + /// annotated assignment target (`a: str = any_val`) + pub assignments: bool, + /// a returned value validated against the enclosing function's declared + /// return type (`def g() -> str: return any_val`) + pub returns: bool, + /// a call argument validated against its matched parameter's annotation + /// (`takes(any_val)` where `takes(s: str)`) + pub arguments: bool, + /// a function's own parameters validated against their annotations at + /// entry, defending its contract against callers the checker never saw + /// (untyped or third-party code). off in the default set — it runs on + /// every call, so it's opt-in — but included in [`Self::all`] + pub parameters: bool, +} + +impl SoundnessPositions { + /// every position enabled, including the opt-in defensive `parameters` + /// entry checks (the CLI `--soundness all`) + pub fn all() -> Self { + Self { + parameters: true, + ..Self::defaults() + } + } + + /// the positions enabled without an explicit request: every inference-gap + /// check, but *not* the defensive `parameters` entry checks (those add a + /// check on every call, so they stay opt-in) + pub fn defaults() -> Self { + Self { + generic_calls: true, + projections: true, + iterations: true, + assignments: true, + returns: true, + arguments: true, + parameters: false, + } + } + + /// no position enabled — the pass is a no-op + pub fn none() -> Self { + Self { + generic_calls: false, + projections: false, + iterations: false, + assignments: false, + returns: false, + arguments: false, + parameters: false, + } + } + + /// whether any position is enabled + pub fn any(self) -> bool { + self.generic_calls + || self.projections + || self.iterations + || self.assignments + || self.returns + || self.arguments + || self.parameters + } +} + +/// which parameter an argument binds to, for the `arguments` gate +enum ArgSlot<'a> { + Positional(usize), + Keyword(&'a str), +} + +/// one runtime soundness check the transpiled program makes, at the syntax node it is +/// made at +#[derive(Debug, Clone)] +pub enum SoundnessSite<'ast> { + /// the value `expr` produces is checked as it is produced + Value { expr: &'ast Expr, plan: CheckKind }, + /// each element drawn from `iterable` is checked as it is drawn + Elements { + iterable: &'ast Expr, + is_async: bool, + plan: CheckKind, + }, + /// `function`'s own parameters are checked where its body begins, each in the order + /// it is written + Parameters { + function: &'ast StmtFunctionDef, + guards: Vec<(&'ast Parameter, CheckKind)>, + }, +} + +/// where the checks go, decided once over a syntax tree, and what is not a place +/// for one +struct Soundness<'a, 'db, 'ast> { + model: &'a SemanticModel<'db>, + positions: SoundnessPositions, + sites: Vec>, + /// gated expressions already covered by a wrap on an enclosing `!` + /// force-unwrap — wrapping them directly would splice the check's second + /// argument into `_force_unwrap`'s call parens + consumed: Vec, + /// declared-return-type check plans of the enclosing functions (innermost + /// last); `None` when a function has no annotation or an uncheckable one + return_targets: Vec>, +} + +impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { + fn new(model: &'a SemanticModel<'db>, positions: SoundnessPositions) -> Self { + Self { + model, + positions, + sites: Vec::new(), + consumed: Vec::new(), + return_targets: Vec::new(), + } + } + + fn db(&self) -> &'db dyn Db { + self.model.db() + } + + fn env(&self) -> ProgramEnvironment<'db> { + self.model.program_environment() + } + + fn call_result_is_typevar_derived(&self, callee: &Expr) -> bool { + callee + .inferred_type(self.model) + .is_some_and(|ty| call_result_is_typevar_derived(self.db(), &self.env(), ty)) + } + + fn is_specialized_generic_instance(&self, expr: &Expr) -> bool { + expr.inferred_type(self.model) + .is_some_and(|ty| is_specialized_generic_instance(self.db(), &self.env(), ty)) + } + + fn is_any(&self, expr: &Expr) -> bool { + expr.inferred_type(self.model) + .is_some_and(|ty| matches!(ty, Type::Dynamic(DynamicType::Any))) + } + + fn is_field_specifier(&self, expr: &Expr) -> bool { + matches!( + expr.inferred_type(self.model), + Some(Type::KnownInstance(KnownInstanceType::Field(_))) + ) + } + + /// the check for the parameter `selector` binds to in a call through `callee` + fn parameter_check_plan_for( + &self, + callee: &Expr, + selector: ArgSelector<'_>, + ) -> Option { + let ty = callee.inferred_type(self.model)?; + parameter_check_plan(self.db(), &self.env(), self.model.file(), ty, selector) + } + + /// whether `expr` is a value this pass classifies as resting on an + /// unverified claim: a call whose result type came from a typevar + /// solution, or an element read out of a specialized container. this is + /// the position-agnostic *classification*; the direct-wrap sites also + /// require the corresponding position to be enabled ([`Self::gated_enabled`]) + fn gated(&self, expr: &Expr) -> bool { + match expr { + Expr::Call(call) => self.call_result_is_typevar_derived(&call.func), + Expr::Subscript(subscript) => { + subscript.ctx.is_load() && self.is_specialized_generic_instance(&subscript.value) + } + _ => false, + } + } + + /// [`Self::gated`] restricted to the enabled positions: a `Call` needs + /// `generic_calls`, a `Subscript` needs `projections` + fn gated_enabled(&self, expr: &Expr) -> bool { + match expr { + Expr::Call(_) => self.positions.generic_calls && self.gated(expr), + Expr::Subscript(_) => self.positions.projections && self.gated(expr), + _ => false, + } + } + + /// whether `value` is "unhelpful" — its own inferred type doesn't back a + /// standalone check, so a surrounding annotation (assignment / return / + /// parameter) must supply the target: a plain `Any`, or a gated value + /// whose type is unresolved (a typevar solved only by that context) + fn value_needs_context_target(&self, value: &Expr) -> bool { + self.is_any(value) || (self.gated(value) && self.check_plan(value).is_none()) + } + + /// the check plan for `expr`'s inferred type, with the pure-`None` + /// isinstance noise case dropped (validating a `None` result guards nothing) + fn check_plan(&self, expr: &Expr) -> Option { + let ty = expr.inferred_type(self.model)?; + runtime_check_plan(self.db(), &self.env(), self.model.file(), ty) + .filter(|plan| !matches!(plan, CheckKind::Isinstance(CheckTarget::NoneType))) + } + + /// [`Self::check_plan`] for a parameter, read off the parameter rather than off + /// its annotation — which is the only way to see a type the source stated with a + /// default instead of with an annotation + fn parameter_plan(&self, parameter: &Parameter) -> Option { + let ty = parameter.inferred_type(self.model)?; + parameter_runtime_check_plan(self.db(), &self.env(), self.model.file(), ty) + .filter(|plan| !matches!(plan, CheckKind::Isinstance(CheckTarget::NoneType))) + } + + /// check the value `expr` produces against `plan` + fn check_value(&mut self, expr: &'ast Expr, plan: CheckKind) { + self.sites.push(SoundnessSite::Value { expr, plan }); + } + + /// check each element drawn from the iterable of a `for` / comprehension + /// clause when the iterable carries a generic specialization and the element + /// (loop target) type is checkable + fn check_iteration(&mut self, iterable: &'ast Expr, target_expr: &Expr, is_async: bool) { + if !self.is_specialized_generic_instance(iterable) { + return; + } + let Some(plan) = self.check_plan(target_expr) else { + return; + }; + self.sites.push(SoundnessSite::Elements { + iterable, + is_async, + plan, + }); + } + + /// check each argument of `call` whose own type is unhelpful against its + /// matched parameter's annotation. positional mapping stops at the first + /// starred spread (positions past it are unknown); `**kwargs` spreads are + /// skipped + fn check_call_arguments(&mut self, call: &'ast ExprCall) { + let callee = call.func.as_ref(); + for (index, arg) in call.arguments.args.iter().enumerate() { + if arg.is_starred_expr() { + break; + } + self.maybe_check_argument(callee, arg, &ArgSlot::Positional(index)); + } + for keyword in &call.arguments.keywords { + if let Some(name) = &keyword.arg { + self.maybe_check_argument(callee, &keyword.value, &ArgSlot::Keyword(name.as_str())); + } + } + } + + fn maybe_check_argument(&mut self, callee: &Expr, arg: &'ast Expr, slot: &ArgSlot<'_>) { + if self.consumed.contains(&arg.range()) || !self.value_needs_context_target(arg) { + return; + } + let plan = match *slot { + ArgSlot::Positional(index) => { + self.parameter_check_plan_for(callee, ArgSelector::Positional(index)) + } + ArgSlot::Keyword(name) => { + self.parameter_check_plan_for(callee, ArgSelector::Keyword(name)) + } + }; + if let Some(plan) = plan { + self.check_value(arg, plan); + } + } + + /// the plan to validate a returned `value` against — the enclosing + /// function's declared return plan — but only when `value`'s own type + /// is unhelpful (else `generic_calls`/`projections` already covers it) + fn return_plan(&self, value: &Expr) -> Option { + let plan = self.return_targets.last()?.clone()?; + self.value_needs_context_target(value).then_some(plan) + } + + /// check each checkable parameter of `func` where its body begins — the + /// `parameters` position, defending the contract against callers the checker + /// never saw. variadic (`*args` / `**kwargs`) parameters are skipped, and so is + /// any parameter whose source states no type: an unannotated one with no + /// default states nothing, and `x=None` says the argument may be left out rather + /// than that `None` belongs there + /// + /// the plan is read off the *parameter*, not off its annotation, because a + /// default is a written type too. `def f(safe='/')` says `safe` is a `str` + /// — the native backend lays the parameter out at that bound and checks it + /// at the boundary, and asking the annotation node left the interpreted twin + /// silently more permissive than its own compiled form + fn check_parameters(&mut self, func: &'ast StmtFunctionDef) { + let params = &func.parameters; + let mut guards = Vec::new(); + for pwd in params + .posonlyargs + .iter() + .chain(¶ms.args) + .chain(¶ms.kwonlyargs) + { + let parameter = &pwd.parameter; + if let Some(plan) = self.parameter_plan(parameter) { + guards.push((parameter, plan)); + } + } + if !guards.is_empty() { + self.sites.push(SoundnessSite::Parameters { + function: func, + guards, + }); + } + } +} + +impl<'ast> Visitor<'ast> for Soundness<'_, '_, 'ast> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + match stmt { + Stmt::FunctionDef(func) => { + if self.positions.parameters { + self.check_parameters(func); + } + // track the declared return plan so nested `return`s validate + // against the right function's annotation + let plan = func + .returns + .as_deref() + .and_then(|returns| self.check_plan(returns)); + self.return_targets.push(plan); + walk_stmt(self, stmt); + self.return_targets.pop(); + return; + } + Stmt::For(for_stmt) => { + if self.positions.iterations { + self.check_iteration(&for_stmt.iter, &for_stmt.target, for_stmt.is_async); + } + } + Stmt::AnnAssign(ann) => { + // the annotation node's stored type is the declared type; + // it backs two checks the value's own type can't: + // - an explicit `Any` flowing into the declared binding + // - a gated value whose typevar solution came from this very + // declaration (`a: str = t()` stores `Unknown` for `t()` — + // the context, not the expression, carries the claim) + if self.positions.assignments + && let Some(value) = &ann.value + && self.value_needs_context_target(value) + // a field-specifier RHS (`a: int = Field()`) is modelled as + // the field type but is a descriptor object at runtime, so a + // soundness check against the annotation would always fail + && !self.is_field_specifier(value) + && let Some(plan) = self.check_plan(&ann.annotation) + { + self.check_value(value, plan); + } + } + Stmt::Return(ret) => { + if self.positions.returns + && let Some(value) = &ret.value + && let Some(plan) = self.return_plan(value) + { + self.check_value(value, plan); + } + } + // a type-alias value is a type expression; nothing in it executes + Stmt::TypeAlias(_) => return, + _ => {} + } + walk_stmt(self, stmt); + } + + // annotations are type positions — never wrap inside them + fn visit_annotation(&mut self, _expr: &'ast Expr) {} + + fn visit_comprehension(&mut self, comprehension: &'ast Comprehension) { + if self.positions.iterations { + self.check_iteration( + &comprehension.iter, + &comprehension.target, + comprehension.is_async, + ); + } + ruff_python_ast::visitor::walk_comprehension(self, comprehension); + } + + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::UnaryOp(unary) = expr + && unary.op == UnaryOp::Force + { + // the force lowering inserts `_force_unwrap(` at the operand's + // first byte; a check wrapped around the bare operand would land + // its second argument inside that call. wrap the whole `expr!` + // instead (the force edits materialize inside the passthrough) + // and consume the operand — and any inner `!` layers — so they + // aren't wrapped again + let already_consumed = self.consumed.contains(&expr.range()); + let mut operand = unary.operand.as_ref(); + while let Expr::UnaryOp(inner) = operand + && inner.op == UnaryOp::Force + { + self.consumed.push(operand.range()); + operand = &inner.operand; + } + if self.gated_enabled(operand) { + self.consumed.push(operand.range()); + if !already_consumed && let Some(plan) = self.check_plan(expr) { + self.check_value(expr, plan); + } + } + } else if self.gated_enabled(expr) + && !self.consumed.contains(&expr.range()) + && let Some(plan) = self.check_plan(expr) + { + self.check_value(expr, plan); + } + if self.positions.arguments + && let Expr::Call(call) = expr + { + self.check_call_arguments(call); + } + walk_expr(self, expr); + } +} From 01f51f2c18d23348a8b8bf08981c9ed9da3b1296 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:05:20 +1000 Subject: [PATCH 26/47] count an import read only by a type the transpiled program writes out as used: a soundness check's target, at every position a build can enable, a reified call's or construction's specialization, and the arms a union parameter's reified type parameter is bounded by --- .../mdtest/basedpython_unused_import.md | 115 ++++++++++++++++++ crates/ty_python_semantic/src/types.rs | 1 + .../src/types/lowering_reads.rs | 91 ++++++++++++++ .../src/types/soundness_sites.rs | 10 ++ .../src/types/unused_imports.rs | 4 + 5 files changed, 221 insertions(+) create mode 100644 crates/ty_python_semantic/src/types/lowering_reads.rs diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md index 935c2ceb83..9e174a041b 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md @@ -739,6 +739,121 @@ def run() -> None raises Errs: raise Oops() ``` +## an import read only by a soundness check + +the result of a call whose type comes from a type variable is checked at runtime against the type +the checker gave it, `isinstance(…, (A, B))` here, which reads `A` and `B`. a build may turn that +check off, but one that leaves it on reads them, so they are used either way + +`kinds.by`: + +```by +class A: + pass + +class B: + pass + +type AB = A | B +``` + +`main.by`: + +```by +from kinds import AB, A, B + +def pick[T](items: list[T]) -> T: + return items[0] + +def first(xs: list[AB]) -> AB: + y: AB = pick(xs) + return y +``` + +## an import read only by a reified call's specialization + +a bare call of a reified generic function is given the specialization the checker solved, +`describe[Widget]` here + +`widgets.by`: + +```by +class Widget: + pass + +def make() -> Widget: + return Widget() +``` + +`main.by`: + +```by +from widgets import Widget, make + +def describe[reified T](value: T) -> str: + return T.__name__ + +describe(make()) +``` + +## an import read only by a reified construction's specialization + +`widgets.by`: + +```by +class Widget: + pass + +def make() -> Widget: + return Widget() +``` + +`main.by`: + +```by +from widgets import Widget, make + +class Tagged[reified T]: + def __init__(self, value: T) -> None: + self.value = value + +Tagged(make()) +``` + +## an import read only by a union parameter's reified type parameter + +a parameter annotated with a union of specializations of one collection is given a reified type +parameter whose bound spells each arm's class, `(A, B)` here. the annotation spells `A` only through +another module's name for it + +`kinds.by`: + +```by +class A: + pass + +class B: + pass +``` + +`aliases.by`: + +```by +from kinds import A + +Alias = A +``` + +`main.by`: + +```by +from aliases import Alias +from kinds import A, B + +def size(xs: list[Alias] | list[B]) -> int: + return len(xs) +``` + ## explicit re-exports are not reported `import a as a`, `from a import b as b` and basedpython's `from a export b` say the name is part of diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index e22f1ff720..887340a4d1 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -213,6 +213,7 @@ mod known_instance; mod lifetimes; pub mod list_members; mod literal; +mod lowering_reads; mod match_pattern; pub(crate) mod match_type; mod member; diff --git a/crates/ty_python_semantic/src/types/lowering_reads.rs b/crates/ty_python_semantic/src/types/lowering_reads.rs new file mode 100644 index 0000000000..69a38d68b8 --- /dev/null +++ b/crates/ty_python_semantic/src/types/lowering_reads.rs @@ -0,0 +1,91 @@ +//! basedpython: the module-level names the transpiled program reads through what the +//! checker spells for it +//! +//! a lowering writes some types out as the checker spells them — a soundness check's target, +//! a reified call's or construction's specialization, the arms of a union parameter given a +//! reified type parameter — and a spelling names each class by the binding the module's +//! global scope reaches it through. the source spells none of those reads, so the import +//! binding a class read only there would look unused. these are the same queries the +//! lowerings ask, asked once the file is checked: each needs the inference of the scope the +//! node is in to be finished +//! +//! the transpiler asks them of the source after its pre-passes (an erased union given a type +//! parameter, a context-sensitive name qualified, an enum lowered), and these ask them of the +//! source as written, which is the one the checker has. a site only the rewritten source has +//! is not seen here + +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{self as ast, Expr, Stmt}; + +use crate::semantic_model::SemanticModel; +use crate::types::runtime_spelling::RuntimeSpelling; +use crate::types::soundness_sites::{SoundnessPositions, soundness_sites}; + +/// every module-level name a spelling the transpiled program writes for `suite` reads. every +/// soundness position counts, because which ones a build enables is the build's choice and +/// the answer has to hold for any build of the file +pub(crate) fn spelled_module_reads(model: &SemanticModel<'_>, suite: &[Stmt]) -> Vec { + let mut spellings = Spellings { + model, + reads: Vec::new(), + }; + for site in soundness_sites(model, suite, SoundnessPositions::all()) { + for plan in site.plans() { + spellings.read(&plan.spelling()); + } + } + spellings.visit_body(suite); + spellings.reads +} + +struct Spellings<'a, 'db> { + model: &'a SemanticModel<'db>, + reads: Vec, +} + +impl Spellings<'_, '_> { + fn read(&mut self, spelling: &RuntimeSpelling) { + self.reads.extend(spelling.global_reads().cloned()); + } +} + +impl<'ast> Visitor<'ast> for Spellings<'_, '_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + if let Stmt::FunctionDef(function) = stmt { + for parameter in &function.parameters { + let Some(annotation) = parameter.annotation() else { + continue; + }; + if let Some(union) = self.model.erased_union(annotation) { + self.read(&union.origin); + for arm in &union.arms { + self.read(arm); + } + for (_, fixed) in &union.fixed { + self.read(fixed); + } + } + } + } + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Call(call) = expr { + self.call(call); + } + walk_expr(self, expr); + } +} + +impl Spellings<'_, '_> { + fn call(&mut self, call: &ast::ExprCall) { + if let Some(specialization) = self.model.reified_call_specialization(call) { + self.read(&specialization); + } + if let Some(arguments) = self.model.reified_constructor_type_arguments(call) { + self.read(&arguments); + } + } +} diff --git a/crates/ty_python_semantic/src/types/soundness_sites.rs b/crates/ty_python_semantic/src/types/soundness_sites.rs index 20cfaabd8c..2348b99000 100644 --- a/crates/ty_python_semantic/src/types/soundness_sites.rs +++ b/crates/ty_python_semantic/src/types/soundness_sites.rs @@ -143,6 +143,16 @@ pub enum SoundnessSite<'ast> { }, } +impl SoundnessSite<'_> { + /// every check the site makes + pub(crate) fn plans(&self) -> Vec<&CheckKind> { + match self { + Self::Value { plan, .. } | Self::Elements { plan, .. } => vec![plan], + Self::Parameters { guards, .. } => guards.iter().map(|(_, plan)| plan).collect(), + } + } +} + /// where the checks go, decided once over a syntax tree, and what is not a place /// for one struct Soundness<'a, 'db, 'ast> { diff --git a/crates/ty_python_semantic/src/types/unused_imports.rs b/crates/ty_python_semantic/src/types/unused_imports.rs index 10a5c0bb16..ffc1b7f956 100644 --- a/crates/ty_python_semantic/src/types/unused_imports.rs +++ b/crates/ty_python_semantic/src/types/unused_imports.rs @@ -50,6 +50,7 @@ use crate::types::context::InferContext; use crate::types::definition_resolution::user_visible_definitions; use crate::types::diagnostic::{TypeCheckDiagnostics, UNUSED_IMPORT}; use crate::types::extensions::{extension_granting_imports, extensions_in_module}; +use crate::types::lowering_reads::spelled_module_reads; use crate::types::{ProgramEnvironment, Type}; use crate::{Db, FxIndexMap}; @@ -222,6 +223,9 @@ impl<'db> Uses<'db> { .iter() .map(|extension| extension.name(db).clone()), ); + // a type the transpiled program writes out names its classes as the module binds them + let model = crate::SemanticModel::new(db, file); + module_names.extend(spelled_module_reads(&model, parsed.suite())); Self { definitions, From ac4927c51455aa2efce31cce73301f979a53b14b Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:20:33 +1000 Subject: [PATCH 27/47] run every checker-clean mdtest block again with its unused imports removed by `by check --fix`, and fail when it then checks or behaves differently, every soundness check on --- crates/ty/tests/mdtest_divergence.rs | 130 +++++++++++++++++- .../mdtest/basedpython_unused_import.md | 36 +++++ 2 files changed, 165 insertions(+), 1 deletion(-) diff --git a/crates/ty/tests/mdtest_divergence.rs b/crates/ty/tests/mdtest_divergence.rs index c824375074..3135088749 100644 --- a/crates/ty/tests/mdtest_divergence.rs +++ b/crates/ty/tests/mdtest_divergence.rs @@ -9,6 +9,15 @@ //! //! blocks carrying expected diagnostics are skipped — their runtime behaviour //! is intentionally unspecified. +//! +//! a second leg holds `unused-import` to the same contract: each such block has its unused +//! imports removed by `by check --fix`, and must then check the same and do the same at +//! runtime — every soundness check on, so a check that reads an import is run. a removed +//! import whose binding the transpiled program still reads, because the lowering spelled a +//! name the checker never recorded a read of, shows up as a program that behaves +//! differently, or no longer checks. it sees only what this corpus exercises: an import read +//! only by a lowering, in a checker-clean block of a single file. a multi-file section is not +//! run, so a read across modules is out of its reach, as is any lowering no such block makes use std::fs; use std::io::Write; @@ -147,13 +156,17 @@ fn transpile(source: &str, min_version: &str) -> Result { // full of stub bodies (`def f[T]() -> T: ...`) that legitimately violate // their annotations when executed — the checks would report those as // divergences rather than transpiler bugs, so this harness opts out + transpile_with(source, min_version, "none") +} + +fn transpile_with(source: &str, min_version: &str, soundness: &str) -> Result { let mut child = Command::new(env!("CARGO_BIN_EXE_by")) .args([ "transpile", "--min-version", min_version, "--soundness", - "none", + soundness, ]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -174,6 +187,112 @@ fn transpile(source: &str, min_version: &str) -> Result { } } +/// the error-level diagnostics `by check` reports for the module `dir/main.by`, without their +/// positions, which removing an import shifts +fn check_errors(dir: &Path) -> Vec { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args([ + "check", + "--output-format", + "concise", + "--exit-zero", + "--no-progress", + ]) + .arg("main.by") + .current_dir(dir) + .output() + .expect("failed to spawn by"); + let mut errors: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .filter(|line| line.contains(": error[")) + .map(|line| line.splitn(4, ':').nth(3).unwrap_or(line).trim().to_owned()) + .collect(); + errors.sort(); + errors +} + +/// what the module `source` does when it is transpiled with every soundness check on and +/// run as `dir/main.py`: whether it transpiles, its exit status and its output, with `dir` +/// left out so two directories compare +fn behaviour(source: &str, min_version: &str, python: &str, dir: &Path) -> String { + let transpiled = match transpile_with(source, min_version, "all") { + Ok(transpiled) => transpiled, + Err(error) => return format!("transpile failed:\n{error}"), + }; + let py = dir.join("main.py"); + fs::write(&py, with_reveal_stub(&transpiled)).unwrap(); + let run = Command::new(python) + .arg(&py) + .current_dir(dir) + .output() + .expect("failed to run python"); + let text = format!( + "status {:?}\n{}\n{}", + run.status.code(), + String::from_utf8_lossy(&run.stdout), + String::from_utf8_lossy(&run.stderr), + ); + text.replace(&dir.to_string_lossy().into_owned(), "") +} + +/// the second leg: remove the block's unused imports with `by check --fix`, and require the +/// result to check and behave as the block did. `None` when it does, or when there was +/// nothing to remove +fn unused_import_leg( + block: &str, + min_version: &str, + python: &str, + root: &Path, + key: &str, +) -> Option { + let project = format!( + "[project]\nname = \"leg\"\nversion = \"0\"\nrequires-python = \">={min_version}\"\n" + ); + let original = root.join(format!("{key}_written")); + let fixed = root.join(format!("{key}_fixed")); + for dir in [&original, &fixed] { + fs::create_dir_all(dir).unwrap(); + fs::write(dir.join("pyproject.toml"), &project).unwrap(); + fs::write(dir.join("main.by"), block).unwrap(); + } + Command::new(env!("CARGO_BIN_EXE_by")) + .args([ + "check", + "--ignore", + "all", + "--warn", + "unused-import", + "--fix", + "--exit-zero", + "--no-progress", + ]) + .arg("main.by") + .current_dir(&fixed) + .output() + .expect("failed to spawn by"); + let without = fs::read_to_string(fixed.join("main.by")).unwrap(); + if without == block { + return None; + } + let checked = (check_errors(&original), check_errors(&fixed)); + if checked.0 != checked.1 { + return Some(format!( + "removing its unused imports changed what the checker reports:\nbefore: {:?}\nafter: {:?}\n--- after ---\n{without}", + checked.0, checked.1 + )); + } + let behaved = ( + behaviour(block, min_version, python, &original), + behaviour(&without, min_version, python, &fixed), + ); + (behaved.0 != behaved.1).then(|| { + format!( + "removing its unused imports changed what the program does:\n--- before ---\n{}\n--- after ---\n{}\n--- without the imports ---\n{without}", + behaved.0, behaved.1 + ) + }) +} + /// Stub `reveal_type` (an mdtest debugging device with no runtime binding) /// after the `__future__` import, which must stay first. fn with_reveal_stub(transpiled: &str) -> String { @@ -406,6 +525,15 @@ fn clean_mdtest_blocks_run() { "{name} block {i}: checker-clean block crashed at runtime:\n{}\n--- block ---\n{block}", String::from_utf8_lossy(&run.stderr) )); + continue; + } + let key = format!("{}_{i}", name.trim_end_matches(".md").replace('-', "_")); + if let Some(divergence) = + unused_import_leg(block, &version, &python, tmp.path(), &key) + { + failures.lock().unwrap().push(format!( + "{name} block {i}: {divergence}\n--- block ---\n{block}" + )); } } }); diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md index 9e174a041b..fbab84eee6 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_unused_import.md @@ -854,6 +854,42 @@ def size(xs: list[Alias] | list[B]) -> int: return len(xs) ``` +## an import read only by a type test's spelling of a class + +`Number` names `Fraction` as an attribute of `fractions`, but the test checks each arm against the +class as the module binds it, `isinstance(value, Fraction)`, which reads the `from` import + +```by +import fractions +from fractions import Fraction + +type Number = fractions.Fraction | int + +def numeric(value: object) -> bool: + return value is Number + +print(numeric(fractions.Fraction(1, 2)), numeric("x")) +``` + +## an import read only by a soundness check's spelling of a class + +the check on `pick`'s result reads `Fraction` the same way. a build that runs it rejects the `1` +smuggled into the list as `Any`; without the import there is no check to reject it + +```by +import fractions +from fractions import Fraction +from typing import Any + +def pick[T](items: list[T]) -> T: + return items[0] + +smuggled: Any = 1 +numbers: list[fractions.Fraction] = [smuggled] +first = pick(numbers) +print(type(first).__name__) +``` + ## explicit re-exports are not reported `import a as a`, `from a import b as b` and basedpython's `from a export b` say the name is part of From 93f311ca6555ba079365be35a5c22383871b34d6 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:17:05 +1000 Subject: [PATCH 28/47] =?UTF-8?q?construct=20a=20plain=20tuple=20literal?= =?UTF-8?q?=20as=20the=20anonymous=20named=20tuple=20the=20checker=20reads?= =?UTF-8?q?=20it=20as,=20wherever=20one=20is=20expected=20=E2=80=94=20an?= =?UTF-8?q?=20argument,=20a=20keyword=20argument,=20an=20element=20of=20`*?= =?UTF-8?q?args`,=20a=20default,=20a=20`yield`,=20a=20method=20call=20such?= =?UTF-8?q?=20as=20`items.append((1,=202))`,=20a=20collection=20literal=20?= =?UTF-8?q?at=20any=20depth,=20a=20write=20to=20an=20attribute,=20an=20`in?= =?UTF-8?q?it(let=20p:=20...)`=20parameter=20=E2=80=94=20instead=20of=20on?= =?UTF-8?q?ly=20under=20an=20annotated=20assignment,=20a=20`return`=20or?= =?UTF-8?q?=20a=20list=20or=20set=20literal,=20where=20everything=20else?= =?UTF-8?q?=20stayed=20a=20plain=20tuple=20and=20field=20access=20raised?= =?UTF-8?q?=20`AttributeError`=20with=20`by=20check`=20clean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the transpiler now finds the literals by their inferred type rather than by syntactic hooks, and builds the class the checker identified: the one a type form of the same type declares, or one declared from the checker's field types for a class from another module, refusing a field type the module cannot spell. in a `.by` file an attribute write records its value as it was checked against the declaration; a write through a union receiver is still checked against each member's own declaration, and is rejected only where the members would build different anonymous named tuples from one literal. an anonymous named tuple is built only where its fields accept the literal, which is otherwise inferred against the rest of the expected type; a literal more than one accepts is `ambiguous-conversion` instead of becoming the first; a `.py` file no longer reads a plain tuple as a named tuple --- crates/by_transforms/src/lib.rs | 1 + .../src/transforms/anon_named_tuple.rs | 523 ++++++------------ crates/by_transforms/src/type_info.rs | 95 +++- .../tests/anon_named_tuple_runtime.rs | 281 +++++++++- .../tests/attribute_write_runtime.rs | 59 ++ crates/ty/docs/rules.md | 209 +++---- ...edpython_anonymous_named_tuple_literals.md | 303 ++++++++++ crates/ty_python_semantic/src/lib.rs | 7 +- .../ty_python_semantic/src/semantic_model.rs | 12 + crates/ty_python_semantic/src/types.rs | 2 + .../src/types/anon_named_tuple.rs | 131 +++++ .../src/types/attribute_write.rs | 107 ++++ .../ty_python_semantic/src/types/call/bind.rs | 18 + .../src/types/diagnostic.rs | 9 +- .../src/types/infer/builder.rs | 261 +++++---- .../infer/builder/attribute_assignment.rs | 129 ++++- .../features/reified-class-generics.md | 6 + ty.schema.json | 2 +- 18 files changed, 1582 insertions(+), 573 deletions(-) create mode 100644 crates/by_transforms/tests/attribute_write_runtime.rs create mode 100644 crates/ty_python_semantic/resources/mdtest/basedpython_anonymous_named_tuple_literals.md create mode 100644 crates/ty_python_semantic/src/types/anon_named_tuple.rs diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 8c0bf6e6ba..32b2e01cea 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -638,6 +638,7 @@ fn run_anon_named_tuple_cleanup( for stmt in module.suite() { anon.visit_stmt(stmt); } + anon.finish(); if let Some(err) = anon.errors.first() { return Err(err.clone()); } diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index 7320570491..3ecb1eaaca 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -6,6 +6,9 @@ //! across the module collapse to one class so structural equivalence is //! preserved at the type level. //! +//! a plain tuple literal the checker reads as an instance of an anonymous named +//! tuple — one standing wherever one is expected — is constructed as its class +//! //! Example: //! ```by //! def foo(x: (name: str, age: int)) -> (name: str, age: int): @@ -23,7 +26,7 @@ //! age: int //! //! def foo(x: _AnonNamedTuple_) -> _AnonNamedTuple_: -//! return ("asdf", 1) +//! return _AnonNamedTuple_("asdf", 1) //! //! a = _AnonNamedTuple_ //! ``` @@ -34,13 +37,14 @@ use std::hash::Hash; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt, StmtAnnAssign, StmtFunctionDef, StmtReturn}; +use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::Ranged; +use ty_python_semantic::AnonNamedTupleId; use crate::transforms::callable::CallableSyntax; use crate::transforms::generics::PolyfilledTypeParams; use crate::transforms::repeated_underscore::WrittenNames; -use crate::type_info::TypeInfo; +use crate::type_info::{AnonNamedTupleConstruction, AnonNamedTupleField, TypeInfo}; /// Synthetic field name for the i-th positional element of a mixed /// anonymous named tuple. `NamedTuple` disallows leading-underscore field @@ -115,7 +119,7 @@ pub(crate) struct AnonNamedTuple<'src> { /// emitted `NamedTuple` definition is valid Python. /// /// Stored as a `Vec` (`TextRange` isn't `Ord`) and walked linearly when - /// rendering. Inserts append; `render_subbed` skips entries that don't + /// rendering. Inserts append; `render_subbed_value` skips entries that don't /// fall within the requested range. range_to_class: Vec<(ruff_text_size::TextRange, String)>, /// Source range of every anonymous-named-tuple *value* form (`(name=v, @@ -136,10 +140,6 @@ pub(crate) struct AnonNamedTuple<'src> { typing_imports: std::cell::RefCell>, /// whether python evaluates the synthesized class's field annotations as it is made annotations_evaluated: bool, - /// Active function-scope return-annotation shape stack. Empty when not - /// inside a function. The innermost (last) entry governs how a `return` - /// statement inside the current function is coerced. - return_shape_stack: Vec>, /// Stack of typevar rename maps (`T` → `_T`) accumulated as the visitor /// descends into PEP 695 generics. Synthesized `NamedTuple` class bodies /// are hoisted to module scope, but the original source text references @@ -152,12 +152,15 @@ pub(crate) struct AnonNamedTuple<'src> { typevar_rename_stack: Vec>, /// the name the generics polyfill declares each type parameter under polyfilled: PolyfilledTypeParams, - /// Names bound to an anonymous named tuple by an alias — `P = (a: int)` or - /// `type P = (a: int)`. Coercion asks what the annotation *means*, not how - /// it is spelled, so a plain tuple literal under `x: P` is wrapped exactly - /// as one under `x: (a: int)` is. Populated in source order, so an alias - /// only applies below its own declaration. - aliases: HashMap, + /// the class emitted for each anonymous named tuple the checker identifies, so + /// that every type form it reads as one type, and every plain tuple literal it + /// reads as an instance of it, names one class + identity_classes: HashMap, + /// the plain tuple literals the checker reads as an anonymous named tuple, in + /// source order with an enclosing literal ahead of the ones inside it. each is + /// built as its class once the whole module has been visited, because the + /// type form that names the class may come after the literal + constructions: Vec, /// Hard transpile errors — each one aborts the whole transpilation so we /// never emit syntactically invalid Python. The transform aborts on the /// *first* failure during the visit pass; subsequent visits are no-ops. @@ -187,9 +190,9 @@ impl<'src> AnonNamedTuple<'src> { type_only_imports: std::cell::RefCell::default(), typing_imports: std::cell::RefCell::default(), annotations_evaluated: false, - return_shape_stack: Vec::new(), typevar_rename_stack: Vec::new(), - aliases: HashMap::new(), + identity_classes: HashMap::new(), + constructions: Vec::new(), errors: Vec::new(), } } @@ -213,26 +216,6 @@ impl<'src> AnonNamedTuple<'src> { &self.source[usize::from(range.start())..usize::from(range.end())] } - /// Render `range`'s source, substituting any anonymous-named-tuple - /// expression contained within `range` with its synthesized class name. - /// Used wherever a stretch of original source ends up in the transpiled - /// output (class field type, constructor argument, coerced literal) so - /// nested anon-NT regions don't leak through verbatim. - fn render_subbed(&self, range: ruff_text_size::TextRange) -> String { - let raw = self.render_subbed_with(range, &self.range_to_class); - let renamed = self.apply_typevar_renames(&raw); - // Synthesized NamedTuple classes are hoisted to module scope, but - // the renamed typevars (`_T`) get declared *after* them by the - // generics polyfill. Wrap typevar references in PEP 484 forward-ref - // quotes so the annotation is stored as a string and only resolved - // by type checkers, not at class-construction time. - if renamed != raw && !raw.is_empty() { - format!("\"{renamed}\"") - } else { - renamed - } - } - /// Lower a field's type expression to python source. /// /// The synthesized class body is hoisted out of the statement the field type @@ -243,8 +226,8 @@ impl<'src> AnonNamedTuple<'src> { fn render_field_type(&mut self, expr: &Expr) -> String { let lowered = self.callable.lower_type_expr(expr); let renamed = self.apply_typevar_renames(&lowered); - // same forward-reference quoting as `render_subbed`: a renamed typevar is - // declared below the hoisted class, so the annotation must stay a string + // a renamed typevar is declared below the hoisted class, so the annotation + // must stay a string if renamed != lowered && !lowered.is_empty() { format!("\"{renamed}\"") } else { @@ -277,10 +260,11 @@ impl<'src> AnonNamedTuple<'src> { super::source_util::rename_identifiers(text, &borrowed) } - /// Like `render_subbed` but substitutes value-form constructor calls - /// instead of bare class names. Used when an outer anon-NT *value* form - /// is rendering its argument list — a nested value form there must call - /// its inner constructor, not just reference the inner class. + /// Render `range`'s source with every anonymous named tuple inside it + /// substituted: a value form by its constructor call, a type form by its + /// class name. Used when an outer anon-NT *value* form is rendering its + /// argument list — a nested value form there must call its inner + /// constructor, not just reference the inner class. fn render_subbed_value(&self, range: ruff_text_size::TextRange) -> String { // Combine value-form constructor calls (preferred) with class-name // subs for any nested *type* form in the same range. Value renders @@ -301,20 +285,6 @@ impl<'src> AnonNamedTuple<'src> { self.sweep(range, subs) } - fn render_subbed_with( - &self, - range: ruff_text_size::TextRange, - table: &[(ruff_text_size::TextRange, String)], - ) -> String { - let mut subs: Vec<(ruff_text_size::TextRange, &str)> = table - .iter() - .filter(|(r, _)| r.start() >= range.start() && r.end() <= range.end()) - .map(|(r, n)| (*r, n.as_str())) - .collect(); - subs.sort_by_key(|(r, _)| r.start()); - self.sweep(range, subs) - } - fn sweep( &self, range: ruff_text_size::TextRange, @@ -463,13 +433,20 @@ impl<'src> AnonNamedTuple<'src> { Ok(Some(Shape { fields })) } - fn class_name_for(&mut self, shape: Shape, source_range: ruff_text_size::TextRange) -> String { - let name = if let Some(existing) = self.shapes.get(&shape) { + fn class_name_for( + &mut self, + shape: Shape, + source_range: ruff_text_size::TextRange, + id: Option, + ) -> String { + let name = if let Some(existing) = id.and_then(|id| self.identity_classes.get(&id)) { existing.clone() } else { - let n = shape.class_name(self.written); - self.shapes.insert(shape, n.clone()); - n + let name = self.shape_class_name(shape); + if let Some(id) = id { + self.identity_classes.insert(id, name.clone()); + } + name }; // Track this anon-NT's source range → class name so any *outer* anon-NT // that contains this one can substitute its source span when rendering @@ -490,7 +467,8 @@ impl<'src> AnonNamedTuple<'src> { return; } }; - let class_name = self.class_name_for(shape, tuple.range()); + let id = self.types.anon_named_tuple_id(tuple); + let class_name = self.class_name_for(shape, tuple.range(), id); self.needs_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( class_name, @@ -508,7 +486,7 @@ impl<'src> AnonNamedTuple<'src> { } }; // Render constructor call: `ClassName(v1, v2, ...)` using positional - // args sourced from each field's value expression. We `render_subbed` + // args sourced from each field's value expression. We `render_subbed_value` // so a nested anon-NT inside any value gets substituted with its own // class-constructor call rather than emitted verbatim. let mut value_args: Vec = Vec::with_capacity(tuple.elts.len()); @@ -519,7 +497,9 @@ impl<'src> AnonNamedTuple<'src> { }; value_args.push(self.render_subbed_value(value_expr.range())); } - let class_name = self.class_name_for(shape, tuple.range()); + // a value form is left out of the checker's identities: its class is the one + // its own field values make, which a type form's need not be + let class_name = self.class_name_for(shape, tuple.range(), None); self.needs_import = true; let constructor_call = format!("{class_name}({})", value_args.join(", ")); self.range_to_value_render @@ -530,171 +510,6 @@ impl<'src> AnonNamedTuple<'src> { ))); } - /// Rewrite a `return ...` statement inside a function annotated with an - /// anonymous-named-tuple return type, when the returned value is a plain - /// tuple literal of matching arity. The plain tuple is wrapped as a - /// constructor call so runtime field access works. - fn rewrite_return_coercion(&mut self, ret: &StmtReturn) { - let Some(Some(shape)) = self.return_shape_stack.last().cloned() else { - return; - }; - let Some(value) = ret.value.as_ref() else { - return; - }; - if let Expr::Tuple(tuple) = value.as_ref() { - self.wrap_plain_tuple(tuple, shape); - } - } - - /// Walk an `x: T = ...` statement and coerce plain tuple literals on the - /// RHS when the annotation is an anonymous named tuple (directly or - /// inside a recognized container generic). - fn rewrite_ann_assign_coercion(&mut self, stmt: &StmtAnnAssign) { - let Some(value) = stmt.value.as_deref() else { - return; - }; - let Some(target) = self.coercion_target(&stmt.annotation) else { - return; - }; - self.apply_coercion(&target, value); - } - - fn return_shape_for(&mut self, func: &StmtFunctionDef) -> Option { - // Errors in shape extraction surface through the visitor's own pass - // over this annotation tuple — silently treat them as "no coercion" - // here so we don't double-report. - self.annotated_shape(func.returns.as_deref()?) - } - - /// Extract a coercion target from an annotation expression. Returns the - /// expected shape and the surrounding container so the caller knows - /// whether to wrap the value directly or to wrap each element of an - /// outer collection literal. - fn coercion_target(&mut self, annotation: &Expr) -> Option { - // a declaration modifier keeps the declared type in a marker subscript - // (`let x: T` parses as `x: __let__[T]`), and the annotation this reads - // is the one the source wrote - if let Some(declared) = ruff_python_ast::helpers::declaration_annotation_type(annotation) { - return self.coercion_target(declared); - } - // Direct: `x: (name: T, ...)` or `x: Alias` - if let Some(shape) = self.annotated_shape(annotation) { - return Some(CoercionTarget { - shape, - container: ContainerKind::Direct, - }); - } - // Containers: `x: list[(name: T, ...)]` and friends. The annotation - // is `Subscript(value=Name("list"), slice=Tuple{is_anon_named_tuple=true})`. - if let Expr::Subscript(sub) = annotation { - let Expr::Name(name) = sub.value.as_ref() else { - return None; - }; - let kind = match name.id.as_str() { - "list" | "List" => ContainerKind::ListLike, - "set" | "Set" | "frozenset" | "FrozenSet" => ContainerKind::SetLike, - _ => return None, - }; - let shape = self.annotated_shape(sub.slice.as_ref())?; - return Some(CoercionTarget { - shape, - container: kind, - }); - } - None - } - - /// The anonymous-named-tuple shape an annotation denotes — written out, or - /// reached through an alias declared above it. - fn annotated_shape(&mut self, annotation: &Expr) -> Option { - match annotation { - Expr::Tuple(t) => self.extract_type_shape(t).ok().flatten(), - Expr::Name(name) => self.aliases.get(name.id.as_str()).cloned(), - _ => None, - } - } - - /// Record `P = (name: T, ...)` / `type P = (name: T, ...)` so a later - /// annotation spelling `P` coerces the same way the written-out form does. - fn record_alias(&mut self, name: &Expr, value: &Expr) { - let (Expr::Name(target), Expr::Tuple(t)) = (name, value) else { - return; - }; - if !t.is_anon_named_tuple { - return; - } - // the tuple has already been visited, so a shape error is reported there - if let Ok(Some(shape)) = self.extract_type_shape(t) { - self.aliases.insert(target.id.to_string(), shape); - } - } - - /// Wrap a plain tuple literal `(v1, v2, ...)` as a constructor call to the - /// synthesized `NamedTuple` class for `shape`. No-op if the tuple is - /// already a basedpython anon form, has wrong arity, or contains starred - /// elements. - fn wrap_plain_tuple(&mut self, tuple: &ruff_python_ast::ExprTuple, shape: Shape) { - if tuple.is_anon_named_tuple || tuple.is_anon_named_tuple_value { - return; - } - if tuple.elts.len() != shape.fields.len() { - return; - } - if tuple.elts.iter().any(Expr::is_starred_expr) { - return; - } - // Build the constructor call. Each argument is rendered against the - // shape's corresponding field type so that a plain tuple element - // whose target field is itself a synthesized `NamedTuple` class is - // recursively wrapped — otherwise `record.point.x` would fail at - // runtime because the inner plain tuple doesn't carry field names. - self.needs_import = true; - let args: Vec = tuple - .elts - .iter() - .zip(shape.fields.iter()) - .map(|(elt, (_, field_type))| self.render_arg_against_field_type(elt, field_type)) - .collect(); - let name = if let Some(existing) = self.shapes.get(&shape) { - existing.clone() - } else { - let n = shape.class_name(self.written); - self.shapes.insert(shape, n.clone()); - n - }; - let constructor_call = format!("{name}({})", args.join(", ")); - self.edits.push(Fix::safe_edit(Edit::range_replacement( - constructor_call, - tuple.range(), - ))); - } - - /// Render a constructor argument. If `field_type` is the name of a - /// synthesized `NamedTuple` class and the argument is a plain tuple - /// matching that shape, recursively wrap it so nested field-name access - /// (`outer.inner.x`) keeps working at runtime. Otherwise fall back to - /// `render_subbed` which substitutes any inner anon-NT spans. - fn render_arg_against_field_type(&self, elt: &Expr, field_type: &str) -> String { - if let Some(inner_shape) = self.shape_for_class_name(field_type) { - if let Expr::Tuple(t) = elt { - if !t.is_anon_named_tuple - && !t.is_anon_named_tuple_value - && t.elts.len() == inner_shape.fields.len() - && !t.elts.iter().any(Expr::is_starred_expr) - { - let inner_args: Vec = t - .elts - .iter() - .zip(inner_shape.fields.iter()) - .map(|(e, (_, ft))| self.render_arg_against_field_type(e, ft)) - .collect(); - return format!("{}({})", field_type, inner_args.join(", ")); - } - } - } - self.render_subbed(elt.range()) - } - /// Push the typevar rename frame the generics polyfill gives a `[T, U, ...]` /// type-parameter list. Returns whether a frame was pushed so the caller /// knows to pop. No-op (returns false) for a list kept as native syntax @@ -720,137 +535,103 @@ impl<'src> AnonNamedTuple<'src> { .map(|(_, n)| n.clone()) } - fn shape_for_class_name(&self, name: &str) -> Option<&Shape> { - if !name.starts_with("_AnonNamedTuple_") { - return None; + /// the name of the class declaring `shape`, one class per shape + fn shape_class_name(&mut self, shape: Shape) -> String { + if let Some(existing) = self.shapes.get(&shape) { + return existing.clone(); } - self.shapes - .iter() - .find_map(|(shape, n)| (n == name).then_some(shape)) + let name = shape.class_name(self.written); + self.shapes.insert(shape, name.clone()); + name } - /// Apply the coercion target to the right-hand side of an assignment. - fn apply_coercion(&mut self, target: &CoercionTarget, value: &Expr) { - match target.container { - ContainerKind::Direct => { - if let Expr::Tuple(tuple) = value { - self.wrap_plain_tuple(tuple, target.shape.clone()); - } - } - ContainerKind::ListLike => { - let elts = match value { - Expr::List(list) => &list.elts[..], - // `[a, b]` is the common case; a parenthesized - // generator/expr would not match `list[X]` typewise. - _ => return, - }; - for elt in elts { - if let Expr::Tuple(tuple) = elt { - self.wrap_plain_tuple(tuple, target.shape.clone()); - } - } - } - ContainerKind::SetLike => { - let elts = match value { - Expr::Set(set) => &set.elts[..], - _ => return, - }; - for elt in elts { - if let Expr::Tuple(tuple) = elt { - self.wrap_plain_tuple(tuple, target.shape.clone()); - } + /// build each plain tuple literal the checker reads as an anonymous named tuple + /// as that class. called once the module has been visited + pub(crate) fn finish(&mut self) { + for construction in std::mem::take(&mut self.constructions) { + let class_name = match self.construction_class(&construction.target) { + Ok(name) => name, + Err(field) => { + self.errors.push(format!( + "the tuple at offset {} is built as an anonymous named tuple whose \ + field `{field}` has no type the emitted python can name; construct \ + it explicitly, or give it a `NamedTuple` class of its own", + u32::from(construction.range.start()) + )); + continue; } + }; + self.needs_import = true; + // a parenthesized literal is already the argument list of the call + if construction.parenthesized { + self.edits.push(Fix::safe_edit(Edit::insertion( + class_name, + construction.range.start(), + ))); + } else { + self.edits.push(Fix::safe_edit(Edit::insertion( + format!("{class_name}("), + construction.range.start(), + ))); + self.edits.push(Fix::safe_edit(Edit::insertion( + ")".to_owned(), + construction.range.end(), + ))); } } } -} -/// What surrounding shape a plain tuple literal at this position should be -/// coerced into. -#[derive(Clone, Debug)] -enum ContainerKind { - /// Annotation is the anon-NT directly: `x: (name: T, ...)`. - Direct, - /// Annotation wraps in a list-like generic: `list[X]`, `List[X]`. - ListLike, - /// Annotation wraps in a set-like generic: `set[X]`, `frozenset[X]`, - /// `Set[X]`, `FrozenSet[X]`. - SetLike, + /// the class a plain tuple literal is built as: the one emitted for a type form + /// the checker reads as the same type, or else one declared from the checker's + /// own field types. `Err` names a field whose type the output cannot spell + fn construction_class( + &mut self, + target: &AnonNamedTupleConstruction, + ) -> Result { + if let Some(existing) = self.identity_classes.get(&target.id) { + return Ok(existing.clone()); + } + let mut fields = Vec::with_capacity(target.fields.len()); + for (name, field) in &target.fields { + let field_type = match field { + AnonNamedTupleField::Spelled(spelled) => { + self.type_only_imports + .borrow_mut() + .extend(spelled.modules.iter().cloned()); + self.typing_imports + .borrow_mut() + .extend(spelled.typing_imports_in(self.written)); + spelled.annotation_in(self.written, self.annotations_evaluated) + } + AnonNamedTupleField::NamedTuple(nested) => self.construction_class(nested)?, + AnonNamedTupleField::Unspellable => return Err(name.clone()), + }; + fields.push((name.clone(), field_type)); + } + let class_name = self.shape_class_name(Shape { fields }); + self.identity_classes.insert(target.id, class_name.clone()); + Ok(class_name) + } } -#[derive(Clone, Debug)] -struct CoercionTarget { - shape: Shape, - container: ContainerKind, +/// a plain tuple literal to build as an anonymous named tuple +struct PendingConstruction { + range: ruff_text_size::TextRange, + parenthesized: bool, + target: AnonNamedTupleConstruction, } impl<'ast> Visitor<'ast> for AnonNamedTuple<'_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { - match stmt { - Stmt::FunctionDef(func) => { - // Visit non-body parts first so their anonymous-named-tuple - // expressions register synthesized classes before any body - // `return` statement is coerced. Walking via `walk_stmt` - // would visit the body before we've pushed the return shape - // onto the stack, so we manually drive the visit order. - let pushed = self.push_typevar_scope_from(func.type_params.as_deref()); - if let Some(tp) = func.type_params.as_deref() { - self.visit_type_params(tp); - } - self.visit_parameters(&func.parameters); - if let Some(returns) = func.returns.as_deref() { - self.visit_expr(returns); - } - let shape = self.return_shape_for(func); - self.return_shape_stack.push(shape); - for s in &func.body { - self.visit_stmt(s); - } - self.return_shape_stack.pop(); - for d in &func.decorator_list { - self.visit_decorator(d); - } - if pushed { - self.typevar_rename_stack.pop(); - } - } - Stmt::ClassDef(cls) => { - let pushed = self.push_typevar_scope_from(cls.type_params.as_deref()); - ruff_python_ast::visitor::walk_stmt(self, stmt); - if pushed { - self.typevar_rename_stack.pop(); - } - } - Stmt::Return(ret) => { - if let Some(v) = ret.value.as_deref() { - self.visit_expr(v); - } - self.rewrite_return_coercion(ret); - } - Stmt::AnnAssign(ann) => { - // Visit annotation first so any anon-NT inside it (including - // nested ones) registers its synthesized class. Then visit - // the RHS — any anon-NT children there register too. THEN - // coerce: the shape extracted from the annotation now uses - // the fully populated `range_to_class` so emitted class - // bodies don't embed raw inner anon-NT source. - self.visit_expr(&ann.annotation); - if let Some(v) = ann.value.as_deref() { - self.visit_expr(v); - } - self.rewrite_ann_assign_coercion(ann); - } - Stmt::Assign(assign) => { - walk_stmt(self, stmt); - if let [target] = assign.targets.as_slice() { - self.record_alias(target, &assign.value); - } - } - Stmt::TypeAlias(alias) => { - walk_stmt(self, stmt); - self.record_alias(&alias.name, &alias.value); - } - _ => walk_stmt(self, stmt), + let type_params = match stmt { + Stmt::FunctionDef(func) => func.type_params.as_deref(), + Stmt::ClassDef(cls) => cls.type_params.as_deref(), + _ => None, + }; + let pushed = self.push_typevar_scope_from(type_params); + walk_stmt(self, stmt); + if pushed { + self.typevar_rename_stack.pop(); } } @@ -868,6 +649,16 @@ impl<'ast> Visitor<'ast> for AnonNamedTuple<'_> { } return; } + if let Some(target) = self + .types + .anon_named_tuple_construction(tuple, self.config.min_version) + { + self.constructions.push(PendingConstruction { + range: tuple.range(), + parenthesized: tuple.parenthesized, + target, + }); + } } walk_expr(self, expr); } @@ -924,6 +715,7 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { for stmt in stmts { inner.visit_stmt(stmt); } + inner.finish(); for err in std::mem::take(&mut inner.errors) { ctx.errors.push(err); } @@ -1133,10 +925,11 @@ mod tests { } #[test] - fn an_alias_below_its_use_does_not_coerce() { - // the table is built in source order, so a name is only an anon-NT alias - // from its declaration down — the same rule the binding itself follows. - // the annotation runs before `P` is bound, so it is quoted + fn an_alias_below_its_use_still_builds_the_class() { + // a `.by` annotation may name what is bound below it, and the checker + // reads `P` there as the alias. the class is declared above every + // statement, so building it there is fine; the annotation runs before `P` + // is bound, so it is quoted check( indoc! {" v: P = (\"a\", 1) @@ -1148,7 +941,7 @@ mod tests { name: str age: int - v: \"P\" = (\"a\", 1) + v: \"P\" = _AnonNamedTuple_7bfb4772(\"a\", 1) P = _AnonNamedTuple_7bfb4772 "}, ); @@ -1682,9 +1475,7 @@ mod tests { } #[test] - fn plain_tuple_arg_unchanged() { - // Plain tuple literal at a call argument is not coerced — only return - // values inside anon-NT-returning functions are. + fn a_plain_tuple_argument_is_built_as_its_parameter() { let out = transpile( indoc! {" def foo(x: (name: str, age: int)) -> None: ... @@ -1693,7 +1484,29 @@ mod tests { &Config::test_default(), ) .unwrap(); - assert!(out.contains("foo((\"asdf\", 1))"), "got: {out}"); + assert!( + out.contains("foo(_AnonNamedTuple_7bfb4772(\"asdf\", 1))"), + "got: {out}" + ); + } + + #[test] + fn two_spellings_of_one_type_are_one_class() { + // the checker reads `(a: List[int])` and `(a: list[int])` as one type, so + // the literal it reads as that type is built as the one class both + // annotations name + let out = transpile( + indoc! {" + from typing import List + def take(p: (a: List[int])) -> None: ... + def give(q: (a: list[int])) -> None: ... + take(([1],)) + "}, + &Config::test_default(), + ) + .unwrap(); + assert_eq!(out.matches("(NamedTuple):").count(), 1, "got: {out}"); + assert!(out.contains("take(_AnonNamedTuple_"), "got: {out}"); } #[test] diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index f154e75fcd..16989db14f 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -5,7 +5,8 @@ use crate::transforms::trailing_lambda::RECEIVER_PARAMETER; use ruff_db::parsed::parsed_module; use ruff_python_ast::helpers::is_dotted_name; use ruff_python_ast::{ - Expr, ExprCall, ExprName, ExprRef, ParameterWithDefault, Stmt, StmtClassDef, StmtFunctionDef, + Expr, ExprCall, ExprName, ExprRef, ExprTuple, ParameterWithDefault, Stmt, StmtClassDef, + StmtFunctionDef, }; use ruff_python_parser::parse_expression; use ruff_text_size::{Ranged, TextRange}; @@ -20,7 +21,8 @@ use ty_python_semantic::types::{ DisplaySettings, DynamicType, KnownClass, KnownInstanceType, Type, UnpackedKwargs, character, }; use ty_python_semantic::{ - Db, HasType, ImplicitReceiverReference, ProgramEnvironment, SemanticModel, + AnonNamedTupleClass, AnonNamedTupleId, Db, HasType, ImplicitReceiverReference, + ProgramEnvironment, SemanticModel, }; /// How the postfix `^` / `!` operators test the "absent" arm of an operand's @@ -182,6 +184,29 @@ impl SynthesizedType { } } +/// the anonymous named tuple class the checker reads a plain tuple literal as, which the +/// emitted python constructs around it +#[derive(Clone, Debug)] +pub(crate) struct AnonNamedTupleConstruction { + /// the checker's identity of the class, shared with every type form it reads as the + /// same type + pub(crate) id: AnonNamedTupleId, + /// each field's name and type, for a module that has no type form of the class to + /// emit it from + pub(crate) fields: Vec<(String, AnonNamedTupleField)>, +} + +/// the type of one field of an [`AnonNamedTupleConstruction`] +#[derive(Clone, Debug)] +pub(crate) enum AnonNamedTupleField { + /// a type the emitted python can name + Spelled(SynthesizedType), + /// another anonymous named tuple, emitted as a class of its own + NamedTuple(AnonNamedTupleConstruction), + /// a type the emitted python cannot name + Unspellable, +} + /// How an assignment inside a trailing-lambda block reaches an enclosing scope, /// so the block writes through instead of shadowing with a fresh local. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -544,6 +569,19 @@ pub(crate) trait TypeInfo { /// rather than as a global fn shares_a_cell_scope(&self, reference: &ExprName, anchor: &Expr) -> bool; + /// the checker's identity of the anonymous named tuple class a type form + /// `(name: T, ...)` declares + fn anon_named_tuple_id(&self, tuple: &ExprTuple) -> Option; + + /// the anonymous named tuple a plain tuple literal is built as: the checker reads + /// one standing where an anonymous named tuple is expected as an instance of it. + /// `None` for a literal that stays a tuple + fn anon_named_tuple_construction( + &self, + tuple: &ExprTuple, + min_version: ruff_python_ast::PythonVersion, + ) -> Option; + /// rendered inferred (literal-promoted) type of `expr`, or `None` when ty /// cannot resolve a type (unresolved import, parse error, etc.) or the type /// it resolves to has no python type-expression spelling the emitted file @@ -1231,6 +1269,26 @@ impl TypeInfo for SemanticModel<'_> { }) } + fn anon_named_tuple_id(&self, tuple: &ExprTuple) -> Option { + self.anon_named_tuple_class(tuple).map(|class| class.id) + } + + fn anon_named_tuple_construction( + &self, + tuple: &ExprTuple, + min_version: ruff_python_ast::PythonVersion, + ) -> Option { + if tuple.is_anon_named_tuple + || tuple.is_anon_named_tuple_value + || tuple.has_parameter_shape() + || !tuple.ctx.is_load() + { + return None; + } + let class = self.anon_named_tuple_class(tuple)?; + Some(anon_named_tuple_construction(self, class, min_version)) + } + fn promoted_type_display( &self, expr: &Expr, @@ -1657,6 +1715,39 @@ fn display_for_python<'db>( .to_string() } +/// `class` with each field's type spelled for the emitted python +fn anon_named_tuple_construction<'db>( + model: &SemanticModel<'db>, + class: AnonNamedTupleClass<'db>, + min_version: ruff_python_ast::PythonVersion, +) -> AnonNamedTupleConstruction { + let db = model.db(); + let env = model.program_environment(); + let fields = class + .fields + .into_iter() + .map(|(name, ty)| { + let field = if let Some(nested) = AnonNamedTupleClass::of(db, &env, ty) { + AnonNamedTupleField::NamedTuple(anon_named_tuple_construction( + model, + nested, + min_version, + )) + } else { + spell_for_python(db, &env, model, ty, min_version, true, None).map_or( + AnonNamedTupleField::Unspellable, + AnonNamedTupleField::Spelled, + ) + }; + (name.to_string(), field) + }) + .collect(); + AnonNamedTupleConstruction { + id: class.id, + fields, + } +} + /// Spell `ty` as a type expression the *emitted* file can resolve, or `None` when it has /// no such spelling. /// diff --git a/crates/by_transforms/tests/anon_named_tuple_runtime.rs b/crates/by_transforms/tests/anon_named_tuple_runtime.rs index 126789e7de..e2a6e45226 100644 --- a/crates/by_transforms/tests/anon_named_tuple_runtime.rs +++ b/crates/by_transforms/tests/anon_named_tuple_runtime.rs @@ -9,9 +9,14 @@ mod common; +use std::fs; +use std::path::{Path, PathBuf}; use std::process::Command; -use by_transforms::{Config, PythonVersion, transpile}; +use by_transforms::{Config, PythonVersion, SoundnessPositions, transpile, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; /// every field type here is basedpython surface syntax that has no python /// spelling, so an unlowered one is either a `NameError` at class construction @@ -70,21 +75,155 @@ assert b._replace(name="bob").name == "bob", "named spelling" print("ok") "#; +/// the checker reads a plain tuple literal standing where an anonymous named tuple is +/// expected as an instance of it, wherever the expectation comes from. each one used to +/// stay a plain tuple at runtime unless it was an annotated assignment, a `return` or an +/// element of a list or set literal, so `by check` was clean and every field access below +/// raised `AttributeError` +const POSITIONS_PROGRAM: &str = r#" +from typing import Iterator + +type P = (a: int, b: int) + + +def take(p: (a: int, b: int)) -> int: + return p.a + + +def keyword(*, p: (a: int, b: int)) -> int: + return p.b + + +def star(*ps: (a: int, b: int)) -> int: + return sum(p.a for p in ps) + + +def default(p: (a: int, b: int) = (5, 6)) -> int: + return p.a + + +def pairs() -> Iterator[(a: int, b: int)]: + yield (1, 2) + + +def through_alias() -> P: + return (9, 10) + + +class Holder: + init(let pair: (a: int, b: int)) + + +class Box: + var pair: (a: int, b: int) = (0, 0) + + +assert take((7, 8)) == 7, "argument" +assert keyword(p=(7, 8)) == 8, "keyword argument" +assert star((1, 2), (3, 4)) == 4, "*args" +assert default() == 5, "default" +assert next(pairs()).b == 2, "yield" +assert through_alias().a == 9, "return through an alias" +assert Holder((3, 4)).pair.b == 4, "init parameter" + +items: list[(a: int, b: int)] = [] +items.append((1, 2)) +assert items[0].a == 1, "method argument" + +by_name: dict[str, (a: int, b: int)] = {"k": (1, 2)} +assert by_name["k"].b == 2, "dict value" + +unique: set[(a: int, b: int)] = {(1, 2)} +assert next(iter(unique)).a == 1, "set element" + +nested: list[list[(a: int, b: int)]] = [[(1, 2)]] +assert nested[0][0].b == 2, "nested list" + +box = Box() +assert box.pair.a == 0, "class default" +box.pair = (3, 4) +assert box.pair.b == 4, "attribute write" +print("ok") +"#; + +/// a field that is itself an anonymous named tuple builds the inner literal too, a union +/// builds the one arm that accepts the literal, and a write through a union receiver whose +/// members would each build the same anonymous named tuple builds it +const NESTED_AND_UNION_PROGRAM: &str = r#" +def outer(p: (inner: (x: int, y: int), label: str)) -> int: + return p.inner.y + + +class Left: + var pair: (a: int, b: int) = (0, 0) + + +class Right: + var pair: (a: int, b: int) = (0, 0) + + +class Maybe: + var pair: (a: int, b: int) | None = None + + +def assign(target: Left | Right | Maybe) -> int: + target.pair = (1, 2) + return target.pair.b + + +assert outer(((1, 2), "a")) == 2, "nested field" + +maybe: (a: int) | None = (1,) +assert maybe.a == 1, "optional" + +either: (a: int) | (b: str) = ("x",) +assert either.b == "x", "the arm whose fields accept the elements" + +assert assign(Left()) == 2, "union receiver" +assert assign(Right()) == 2, "union receiver" +assert assign(Maybe()) == 2, "a member that may also hold None builds the same class" +print("ok") +"#; + +/// a value that already is the named tuple passes through untouched, and a literal where a +/// plain tuple is expected stays a plain tuple +const UNCHANGED_PROGRAM: &str = r#" +def same(p: (a: int, b: int)) -> (a: int, b: int): + return p + + +def kind(t: tuple[int, int]) -> str: + return type(t).__name__ + + +built: (a: int, b: int) = (1, 2) +assert same(built) === built, "an existing named tuple is not rebuilt" +assert kind((1, 2)) == "tuple", "a plain tuple stays a tuple" +assert kind(built) == type(built).__name__, "a named tuple passed as a tuple stays itself" +print("ok") +"#; + +fn run(program: &str) { + run_with( + program, + &Config { + min_version: PythonVersion::PY313, + ..Config::default() + }, + ); +} + #[expect( clippy::print_stderr, reason = "a skipped test must say why it skipped, or it reads as a pass" )] -fn run(program: &str) { +fn run_with(program: &str, config: &Config) { let Some(python) = common::python() else { eprintln!("skipping anon named tuple runtime test: no interpreter found"); return; }; - let config = Config { - min_version: PythonVersion::PY313, - ..Config::default() - }; - let transpiled = transpile(program, &config).expect("transpile should succeed"); + let transpiled = transpile(program, config).expect("transpile should succeed"); let output = Command::new(&python) .arg("-c") @@ -131,3 +270,131 @@ print("ok") fn a_hoisted_class_sees_the_runtime() { run(WRAPPED_OPTIONAL_PROGRAM); } + +#[test] +fn a_plain_tuple_is_built_wherever_one_is_expected() { + run(POSITIONS_PROGRAM); +} + +#[test] +fn a_nested_field_and_a_union_arm_are_built() { + run(NESTED_AND_UNION_PROGRAM); +} + +#[test] +fn a_named_tuple_and_a_plain_tuple_are_left_as_they_are() { + run(UNCHANGED_PROGRAM); +} + +/// every runtime soundness check on, a parameter's entry check among them: each checks +/// the value against the class the literal was built as +#[test] +fn built_values_pass_every_soundness_check() { + for program in [ + POSITIONS_PROGRAM, + NESTED_AND_UNION_PROGRAM, + UNCHANGED_PROGRAM, + ] { + run_with( + program, + &Config { + min_version: PythonVersion::PY313, + soundness: SoundnessPositions::all(), + ..Config::default() + }, + ); + } +} + +/// transpile every one of `files` as one project, into a fresh directory under the cargo temp +/// dir, or the first transpile error +fn transpile_project(name: &str, files: &[(&str, &str)]) -> Result { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(name); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + for (path, _) in files { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = + transpile_typed(&db, file, &config, None).map_err(|error| error.message)?; + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + Ok(dir) +} + +/// a literal standing where another module's anonymous named tuple is expected is built in +/// this module, as a class declared from the checker's field types: the other module's class +/// is named by its own transpilation, which this one cannot see +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_literal_for_another_modules_named_tuple_is_built() { + let Some(python) = common::python() else { + eprintln!("skipping anon named tuple runtime test: no interpreter found"); + return; + }; + let dir = transpile_project( + "anon_named_tuple_cross_module", + &[ + ( + "/shapes.by", + "def take(p: (a: int, b: list[str])) -> str:\n return p.b[p.a]\n", + ), + ( + "/main.by", + "from shapes import take\n\nassert take((1, [\"x\", \"y\"])) == \"y\"\nprint(\"ok\")\n", + ), + ], + ) + .expect("transpile should succeed"); + let output = Command::new(&python) + .arg("main.py") + .current_dir(&dir) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- main.py ---\n{}", + String::from_utf8_lossy(&output.stderr), + fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +/// a field type the module building the literal has no python spelling for is refused, rather +/// than declared in a class that would not run +#[test] +fn a_field_the_module_cannot_spell_is_refused() { + let error = transpile_project( + "anon_named_tuple_unspellable_field", + &[ + ( + "/callbacks.by", + "def run(p: (f: (int) -> str)) -> str:\n return p.f(1)\n", + ), + ("/main.by", "from callbacks import run\n\nrun((str,))\n"), + ], + ) + .expect_err("the field type has no spelling"); + assert_eq!( + error, + "the tuple at offset 31 is built as an anonymous named tuple whose field `f` has no \ + type the emitted python can name; construct it explicitly, or give it a `NamedTuple` \ + class of its own" + ); +} diff --git a/crates/by_transforms/tests/attribute_write_runtime.rs b/crates/by_transforms/tests/attribute_write_runtime.rs new file mode 100644 index 0000000000..ddb2fc55be --- /dev/null +++ b/crates/by_transforms/tests/attribute_write_runtime.rs @@ -0,0 +1,59 @@ +//! Runtime test for a write to a declared attribute in a `.by` file. +//! +//! The value is read as what it is checked as: the declaration is its expected type, as it is +//! for an annotated assignment. A construction of a reified generic class is told the +//! specialization it builds from that, where without it the class could not say which one. + +mod common; + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +const PROGRAM: &str = r#" +class Box[reified T]: + def kind(self) -> str: + return T.__name__ + + +class Holder: + box: Box[int] + + def __init__(self): + self.box = Box() + + +holder = Holder() +assert holder.box.kind() == "int", "construction in a method" +holder.box = Box() +assert holder.box.kind() == "int", "construction through an instance" +print("ok") +"#; + +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +#[test] +fn a_reified_construction_takes_the_declared_specialization() { + let Some(python) = common::python() else { + eprintln!("skipping attribute write runtime test: no interpreter found"); + return; + }; + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index bac390e807..6730aacd29 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -8,7 +8,7 @@ Default level: error · Added in 0.0.64 · Related issues · -View source +View source @@ -44,7 +44,7 @@ class Base(ABC): Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -90,7 +90,7 @@ class Derived(Base): # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -131,13 +131,18 @@ Added in 0 Checks for conversion sites where more than one conversion applies — two dunders, a dunder and an in-scope conformance, or two applicable -`implementation`s of the same interface and type. +`implementation`s of the same interface and type. A plain tuple literal +that more than one expected anonymous named tuple accepts is one too: +`(1,)` where `(a: int) | (b: int)` is expected. **Why is this bad?** `__from__` and `__into__` are hand-written bodies that can disagree, so which one runs must not depend on arbitrary ordering. Remove one of them, -or write the conversion you want explicitly. +or write the conversion you want explicitly. A plain tuple literal is +constructed as the anonymous named tuple it stands for, which decides the +field names the value has, and nothing in the literal says which one was +meant: name the fields, as in `(a=1)`. **Example** @@ -258,7 +263,7 @@ class SubProto(BaseProto, Protocol): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -340,7 +345,7 @@ value = unknown # ty: ignore[unresolved-reference] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -383,7 +388,7 @@ a4 = True + 1 # ok — a boolean used as a boolean Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -498,7 +503,7 @@ def f(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -546,7 +551,7 @@ def main(): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -660,7 +665,7 @@ class C(A, B): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -860,7 +865,7 @@ old_func() # error: [deprecated] Default level: ignore · Added in 0.0.78 · Related issues · -View source +View source @@ -1295,7 +1300,7 @@ def bar() -> str: # error: [empty-body] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -1356,7 +1361,7 @@ def h(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -1458,7 +1463,7 @@ def foo() -> "intt\b": ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1497,7 +1502,7 @@ def f(local fn: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1578,7 +1583,7 @@ def g(value: ~A) -> None: ... # error: [experimental-syntax] Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -1612,7 +1617,7 @@ def my_function() -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.40 · Related issues · -View source +View source @@ -1647,7 +1652,7 @@ let a = 1 Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -1817,7 +1822,7 @@ unpacking — is not a declaration, and is never reported. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -1996,7 +2001,7 @@ t[3] # error Default level: warn · Added in 0.0.1-alpha.33 · Related issues · -View source +View source @@ -2226,7 +2231,7 @@ a: int = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2307,7 +2312,7 @@ box.value = 1 # okay Default level: error · Added in 0.0.33 · Related issues · -View source +View source @@ -2871,7 +2876,7 @@ This rule corresponds to Ruff's Default level: error · Added in 0.0.1-alpha.28 · Related issues · -View source +View source @@ -2958,7 +2963,7 @@ extension list[T: int]: # error: `list` declares no type parameter `T` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -2987,7 +2992,7 @@ Author.objects.filter(name__startswith=1) # error: lookup wants `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -3023,7 +3028,7 @@ def test_user(user: int) -> None: # error: fixture provides `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -3071,7 +3076,7 @@ f"{'name':>10}" # ok Default level: error · Added in 0.0.1-alpha.35 · Related issues · -View source +View source @@ -3318,7 +3323,7 @@ carol = Person(name="Carol", aeg=25) # typo! Default level: warn · Added in 0.0.15 · Related issues · -View source +View source @@ -3504,7 +3509,7 @@ class B(metaclass=42): ... # error Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -4021,7 +4026,7 @@ def h(a: S = 1) -> S: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4213,7 +4218,7 @@ def g(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -4248,7 +4253,7 @@ def g[T](value: T) raises T: # error: `T@g` is not always an exception Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4281,7 +4286,7 @@ if m := re.match("(a)(b)", "ab"): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -4677,7 +4682,7 @@ class C: ... Default level: error · Added in 0.0.10 · Related issues · -View source +View source @@ -5130,7 +5135,7 @@ U = TypeVar("U", int, str, default=bytes) # error: [invalid-type-variable-defau Default level: error · Added in 0.0.28 · Related issues · -View source +View source @@ -5166,7 +5171,7 @@ class Child(Base): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5207,7 +5212,7 @@ def f(options: dict[str, object]): Default level: error · Added in 0.0.9 · Related issues · -View source +View source @@ -5241,7 +5246,7 @@ class Foo(TypedDict): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5471,7 +5476,7 @@ def g(arg: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -5544,7 +5549,7 @@ Movie = TypedDict("Film", {"title": str}) # error: [mismatched-type-name] Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -5643,7 +5648,7 @@ func() # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5682,7 +5687,7 @@ def g(): ... Default level: ignore · Preview (since 0.0.76) · Related issues · -View source +View source @@ -5851,7 +5856,7 @@ def ignore(event: str): ... Default level: error · Level under ty-compatible: ignore · Added in 0.0.41 · Related issues · -View source +View source @@ -5912,7 +5917,7 @@ class ExplicitChild(Parent): Default level: error · Added in 0.0.75 · Related issues · -View source +View source @@ -6037,7 +6042,7 @@ def handle(m: re.Match[str]) -> str: Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -6076,7 +6081,7 @@ alice["age"] # KeyError Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -6123,7 +6128,7 @@ def App(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6231,7 +6236,7 @@ class Sub(Super): ... # error: [non-callable-init-subclass] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -6262,7 +6267,7 @@ def f(x: int | str) -> int: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6291,7 +6296,7 @@ def f(a: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6392,7 +6397,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6421,7 +6426,7 @@ def f(once done: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6449,7 +6454,7 @@ def f(once done: () -> None): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6491,7 +6496,7 @@ def f(x: int?): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6546,7 +6551,7 @@ def g(name: str | None): Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -6583,7 +6588,7 @@ class B(A): Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -6620,7 +6625,7 @@ class B(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.38 · Related issues · -View source +View source @@ -6694,7 +6699,7 @@ f(1, x=2) # error Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -7015,7 +7020,7 @@ from helpers import Key # error: `Key` is private to `helpers` Default level: warn · Added in 0.0.60 · Related issues · -View source +View source @@ -7090,7 +7095,7 @@ def test() -> "int": Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7132,7 +7137,7 @@ def g(a: bool | None): Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7167,7 +7172,7 @@ cast(int, f()) # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7221,7 +7226,7 @@ if sys.version_info >= (3, 12): # ok — artificially constant Default level: warn · Added in 0.0.18 · Related issues · -View source +View source @@ -7259,7 +7264,7 @@ class C: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7427,7 +7432,7 @@ def g(values: tuple[int, ...]) -> None: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -7460,7 +7465,7 @@ class C: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -7496,7 +7501,7 @@ class C[T]: Default level: warn · Added in 0.0.71 · Related issues · -View source +View source @@ -7539,7 +7544,7 @@ def build(t: Tag) -> None: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -7583,7 +7588,7 @@ class Outer[T]: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7632,7 +7637,7 @@ def Observed(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7676,7 +7681,7 @@ def Counter(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7710,7 +7715,7 @@ static_assert(int(2.0 * 3.0) == 6) # error Default level: warn · Added in 0.0.39 · Related issues · -View source +View source @@ -7828,7 +7833,7 @@ class Circle(Shape): ... # error: `Shape` is sealed in another workspace Default level: error · Added in 0.0.1-alpha.30 · Related issues · -View source +View source @@ -7937,7 +7942,7 @@ class Book: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7967,7 +7972,7 @@ f("foo") # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8006,7 +8011,7 @@ def find(items: list[int]) -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8053,7 +8058,7 @@ g: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8088,7 +8093,7 @@ f: # error: the block returns `None`, not `str` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8127,7 +8132,7 @@ def _(x: int): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -8158,7 +8163,7 @@ class User(BaseModel): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8290,7 +8295,7 @@ the project registers with `@register.simple_block_tag`. Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -8356,7 +8361,7 @@ what the projects depending on it read. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8385,7 +8390,7 @@ def f() raises TypeError: Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8414,7 +8419,7 @@ reveal_type(1) # revealed: Literal[1] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8452,7 +8457,7 @@ def check(x: int | None) -> asserts x: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8482,7 +8487,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8513,7 +8518,7 @@ f(x=1, y=2) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -8758,7 +8763,7 @@ implements Backend # error: this module does not answer `Backend` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8817,7 +8822,7 @@ def Held(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8850,7 +8855,7 @@ A().foo # error Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -8925,7 +8930,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8953,7 +8958,7 @@ import foo # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8982,7 +8987,7 @@ def check(value: int | None) -> asserts values: # error: `values` is nothing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9228,7 +9233,7 @@ Python code. Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -9561,7 +9566,7 @@ generator boundaries. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -9611,7 +9616,7 @@ A() # error: nothing says which specialization this is Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -9798,7 +9803,7 @@ def factory(base: type[Base]) -> type: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9873,7 +9878,7 @@ reveal_type(project.root) # revealed: "." Default level: warn · Added in 0.0.21 · Related issues · -View source +View source @@ -9952,7 +9957,7 @@ to `false` to prevent this rule from reporting unused `type: ignore` comments. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-beta.2 · Related issues · -View source +View source @@ -10032,7 +10037,7 @@ on. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.71 · Related issues · -View source +View source @@ -10242,7 +10247,7 @@ def foo(x: int | str) -> int | str: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_anonymous_named_tuple_literals.md b/crates/ty_python_semantic/resources/mdtest/basedpython_anonymous_named_tuple_literals.md new file mode 100644 index 0000000000..604e9ef371 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_anonymous_named_tuple_literals.md @@ -0,0 +1,303 @@ +# basedpython: a plain tuple literal built as an anonymous named tuple + +a plain tuple literal standing where an [anonymous named tuple](anonymous_named_tuple.md) is +expected is an instance of it, and the transpiler constructs that class around it, so the fields the +checker accepts reading are there at runtime. every block here runs + +## Every position that gives the literal an expected type + +the expectation can come from anywhere bidirectional inference reaches: an argument, a keyword +argument, an element of `*args`, a default, a `yield`, an element or value of a collection literal +at any depth, a method call on a collection, or a write to an attribute. each value below has its +fields at runtime + +```by +from typing import Iterator + +def take(p: (a: int, b: int)) -> int: + return p.a + +def keyword(*, p: (a: int, b: int)) -> int: + return p.b + +def star(*ps: (a: int, b: int)) -> int: + return sum(p.a for p in ps) + +def default(p: (a: int, b: int) = reveal_type((5, 6))) -> int: # revealed: (a: int, b: int) + return p.a + +def pairs() -> Iterator[(a: int, b: int)]: + yield reveal_type((1, 2)) # revealed: (a: int, b: int) + +class Box: + var pair: (a: int, b: int) = (0, 0) + +assert take(reveal_type((7, 8))) == 7 # revealed: (a: int, b: int) +assert keyword(p=reveal_type((7, 8))) == 8 # revealed: (a: int, b: int) +assert star(reveal_type((1, 2)), (3, 4)) == 4 # revealed: (a: int, b: int) +assert default() == 5 +assert next(pairs()).b == 2 + +items: list[(a: int, b: int)] = [] +items.append(reveal_type((1, 2))) # revealed: (a: int, b: int) +assert items[0].a == 1 + +by_name: dict[str, (a: int, b: int)] = {"k": reveal_type((1, 2))} # revealed: (a: int, b: int) +assert by_name["k"].b == 2 + +nested: list[list[(a: int, b: int)]] = [[reveal_type((1, 2))]] # revealed: (a: int, b: int) +assert nested[0][0].a == 1 + +box = Box() +box.pair = reveal_type((3, 4)) # revealed: (a: int, b: int) +assert box.pair.b == 4 +``` + +where a plain tuple type is expected, a literal stays a plain tuple: + +```by +t: tuple[int, int] = reveal_type((3, 4)) # revealed: (3, 4) +``` + +## A field that is itself an anonymous named tuple + +the literal for such a field is expected as that field's type, so it is built as the inner class: + +```by +def outer(p: (inner: (x: int, y: int), label: str)) -> int: + return p.inner.y + +assert outer(((1, 2), "a")) == 2 +``` + +## One arm of a union + +an optional anonymous named tuple builds the literal as its one anonymous named tuple arm: + +```by +maybe: (a: int) | None = reveal_type((1,)) # revealed: (a: int) +assert maybe.a == 1 +``` + +where the expected type allows several anonymous named tuples with as many fields as the literal +has, the one whose field types accept its elements is built: + +```by +either: (a: int) | (b: str) = reveal_type(("x",)) # revealed: (b: str) +assert either.b == "x" + +other: (a: int) | (b: str) = reveal_type((1,)) # revealed: (a: int) +assert other.a == 1 +``` + +an anonymous named tuple arm that does not accept the literal leaves it a plain tuple, inferred +against the rest of the expected type: + +```by +loose: (a: list[int]) | tuple[list[object]] = ([1, "s"],) +assert loose == ([1, "s"],) +``` + +an anonymous named tuple is a subtype of the plain tuple of its field types, so a union with that +tuple is the tuple alone, and the literal stays a plain tuple: + +```by +plain: (a: int) | tuple[int] = reveal_type((1,)) # revealed: (1,) +assert type(plain).__name__ == "tuple" +``` + +## A literal more than one arm accepts is an error + +the class decides which field names the value has at runtime, and nothing in a plain tuple says +which one was meant, so a literal more than one anonymous named tuple accepts is reported rather +than built as whichever the union lists first: + +```by +# error: [ambiguous-conversion] "More than one anonymous named tuple accepts this tuple" +either: (a: int) | (b: int) = (1,) +``` + +## A write through a union receiver whose members agree + +one value reaches whichever class the receiver turns out to be. where each declares the same +anonymous named tuple, the literal is built as it: + +```by +class Left: + var pair: (a: int, b: int) = (0, 0) + +class Right: + var pair: (a: int, b: int) = (0, 0) + +def assign(target: Left | Right) -> None: + target.pair = reveal_type((1, 2)) # revealed: (a: int, b: int) + assert target.pair.a == 1 + +assign(Left()) +assign(Right()) +``` + +## A write through a union receiver whose members build the same class + +the members need not declare the same type, only build the same class: one that may also hold `None` +builds the literal as the same anonymous named tuple + +```by +type P = (a: int, b: int) + +class First: + var pair: P = (0, 0) + +class Second: + var pair: P = (0, 0) + +class Optional: + var pair: P | None = None + +def assign(target: First | Second | Optional) -> None: + target.pair = (1, 2) + assert target.pair.b == 2 + +assign(First()) +assign(Optional()) +``` + +## A write through a union receiver one of whose members builds nothing + +a member that builds nothing from the literal accepts the class another builds, which is a subtype +of the plain tuple: + +```by +type P = (a: int, b: int) + +class Built: + var pair: P = (0, 0) + +class Loose: + var pair: object = 0 + +def loosen(target: Built | Loose) -> None: + target.pair = (1, 2) + assert target.pair == (1, 2) + +loosen(Built()) +loosen(Loose()) +``` + +## A write through a union receiver whose members disagree + +where the members declare different anonymous named tuples, no single value is both, so the write is +rejected: + +```by +class Left: + var pair: (a: int, b: int) = (0, 0) + +class Other: + var pair: (c: int, d: int) = (0, 0) + +def assign(target: Left | Other) -> None: + # error: [invalid-assignment] "The members of `Left | Other` would build different anonymous named tuples from this value for attribute `pair`" + target.pair = (1, 2) +``` + +## A write through a property setter + +a property's setter takes the value, so a literal written to the property is built as what the +setter's parameter declares + +```by +type Q = (c: int, d: int) + +class Store: + var held: Q = (0, 0) + + @property + def pair(self) -> Q: + return self.held + + @pair.setter + def pair(self, value: Q): + self.held = value + +store = Store() +store.pair = (1, 2) +assert store.pair.d == 2 +``` + +and a receiver that may be either a class declaring one anonymous named tuple or one whose setter +takes another would build two different classes: + +```by +type P = (a: int, b: int) + +class Declared: + var pair: P = (0, 0) + +class Through: + var held: Q = (0, 0) + + @property + def pair(self) -> Q: + return self.held + + @pair.setter + def pair(self, value: Q): + self.held = value + +def assign(target: Declared | Through) -> None: + # error: [invalid-assignment] "The members of `Declared | Through` would build different anonymous named tuples from this value for attribute `pair`" + target.pair = (1, 2) +``` + +## A write through a union receiver of other differing declarations + +a value with no plain tuple literal for the members to build differently is checked against each +member's own declaration, as python is, so a literal each of them accepts in its own way is fine: + +```by +from typing import TypedDict, NotRequired + +class Narrow(TypedDict): + a: int + +class Wide(TypedDict): + a: int + b: NotRequired[str] + +class Ints: + items: list[int] = [] + cfg: Narrow = {"a": 0} + +class Mixed: + items: list[int | str] = [] + cfg: Wide = {"a": 0} + +def assign(target: Ints | Mixed) -> None: + target.items = [1] + target.cfg = {"a": 1} + assert target.items == [1] + +assign(Ints()) +assign(Mixed()) +``` + +## A `.py` file keeps a plain tuple a tuple + +a `.py` file is python, where a plain tuple is never a named tuple, so passing one where a `.by` +module expects an anonymous named tuple is an error there: + +`shapes.by`: + +```by +def take(p: (a: int, b: int)) -> int: + return p.a +``` + +`main.py`: + +```py +from shapes import take + +take((1, 2)) # error: [invalid-argument-type] +``` diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index f0c3a57075..a0721a8a19 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -73,9 +73,10 @@ pub use types::static_resource::{ResourceError, render_as, resolve_static_resour pub use types::template::finite_string_set; pub use types::visibility::private_symbols; pub use types::{ - CollectedTest, CollectedTestClass, DisplaySettings, FixtureBinding, FixtureExposure, - FixtureNameSource, ProgramEnvironment, TypeQualifiers, collected_pytest_tests, - fixture_bindings_for_parameter, fixture_exposures_for_definition, pytest_global_plugin_files, + AnonNamedTupleClass, AnonNamedTupleId, CollectedTest, CollectedTestClass, DisplaySettings, + FixtureBinding, FixtureExposure, FixtureNameSource, ProgramEnvironment, TypeQualifiers, + collected_pytest_tests, fixture_bindings_for_parameter, fixture_exposures_for_definition, + pytest_global_plugin_files, }; pub mod api_lockfile; diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 24dc72dc0e..d92b66c145 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -19,6 +19,7 @@ use crate::RuntimeSpelling; use crate::place::implicit_globals::all_implicit_module_globals; use crate::place::imported_symbol; use crate::place::{builtins_module_scope, implicit_builtins_symbol_scope}; +use crate::types::AnonNamedTupleClass; use crate::types::ide_support::{ImportAliasResolution, definition_for_name}; use crate::types::implicit_names::implicit_name; use crate::types::list_members::{all_members, all_reachable_members}; @@ -747,6 +748,17 @@ impl<'db> SemanticModel<'db> { conversions } + /// basedpython: the anonymous named tuple class the checker read `tuple` as — + /// the one a type form or a value form declares, or the one a plain tuple + /// literal is built as where one was expected. `None` for any other tuple + pub fn anon_named_tuple_class( + &self, + tuple: &ast::ExprTuple, + ) -> Option> { + let ty = tuple.inferred_type(self)?; + AnonNamedTupleClass::of(self.db, &self.program_environment(), ty) + } + /// could any conversion apply at this call at all? /// /// Binding the call a second time to find out is the expensive part, and diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 887340a4d1..5b92b91b3f 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -25,6 +25,7 @@ use ty_module_resolver::{ ImportingFile, KnownModule, Module, ModuleName, file_to_module, resolve_module, }; +pub use self::anon_named_tuple::{AnonNamedTupleClass, AnonNamedTupleId}; pub(crate) use self::callable::UpcastPolicy; use self::class::ClassInstanceFlags; pub use self::cyclic::CycleDetector; @@ -170,6 +171,7 @@ use ty_python_core::place::ScopedPlaceId; use ty_python_core::scope::ScopeId; use ty_python_core::{ProgramFile, Truthiness, place_table, semantic_index, use_def_map}; +mod anon_named_tuple; mod attribute_write; mod bool; mod bound_super; diff --git a/crates/ty_python_semantic/src/types/anon_named_tuple.rs b/crates/ty_python_semantic/src/types/anon_named_tuple.rs new file mode 100644 index 0000000000..d5d57b8db1 --- /dev/null +++ b/crates/ty_python_semantic/src/types/anon_named_tuple.rs @@ -0,0 +1,131 @@ +//! basedpython anonymous named tuples, `(name: T, ...)` +//! +//! each one is a synthesized `NamedTuple` class. a plain tuple display standing +//! where one is expected — `take((1, 2))` with `def take(p: (a: int, b: int))` — +//! is read as an instance of that class, and the transpiler constructs it there. +//! both sides read the decision off the display's inferred type, so the checker +//! never accepts field access on a value the emitted python leaves a plain tuple + +use ruff_python_ast::name::Name; +use salsa::plumbing::AsId; + +use crate::Db; +use crate::types::class::{ + ClassLiteral, DynamicNamedTupleAnchor, DynamicNamedTupleLiteral, NamedTupleSpec, +}; +use crate::types::{ProgramEnvironment, Type, UnionType}; + +/// the prefix of every synthesized anonymous named tuple class's name +const CLASS_PREFIX: &str = "_AnonNamedTuple_"; + +/// the class and field specification of an anonymous named tuple `ty` is an instance +/// of, or (for a type form in a value position) the class itself +pub(crate) fn anon_named_tuple_class<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, +) -> Option<(DynamicNamedTupleLiteral<'db>, NamedTupleSpec<'db>)> { + let literal = match ty { + Type::NominalInstance(instance) => instance.class(db, env).class_literal(db), + Type::ClassLiteral(literal) => literal, + _ => return None, + }; + let ClassLiteral::DynamicNamedTuple(class) = literal else { + return None; + }; + if !class.name(db).as_str().starts_with(CLASS_PREFIX) { + return None; + } + match class.anchor(db) { + DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => Some((class, *spec)), + DynamicNamedTupleAnchor::CollectionsDefinition { .. } + | DynamicNamedTupleAnchor::TypingDefinition(_) => None, + } +} + +/// every anonymous named tuple instance `expected` allows, in the order its union +/// lists them. a type alias is read through, so this follows what an annotation +/// means rather than how it is spelled +pub(crate) fn expected_anon_named_tuples<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + expected: Type<'db>, +) -> Vec<(Type<'db>, NamedTupleSpec<'db>)> { + fn collect<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, + found: &mut Vec<(Type<'db>, NamedTupleSpec<'db>)>, + ) { + let ty = ty.resolve_type_alias(db); + if let Type::Union(union) = ty { + for member in union.elements(db) { + collect(db, env, *member, found); + } + return; + } + if let Type::NominalInstance(_) = ty + && let Some((_, spec)) = anon_named_tuple_class(db, env, ty) + && !found.iter().any(|(seen, _)| *seen == ty) + { + found.push((ty, spec)); + } + } + let mut found = Vec::new(); + collect(db, env, expected, &mut found); + found +} + +/// `expected` without its anonymous named tuple arms, or `None` when nothing is left. +/// +/// a plain tuple literal is an instance of an anonymous named tuple only where it is built +/// as one, so one that is not cannot use such an arm as the shape it is inferred against — +/// a `(a: list[int])` arm would make `([1, "s"],)` fail a `tuple[list[object]]` arm beside it +pub(crate) fn without_anon_named_tuples<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + expected: Type<'db>, +) -> Option> { + let resolved = expected.resolve_type_alias(db); + let arms: Vec> = match resolved { + Type::Union(union) => union.elements(db).to_vec(), + other => vec![other], + }; + let kept: Vec> = arms + .into_iter() + .filter(|arm| { + !(matches!(arm, Type::NominalInstance(_)) + && anon_named_tuple_class(db, env, *arm).is_some()) + }) + .collect(); + (!kept.is_empty()).then(|| UnionType::from_elements(db, env, kept)) +} + +/// one anonymous named tuple class, as the checker identifies it. two type forms +/// with the same identity are the same type, whatever their spelling +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct AnonNamedTupleId(salsa::Id); + +/// an anonymous named tuple class the checker read an expression as +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AnonNamedTupleClass<'db> { + pub id: AnonNamedTupleId, + /// each field's name and declared type, in order + pub fields: Vec<(Name, Type<'db>)>, +} + +impl<'db> AnonNamedTupleClass<'db> { + /// the anonymous named tuple `ty` is an instance of, or the class itself for a type + /// form standing in a value position + pub fn of(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>) -> Option { + let (class, spec) = anon_named_tuple_class(db, env, ty)?; + Some(Self { + id: AnonNamedTupleId(class.as_id()), + fields: spec + .fields(db) + .iter() + .map(|field| (field.name.clone(), field.ty)) + .collect(), + }) + } +} diff --git a/crates/ty_python_semantic/src/types/attribute_write.rs b/crates/ty_python_semantic/src/types/attribute_write.rs index 4ad942b4f9..3adb1ff866 100644 --- a/crates/ty_python_semantic/src/types/attribute_write.rs +++ b/crates/ty_python_semantic/src/types/attribute_write.rs @@ -147,6 +147,113 @@ pub(super) enum ExplicitAttributeWriteRequirement<'db> { }, } +impl<'db> AttributeWriteRequirement<'db> { + /// the one type the assigned value is checked against, where it is checked against one: a + /// declared type, or the value a descriptor's setter takes + pub(super) fn write_type( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ) -> Option> { + match self { + Self::Instance { object_ty, member } => member.write_type(db, env, *object_ty), + Self::Class { object_ty, member } => match member { + ClassAttributeWriteMember::Explicit { member, .. } => { + member.write_type(db, env, *object_ty) + } + ClassAttributeWriteMember::ClassAttribute( + FallbackAttributeWriteRequirement::AssignableTo { ty, .. }, + ) => Some(*ty), + ClassAttributeWriteMember::ClassAttribute( + FallbackAttributeWriteRequirement::PossiblyMissing, + ) + | ClassAttributeWriteMember::Unresolved { .. } => None, + }, + Self::Module(ty) => *ty, + Self::ProtocolMember { + write: Some(ProtocolMemberWriteRequirement::AssignableTo(ty)), + .. + } => Some(*ty), + Self::ProtocolMember { + write: + Some(ProtocolMemberWriteRequirement::Descriptor { + domain: Some(domain), + .. + }), + .. + } => Some(*domain), + Self::ProtocolMember { .. } + | Self::All { .. } + | Self::Any { .. } + | Self::Unconstrained + | Self::CannotAssign => None, + } + } +} + +impl<'db> InstanceAttributeWriteMember<'db> { + /// the type the assigned value is checked against on `object_ty`, where it is checked + /// against one + pub(super) fn write_type( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + object_ty: Type<'db>, + ) -> Option> { + match self { + Self::Explicit { member, .. } => member.write_type(db, env, object_ty), + Self::Instance(FallbackAttributeWriteRequirement::AssignableTo { ty, .. }) => Some(*ty), + Self::Instance(FallbackAttributeWriteRequirement::PossiblyMissing) + | Self::ClassVar + | Self::SetAttr => None, + } + } +} + +impl<'db> ExplicitAttributeWriteRequirement<'db> { + /// the type the assigned value is checked against on `object_ty`, where it is checked + /// against one: the declared type, or the value the descriptor's setter takes + pub(super) fn write_type( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + object_ty: Type<'db>, + ) -> Option> { + match self { + Self::AssignableTo { ty, .. } => Some(*ty), + Self::Descriptor { descriptor_ty, .. } => { + // a property's own `__set__` takes any value; what its setter takes is what + // the write is checked against + if let Some(property) = descriptor_ty.as_property_instance() { + return property + .setter(db) + .and_then(|setter| setter_value_type(db, setter)); + } + match descriptor_setter_domain(db, env, *descriptor_ty, object_ty) { + DescriptorSetterDomain::Known(domain) => Some(domain), + DescriptorSetterDomain::Missing | DescriptorSetterDomain::Deferred => None, + } + } + } + } +} + +/// the value a property setter takes: the type of its second parameter, where it has one +/// signature +fn setter_value_type<'db>(db: &'db dyn Db, setter: Type<'db>) -> Option> { + let Type::FunctionLiteral(function) = setter else { + return None; + }; + let [overload] = function.signature(db).overloads.as_slice() else { + return None; + }; + overload + .parameters() + .iter() + .nth(1) + .map(Parameter::annotated_type) +} + impl ExplicitAttributeWriteRequirement<'_> { pub(super) fn qualifiers(&self) -> TypeQualifiers { match self { diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index 009e6b4bb5..00a03de383 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -10929,6 +10929,24 @@ impl<'db> BindingError<'db> { } add_invariant_generic_hints(db, env, &mut diag, *expected_ty, *provided_ty); + + // a `.by` file builds a plain tuple standing where an anonymous named tuple + // is expected as one, so a caller written in python is where that stops + if !context.file().source_type(db).is_basedpython() + && provided_ty.tuple_instance_spec(db, env).is_some() + && !crate::types::anon_named_tuple::expected_anon_named_tuples( + db, + env, + *expected_ty, + ) + .is_empty() + { + diag.info( + "A `.py` file does not build an anonymous named tuple from a plain \ + tuple: construct it through a name its module binds the type to, \ + as in `P(1, 2)` where the module declares `P = (a: int, b: int)`", + ); + } } Self::InvalidKeyType { diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 2c562be014..b0271ecf79 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -1673,12 +1673,17 @@ declare_lint! { /// ## What it does /// Checks for conversion sites where more than one conversion applies — two /// dunders, a dunder and an in-scope conformance, or two applicable - /// `implementation`s of the same interface and type. + /// `implementation`s of the same interface and type. A plain tuple literal + /// that more than one expected anonymous named tuple accepts is one too: + /// `(1,)` where `(a: int) | (b: int)` is expected. /// /// ## Why is this bad? /// `__from__` and `__into__` are hand-written bodies that can disagree, so /// which one runs must not depend on arbitrary ordering. Remove one of them, - /// or write the conversion you want explicitly. + /// or write the conversion you want explicitly. A plain tuple literal is + /// constructed as the anonymous named tuple it stands for, which decides the + /// field names the value has, and nothing in the literal says which one was + /// meant: name the fields, as in `(a=1)`. /// /// ## Example /// diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 7ea16e3101..29e6ff80c3 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -56,6 +56,7 @@ use crate::reachability::{ }; use crate::subscript::PyIndex; use crate::types::add_inferred_python_version_hint_to_diagnostic; +use crate::types::anon_named_tuple; use crate::types::attribute_write::{AssignmentAttributeMembers, assignment_attribute_members}; use crate::types::call::bind::{ ArgumentTypeContext, CallableDescription, CheckTypesMode, OverloadSet, @@ -76,22 +77,23 @@ use crate::types::dedicated::{django, pydantic}; use crate::types::deferred::{is_integer_operand, is_symbolic_operand}; use crate::types::definition_resolution::AttributeRead; use crate::types::diagnostic::{ - self, AMBIGUOUS_EXTENSION_MEMBER, CALL_NON_CALLABLE, CONFLICTING_DECLARATIONS, - CYCLIC_TYPE_ALIAS_DEFINITION, DYNAMIC_FUNCTION_DECORATOR_RETURN, ERASED_CAST_ARGUMENT, - ERASED_TYPE_CHECK, FINAL_ON_VARIABLE, GeneratorMismatchKind, IMPLICIT_DECLARATION, - INACCESSIBLE_MEMBER, INEFFECTIVE_FINAL, INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, - INVALID_ATTRIBUTE_ACCESS, INVALID_DECLARATION, INVALID_ENUM_MEMBER_ANNOTATION, - INVALID_FIELD_LOOKUP, INVALID_LEGACY_TYPE_VARIABLE, INVALID_NEWTYPE, INVALID_PARAMSPEC, - INVALID_REGEX, INVALID_REIFIED_TYPE_PARAM, INVALID_TYPE_ALIAS_TYPE, INVALID_TYPE_FORM, - INVALID_TYPE_VARIABLE_CONSTRAINTS, INVALID_TYPE_VARIABLE_DEFAULT, INVALID_VARIANCE_DECLARATION, - NARROWING_GUARD_AS_VALUE, NON_EXHAUSTIVE_STATEMENT_EXPRESSION, NON_OVERLAPPING_CAST, - NON_OVERLAPPING_TYPE_TEST, OPTIONAL_OBJECT_CONVERSION, POSSIBLY_MISSING_IMPLICIT_CALL, - POSSIBLY_MISSING_SUBMODULE, PRIVATE_CONSTRUCTOR, REFUTABLE_DESTRUCTURING, REFUTABLE_UNPACKING, - TRAILING_LAMBDA_PARAMETERS, TypeCheckDiagnostics, UNANNOTATED_MODEL_FIELD, - UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, - UNRESOLVED_GLOBAL, UNRESOLVED_REFERENCE, UNSOUND_ASSIGNMENT, UNSOUND_CAST, UNSOUND_YIELD, - UNSPECIALIZED_REIFIED_GENERIC, UNSUPPORTED_OPERATOR, UNUSED_AWAITABLE, USED_UNDERSCORE_NAME, - YieldKind, autofix_with_notimplementederror, display_required_elements, + self, AMBIGUOUS_CONVERSION, AMBIGUOUS_EXTENSION_MEMBER, CALL_NON_CALLABLE, + CONFLICTING_DECLARATIONS, CYCLIC_TYPE_ALIAS_DEFINITION, DYNAMIC_FUNCTION_DECORATOR_RETURN, + ERASED_CAST_ARGUMENT, ERASED_TYPE_CHECK, FINAL_ON_VARIABLE, GeneratorMismatchKind, + IMPLICIT_DECLARATION, INACCESSIBLE_MEMBER, INEFFECTIVE_FINAL, INVALID_ARGUMENT_TYPE, + INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_DECLARATION, + INVALID_ENUM_MEMBER_ANNOTATION, INVALID_FIELD_LOOKUP, INVALID_LEGACY_TYPE_VARIABLE, + INVALID_NEWTYPE, INVALID_PARAMSPEC, INVALID_REGEX, INVALID_REIFIED_TYPE_PARAM, + INVALID_TYPE_ALIAS_TYPE, INVALID_TYPE_FORM, INVALID_TYPE_VARIABLE_CONSTRAINTS, + INVALID_TYPE_VARIABLE_DEFAULT, INVALID_VARIANCE_DECLARATION, NARROWING_GUARD_AS_VALUE, + NON_EXHAUSTIVE_STATEMENT_EXPRESSION, NON_OVERLAPPING_CAST, NON_OVERLAPPING_TYPE_TEST, + OPTIONAL_OBJECT_CONVERSION, POSSIBLY_MISSING_IMPLICIT_CALL, POSSIBLY_MISSING_SUBMODULE, + PRIVATE_CONSTRUCTOR, REFUTABLE_DESTRUCTURING, REFUTABLE_UNPACKING, TRAILING_LAMBDA_PARAMETERS, + TypeCheckDiagnostics, UNANNOTATED_MODEL_FIELD, UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, + UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, UNRESOLVED_REFERENCE, + UNSOUND_ASSIGNMENT, UNSOUND_CAST, UNSOUND_YIELD, UNSPECIALIZED_REIFIED_GENERIC, + UNSUPPORTED_OPERATOR, UNUSED_AWAITABLE, USED_UNDERSCORE_NAME, YieldKind, + autofix_with_notimplementederror, display_required_elements, hint_if_stdlib_attribute_exists_on_other_versions, refutable_unpacking_applies, report_attempted_protocol_instantiation, report_bad_dunder_delattr_call, report_bad_dunder_delete_call, report_bool_as_int, report_bool_as_int_assignment, @@ -8842,6 +8844,87 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Type::ClassLiteral(ClassLiteral::DynamicNamedTuple(nt)) } + /// every plain tuple literal in `value` this builder has read as an anonymous named + /// tuple, with the class it is built as + fn anon_named_tuple_constructions( + &self, + value: &ast::Expr, + ) -> Vec<(TextRange, DynamicNamedTupleLiteral<'db>)> { + struct Literals<'a, 'db, 'ast> { + builder: &'a TypeInferenceBuilder<'db, 'ast>, + found: Vec<(TextRange, DynamicNamedTupleLiteral<'db>)>, + } + impl<'b> ast::visitor::Visitor<'b> for Literals<'_, '_, '_> { + fn visit_expr(&mut self, expr: &'b ast::Expr) { + if let ast::Expr::Tuple(tuple) = expr + && !tuple.is_anon_named_tuple + && !tuple.is_anon_named_tuple_value + && let Some(ty) = self.builder.try_expression_type(expr) + && let Some((class, _)) = anon_named_tuple::anon_named_tuple_class( + self.builder.db(), + self.builder.program_environment(), + ty, + ) + { + self.found.push((tuple.range, class)); + } + ast::visitor::walk_expr(self, expr); + } + } + let mut literals = Literals { + builder: self, + found: Vec::new(), + }; + ast::visitor::Visitor::visit_expr(&mut literals, value); + literals.found + } + + /// whether each element of the plain tuple literal `tuple`, inferred against its + /// field's type, is assignable to it. inferred speculatively, so nothing is + /// recorded for the literal + fn anon_named_tuple_fields_accept( + &self, + tuple: &ast::ExprTuple, + spec: NamedTupleSpec<'db>, + ) -> bool { + let env = self.program_environment(); + let db = self.db(); + let mut speculative = self.speculate_without_diagnostics(); + tuple + .elts + .iter() + .zip(spec.fields(db).iter()) + .all(|(elt, field)| { + speculative + .infer_expression(elt, TypeContext::new(Some(field.ty))) + .is_assignable_to(db, env, field.ty) + }) + } + + /// report a plain tuple literal that more than one expected anonymous named + /// tuple accepts + fn report_ambiguous_anon_named_tuple( + &self, + tuple: &ast::ExprTuple, + accepting: &[(Type<'db>, NamedTupleSpec<'db>)], + ) { + let env = self.program_environment(); + let db = self.db(); + let Some(builder) = self.context.report_lint(&AMBIGUOUS_CONVERSION, tuple) else { + return; + }; + let mut diagnostic = + builder.into_diagnostic("More than one anonymous named tuple accepts this tuple"); + let names = accepting + .iter() + .map(|(target, _)| format!("`{}`", target.display(db, env))) + .join(", "); + diagnostic.info(format_args!( + "{names} all accept it, and a plain tuple does not say which one to build" + )); + diagnostic.help("Name the fields, or construct the one you want explicitly"); + } + fn infer_tuple_expression( &mut self, tuple: &ast::ExprTuple, @@ -8872,13 +8955,17 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // same coercion the plain-tuple spelling gets below: `b: P = (name="a")` // and `b: P = ("a",)` construct the one class the transpiler emits, whose // fields carry the *declared* types — so `b._replace(name="x")` is a call - // the runtime accepts - if let Some(target) = tcx - .annotation() - .and_then(|annotation| find_anon_nt_target(self.db(), env, annotation)) - && literal_instance.is_assignable_to(self.db(), env, target) + // the runtime accepts. with several that accept it, none is chosen: the + // literal one is still assignable to each + if let Some(annotation) = tcx.annotation() + && let [(target, _)] = + anon_named_tuple::expected_anon_named_tuples(db, env, annotation) + .into_iter() + .filter(|(target, _)| literal_instance.is_assignable_to(db, env, *target)) + .collect::>() + .as_slice() { - return target; + return *target; } return literal_instance; } @@ -8896,83 +8983,54 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } return Type::unknown(); } - // basedpython: a *plain* tuple literal in a position whose expected - // type (via bidirectional inference) is an anonymous-named-tuple - // instance gets inferred AS that instance — matching the transpiler's - // implicit constructor coercion. This makes `def f() -> (name: str, - // age: int): return ("a", 1)` type-check, since at runtime the - // returned value is wrapped as `_AnonNamedTuple_xxx("a", 1)`. - // Find an anon-NT instance candidate in the expected type. Accept the - // direct case (`x: anon-NT`) and any union member (`x: anon-NT | None`, - // `x: anon-NT | Other`, …). The first matching union member wins. A - // `type P = (name: str)` alias is resolved through, so the coercion - // follows what the annotation means rather than how it is spelled — - // which is also what the transpiler wraps on. - #[expect(clippy::items_after_statements, reason = "helper colocated with use")] - fn find_anon_nt_target<'db>( - db: &'db dyn crate::Db, - env: &ProgramEnvironment<'db>, - ty: Type<'db>, - ) -> Option> { - let ty = ty.resolve_type_alias(db); - if let Type::NominalInstance(instance) = ty - && let crate::types::class::ClassLiteral::DynamicNamedTuple(nt) = - instance.class(db, env).class_literal(db) - && nt.name(db).as_str().starts_with("_AnonNamedTuple_") - { - return Some(ty); - } - if let Type::Union(union) = ty { - for member in union.elements(db).iter().copied() { - if let Some(found) = find_anon_nt_target(db, env, member) { - return Some(found); - } - } - } - None - } - if let Some(target_instance) = tcx - .annotation() - .and_then(|a| find_anon_nt_target(self.db(), env, a)) - && let Type::NominalInstance(instance) = target_instance - && let crate::types::class::ClassLiteral::DynamicNamedTuple(nt) = - instance.class(self.db(), env).class_literal(self.db()) - && nt.name(self.db()).as_str().starts_with("_AnonNamedTuple_") + // basedpython: a *plain* tuple literal in a position whose expected type + // (via bidirectional inference) is an anonymous named tuple is inferred as + // an instance of it, and the transpiler constructs that class around it — + // so `take((1, 2))` with `def take(p: (a: int, b: int))` reads `p.a` off a + // value that has it. the transpiler finds the sites by this inferred type, + // so whatever position gives the literal its expected type is one. + // + // the expected type may allow several (`(a: int) | (b: str)`), or one alongside + // other arms (`(a: list[int]) | tuple[list[object]]`). those whose field count + // matches are candidates, and a candidate is built only where its fields accept + // the elements — otherwise the literal is inferred as a tuple, against the rest + // of the expected type. a literal more than one candidate accepts is reported + // rather than given to whichever the union lists first: the class decides which + // field names exist at runtime, and nothing in the literal says which. a `.py` + // file has no such construction, so a plain tuple stays a plain tuple there + let mut tcx = tcx; + if self.is_basedpython_file() + && !tuple.elts.iter().any(ast::Expr::is_starred_expr) + && let Some(annotation) = tcx.annotation() { - let spec = match nt.anchor(self.db()) { - crate::types::class::DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => { - Some(*spec) + let expected = anon_named_tuple::expected_anon_named_tuples(db, env, annotation); + let accepting: Vec<_> = expected + .iter() + .copied() + .filter(|(_, spec)| { + spec.fields(db).len() == tuple.elts.len() + && self.anon_named_tuple_fields_accept(tuple, *spec) + }) + .collect(); + match accepting.as_slice() { + [] if expected.is_empty() => {} + [] => { + tcx = TypeContext::new(anon_named_tuple::without_anon_named_tuples( + db, env, annotation, + )); } - _ => None, - }; - if let Some(spec) = spec { - let fields = spec.fields(self.db()); - if fields.len() == tuple.elts.len() - && tuple - .elts - .iter() - .all(|e| !matches!(e, ast::Expr::Starred(_))) - { - // type-check element-wise against the expected field - // types. when every element is assignable, return the - // anon-NT instance so attribute access works; otherwise - // build a plain heterogeneous tuple from the inferred - // types (each element has already been inferred, so we - // can't re-call `infer_expression` in a fall-through - // branch) - let mut elt_tys: Vec> = Vec::with_capacity(tuple.elts.len()); - let mut all_assignable = true; - for (elt, field) in tuple.elts.iter().zip(fields.iter()) { - let elt_ty = self.infer_expression(elt, TypeContext::new(Some(field.ty))); - if !elt_ty.is_assignable_to(self.db(), env, field.ty) { - all_assignable = false; - } - elt_tys.push(elt_ty); + [(target, spec)] => { + for (elt, field) in tuple.elts.iter().zip(spec.fields(db).iter()) { + self.infer_expression(elt, TypeContext::new(Some(field.ty))); } - if all_assignable { - return target_instance; + return *target; + } + ambiguous => { + self.report_ambiguous_anon_named_tuple(tuple, ambiguous); + for elt in &tuple.elts { + self.infer_expression(elt, TypeContext::default()); } - return Type::heterogeneous_tuple(self.db(), env, elt_tys); + return Type::unknown(); } } } @@ -17916,6 +17974,19 @@ impl<'db, 'ast, 'infer> MultiInferenceGuard<'db, 'ast, 'infer> { (self.infer_expr)(&mut builder.speculate_without_diagnostics(), tcx) } + /// infer the expression silently with `tcx`, and read the result off the speculative + /// builder it was inferred in. the context is not remembered as the last one tried + fn infer_silent_with( + &mut self, + builder: &mut TypeInferenceBuilder<'db, 'ast>, + tcx: TypeContext<'db>, + inspect: impl FnOnce(&TypeInferenceBuilder<'db, 'ast>) -> R, + ) -> R { + let mut speculative = builder.speculate_without_diagnostics(); + (self.infer_expr)(&mut speculative, tcx); + inspect(&speculative) + } + fn last_tcx(&self) -> TypeContext<'db> { self.last_tcx.unwrap_or_default() } diff --git a/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs b/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs index 85dd4c22d8..9ad4932a5e 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/attribute_assignment.rs @@ -1,5 +1,7 @@ +use itertools::Itertools; use ruff_python_ast as ast; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextRange}; +use rustc_hash::FxHashMap; use super::{ArgumentsIter, MultiInferenceGuard, TypeInferenceBuilder}; use crate::place::{DefinedPlace, Place, PlaceAndQualifiers}; @@ -9,7 +11,7 @@ use crate::types::attribute_write::{ ProtocolMemberWriteRequirement, attribute_write_requirement, property_setter_returns_never, }; use crate::types::call::{Bindings, CallArguments, CallDiagnosticOverride, CallError}; -use crate::types::class::FrozenDataclassDispatch; +use crate::types::class::{DynamicNamedTupleLiteral, FrozenDataclassDispatch}; use crate::types::dedicated::pydantic; use crate::types::diagnostic::{ DEPRECATED, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, MISSING_SLOT, UNRESOLVED_ATTRIBUTE, @@ -19,7 +21,7 @@ use crate::types::diagnostic::{ use crate::types::safe_variance::private_member_view; use crate::types::{ CallDunderError, DisplaySettings, MemberLookupPolicy, PropertyDeprecations, Type, TypeContext, - TypeQualifiers, + TypeQualifiers, UnionType, }; use crate::{Db, ProgramEnvironment}; @@ -104,6 +106,11 @@ enum AssignmentAttributeWriteDiagnostic<'db> { object_ty: Type<'db>, value_ty: Type<'db>, }, + /// basedpython: the elements of a union receiver would build different anonymous named + /// tuples from a plain tuple literal in the value + DifferentAnonNamedTuples { + object_ty: Type<'db>, + }, CannotAssign, CannotAssignToClassVar, TerminalSetAttr { @@ -252,6 +259,36 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { } } + /// the context the assigned value is recorded with, when the write is checked against + /// `write_type`. + /// + /// the check re-infers the value with `write_type` as its context, so in a `.py` file the + /// recorded type only ever loses precision by leaving it out. in a `.by` file what a + /// literal *is* can depend on its context — a plain tuple literal written where an + /// anonymous named tuple is declared is built as one — so the value is recorded as what + /// it was checked as, which is what the transpiler builds + fn commit_context(&self, write_type: Option>) -> TypeContext<'db> { + if self.builder.is_basedpython_file() { + TypeContext::new(write_type) + } else { + TypeContext::default() + } + } + + /// the anonymous named tuple each plain tuple literal in the value is built as, inferred + /// against `write_type` + fn constructions_against( + &mut self, + write_type: Type<'db>, + ) -> Vec<(TextRange, DynamicNamedTupleLiteral<'db>)> { + let value = self.value; + self.infer_value_ty.infer_silent_with( + self.builder, + TypeContext::new(Some(write_type)), + |speculative| speculative.anon_named_tuple_constructions(value), + ) + } + /// Infer the value again using the context that succeeded. /// /// The earlier inference was only a trial, so its result was not saved. @@ -334,8 +371,53 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { object_ty, element_tys, } => { - let value_ty = self.infer_value(TypeContext::default(), emit_diagnostics); let attribute = self.attribute; + let by = self.builder.is_basedpython_file(); + // each element's declaration, where it has one. in a `.by` file the value is + // recorded against the declaration the elements share, if they share one + let write_types: Vec>> = if by { + element_tys + .iter() + .map(|ty| { + attribute_write_requirement(db, env, *ty, attribute).write_type(db, env) + }) + .collect() + } else { + Vec::new() + }; + // one value reaches whichever element the receiver is, and a plain tuple + // literal in it is built as the anonymous named tuple its expected type + // names. each element is still checked against its own declaration, so where + // the declarations differ, no two elements may build different classes from + // one literal; the value is recorded against the declarations of those that + // build one, which then builds it. a class built where another element builds + // none is one that element accepts too: it is a subtype of the plain tuple. + // where they share a declaration, it is recorded against that one + let shared = write_types.iter().all_equal_value().ok().copied().flatten(); + let mut builds_apart = false; + let tcx = if shared.is_some() { + self.commit_context(shared) + } else { + let mut built = FxHashMap::default(); + let mut building = Vec::new(); + for write_type in write_types.iter().flatten() { + let constructions = self.constructions_against(*write_type); + if !constructions.is_empty() { + building.push(*write_type); + } + for (range, class) in constructions { + builds_apart |= built + .insert(range, class) + .is_some_and(|earlier| earlier != class); + } + } + if builds_apart || building.is_empty() { + TypeContext::default() + } else { + self.commit_context(Some(UnionType::from_elements(db, env, building))) + } + }; + let value_ty = self.infer_value(tcx, emit_diagnostics); let mut requirements = element_tys .iter() .map(|ty| attribute_write_requirement(db, env, *ty, attribute)); @@ -364,6 +446,16 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { ) }); } + if builds_apart { + if emit_diagnostics { + self.report( + AssignmentAttributeWriteDiagnostic::DifferentAnonNamedTuples { + object_ty: *object_ty, + }, + ); + } + return false; + } if valid { self.validate_composite_final_assignment(*object_ty, emit_diagnostics); } else if emit_diagnostics { @@ -627,7 +719,10 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { ) { self.infer_and_try_call_setattr(setattr_receiver, emit_diagnostics) } else { - let value_ty = self.infer_value(TypeContext::default(), emit_diagnostics); + let value_ty = self.infer_value( + self.commit_context(member.write_type(db, env, object_ty)), + emit_diagnostics, + ); let setattr_result = setattr_receiver.try_call_dunder_with_policy( db, env, @@ -772,7 +867,10 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { self.infer_value(TypeContext::default(), emit_diagnostics); return false; } - let value_ty = self.infer_value(TypeContext::default(), emit_diagnostics); + let value_ty = self.infer_value( + self.commit_context(member.write_type(db, env, object_ty)), + emit_diagnostics, + ); let member_valid = self.evaluate_explicit_member(object_ty, member, value_ty, emit_diagnostics); if let Some(fallback) = fallback { @@ -1072,6 +1170,25 @@ impl<'db> AssignmentAttributeWriteEvaluator<'_, 'db, '_, '_> { )); } } + AssignmentAttributeWriteDiagnostic::DifferentAnonNamedTuples { object_ty } => { + if let Some(builder) = self + .builder + .context + .report_lint(&INVALID_ASSIGNMENT, self.target) + { + let mut diagnostic = builder.into_diagnostic(format_args!( + "The members of `{}` would build different anonymous named tuples \ + from this value for attribute `{}`", + object_ty.display(db, env), + self.attribute, + )); + diagnostic.info( + "a plain tuple literal is built as the anonymous named tuple its \ + declaration expects, and one value reaches whichever member the \ + receiver is", + ); + } + } AssignmentAttributeWriteDiagnostic::CannotAssign => { if let Some(builder) = self .builder diff --git a/docs/basedpython/features/reified-class-generics.md b/docs/basedpython/features/reified-class-generics.md index 0d474a0fe6..25774332c5 100644 --- a/docs/basedpython/features/reified-class-generics.md +++ b/docs/basedpython/features/reified-class-generics.md @@ -127,6 +127,12 @@ class Box[T]: b: Box[int] = Box() # from the declaration Box[str]() # written out + +class Holder: + box: Box[int] + + def __init__(self): + self.box = Box() # from the attribute's declaration ``` where nothing solves it, the bare construction is an error diff --git a/ty.schema.json b/ty.schema.json index 020c4c8f73..8687669929 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -723,7 +723,7 @@ }, "ambiguous-conversion": { "title": "detects conversion sites served by more than one conversion", - "description": "## What it does\nChecks for conversion sites where more than one conversion applies — two\ndunders, a dunder and an in-scope conformance, or two applicable\n`implementation`s of the same interface and type.\n\n## Why is this bad?\n`__from__` and `__into__` are hand-written bodies that can disagree, so\nwhich one runs must not depend on arbitrary ordering. Remove one of them,\nor write the conversion you want explicitly.\n\n## Example\n\n```by\nclass Celsius:\n def __into__(self) -> Fahrenheit: ...\n\nclass Fahrenheit:\n @classmethod\n def __from__(cls, value: Celsius) -> Self: ...\n\nreport(Celsius()) # error: two conversions apply\n```", + "description": "## What it does\nChecks for conversion sites where more than one conversion applies — two\ndunders, a dunder and an in-scope conformance, or two applicable\n`implementation`s of the same interface and type. A plain tuple literal\nthat more than one expected anonymous named tuple accepts is one too:\n`(1,)` where `(a: int) | (b: int)` is expected.\n\n## Why is this bad?\n`__from__` and `__into__` are hand-written bodies that can disagree, so\nwhich one runs must not depend on arbitrary ordering. Remove one of them,\nor write the conversion you want explicitly. A plain tuple literal is\nconstructed as the anonymous named tuple it stands for, which decides the\nfield names the value has, and nothing in the literal says which one was\nmeant: name the fields, as in `(a=1)`.\n\n## Example\n\n```by\nclass Celsius:\n def __into__(self) -> Fahrenheit: ...\n\nclass Fahrenheit:\n @classmethod\n def __from__(cls, value: Celsius) -> Self: ...\n\nreport(Celsius()) # error: two conversions apply\n```", "default": "error", "oneOf": [ { From 943269311ab9033ed129b6f258c81202bf3f2231 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:17:15 +1000 Subject: [PATCH 29/47] leave a function to its interpreted definition in `by compile` when it has a plain tuple literal the checker reads as an anonymous named tuple, which the native lowering built as a plain tuple, so the callee's field access raised `AttributeError` --- crates/by_build/tests/differential.rs | 24 ++++++++++++++++++++++++ crates/by_irbuild/src/lib.rs | 13 +++++++++++++ crates/by_irbuild/src/tests.rs | 15 +++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index ee882b2d37..f8bb9a8e8e 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -44945,6 +44945,30 @@ def show() -> str: ); } +#[test] +fn a_plain_tuple_built_as_an_anonymous_named_tuple_agrees() { + // the checker reads each literal as an instance of the named tuple expected where it + // stands, and the fields are read by name: a plain tuple the lowering built has none + agree_with_declines( + "anonbuilt", + "\ +def take(p: (a: int, b: int)) -> int: + return p.a + p.b + + +def call() -> int: + return take((7, 8)) + + +def second() -> int: + items: list[(a: int, b: int)] = [] + items.append((1, 2)) + return items[0].b +", + &["m.call()", "m.second()"], + ); +} + #[test] fn a_checked_cast_agrees_on_the_value_it_refuses() { // the refusal is the point of the form, so the raise is part of the answer diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 80a1b65dfa..fc4f1e9e77 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -15876,6 +15876,19 @@ impl Lowering<'_, '_> { Expr::Attribute(node) => self.attribute(node), Expr::List(node) => self.display(&node.elts, Display::List), Expr::Set(node) => self.display(&node.elts, Display::Set), + // the checker reads a plain tuple standing where an anonymous named tuple is + // expected as an instance of that class, which the interpreted twin constructs. + // a tuple built here would have none of its fields + Expr::Tuple(node) + if !node.is_anon_named_tuple + && !node.is_anon_named_tuple_value + && self.model.anon_named_tuple_class(node).is_some() => + { + Err(Decline::new( + "a tuple literal standing where an anonymous named tuple is expected is \ + constructed as that class", + )) + } Expr::Tuple(node) => self.display(&node.elts, Display::Tuple), Expr::Dict(node) => self.dict_display(&node.items), Expr::Subscript(node) => { diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index 5c5da52c50..4e6a24ca83 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -12311,6 +12311,21 @@ def make() -> (name: str, age: int): assert!(reason.contains("named tuple"), "{reason}"); } +#[test] +fn a_plain_tuple_built_as_an_anonymous_named_tuple_is_declined() { + // the checker reads the literal as an instance of the named tuple expected where it + // stands, so its fields are read by name, and a plain tuple has none of them + let reason = decline( + "\ +def second() -> int: + items: list[(a: int, b: int)] = [] + items.append((1, 2)) + return items[0].b +", + ); + assert!(reason.contains("anonymous named tuple"), "{reason}"); +} + #[test] fn a_checked_cast_is_declined() { // `v cast! int` tests the value and raises when it does not hold; read as a call it From b50bb3f302af11715c5a570cb67cea1b802e952c Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:17:18 +1000 Subject: [PATCH 30/47] document that a plain tuple literal is constructed as the anonymous named tuple expected wherever it stands, how a union chooses one, and that a `.py` file keeps a plain tuple a tuple --- .../features/anonymous-named-tuple.md | 135 +++++++++++++----- 1 file changed, 98 insertions(+), 37 deletions(-) diff --git a/docs/basedpython/features/anonymous-named-tuple.md b/docs/basedpython/features/anonymous-named-tuple.md index 783be2d1cf..e5f1c317bc 100644 --- a/docs/basedpython/features/anonymous-named-tuple.md +++ b/docs/basedpython/features/anonymous-named-tuple.md @@ -55,52 +55,121 @@ duplicate named field (`(name=1, name=2)`) within a single tuple, all named fields must use the same separator; you cannot mix `:` and `=` in the same anonymous named tuple -## plain-tuple coercion +## plain tuples where one is expected -basedpython auto-wraps plain tuple literals as constructor calls when the -surrounding annotation says they should be anonymous named tuples. coercion -fires in three positions: +a plain tuple literal written where an anonymous named tuple is expected *is* one: it is +constructed as that class, so its fields are there at runtime -**return statements** inside a function whose return annotation is an -anonymous named tuple: +```by +def take(p: (name: str, age: int)) -> str: + return p.name + +take(("ada", 36)) +``` + +transpiles to: + +```python +def take(p: _AnonNamedTuple_xxx) -> str: + return p.name + +take(_AnonNamedTuple_xxx("ada", 36)) +``` + +the expectation can come from anywhere the type checker gives a literal its expected type, so +every one of these is constructed: + +```by +type P = (name: str, age: int) + +def make() -> P: + return ("ada", 36) # a return, through an alias or not + +def greet(p: P = ("bob", 1)): ... # a default +def each(*ps: P): ... + +greet(p=("cy", 2)) # a keyword argument +each(("dee", 3), ("eve", 4)) # each element of `*args` + +people: list[P] = [("fay", 5)] # an element of a collection literal +people.append(("gus", 6)) # an argument of a method call +by_name: dict[str, list[P]] = {"h": [("hal", 7)]} # at any depth + +class Team: + var lead: P = ("ivy", 8) # an attribute's default + +team = Team() +team.lead = ("jo", 9) # a write to an attribute, or to a property + # whose setter takes a `P` +``` + +a `yield` in a generator typed `Iterator[P]` is constructed the same way, and a field that is itself +an anonymous named tuple constructs its own literal: `((1, 2), "a")` passed as +`(point: (x: int, y: int), label: str)` builds both classes + +only a literal is constructed. a value that already has a type is checked as it is: a variable +holding a plain tuple is not an anonymous named tuple, and a named tuple passed where a plain +`tuple` is expected stays the named tuple it is. a literal whose length doesn't match the expected +shape is left alone, so the type checker reports the mismatch + +### several expected shapes + +when the expected type is a union, the literal is constructed as the one anonymous named tuple that +accepts it: one with as many fields as the literal has elements, whose field types accept them ```by -def f() -> (age: int, name: str): - return (1, "a") # transpiled to: return _AnonNamedTuple_xxx(1, "a") +a: (age: int) | None = (36,) # constructed as `(age: int)` +b: (age: int) | (name: str) = ("ada",) # constructed as `(name: str)` +``` -f().name # works at runtime +a literal that more than one of them accepts is an error (`ambiguous-conversion`): the class decides +which field names the value has, and a plain tuple doesn't say which one was meant. name the fields +with the value form instead + +```by +c: (age: int) | (year: int) = (36,) # error: [ambiguous-conversion] +d: (age: int) | (year: int) = (age=36) # fine ``` -**annotated assignments** whose annotation is an anonymous named tuple, with or -without a [declaration modifier](modifiers.md): +an anonymous named tuple whose fields don't accept the literal is passed over, and the literal is +checked against the rest of the expected type as a plain tuple: ```by -a: (name: str, age: int) = ("asdf", 1) -let b: (name: str, age: int) = ("asdf", 1) -# transpiled to: a: _AnonNamedTuple_xxx = _AnonNamedTuple_xxx("asdf", 1) +e: (a: list[int]) | tuple[list[object]] = ([1, "s"],) # a plain tuple ``` -**list/set literals** whose annotation is `list[anon-NT]`, `set[anon-NT]`, -or `frozenset[anon-NT]` — every plain tuple element is wrapped: +an anonymous named tuple is a subtype of the plain tuple of its field types, so a union with that +tuple *is* that tuple, and a literal there stays a plain tuple: ```by -a: list[(age: int, name: str)] = [(1, "a"), (2, "b")] -# transpiled to: -# a: list[_AnonNamedTuple_xxx] = [_AnonNamedTuple_xxx(1, "a"), _AnonNamedTuple_xxx(2, "b")] +f: (age: int) | tuple[int] = (36,) # the expected type is `tuple[int]`: a plain tuple ``` -if a tuple literal's arity doesn't match the annotation it's left alone, so -ty diagnoses the mismatch rather than the transpiler silently constructing -the wrong shape +for the same reason, writing a literal to an attribute of a union whose members would build +different anonymous named tuples from it is an error (`invalid-assignment`): no single value is +both. members that would each build the same one — `P` and `P | None` alike — build it, and a +value with no plain tuple literal in it is checked against each member's own declaration as usual -coercion reads what the annotation *means*, not how it is spelled, so all three -positions fire through an alias declared above them — both `P = (name: str)` and -`type P = (name: str)`: +### `.py` files + +a `.py` file is python, where a plain tuple is never a named tuple. passing one where a `.by` +module expects an anonymous named tuple is an error there (`invalid-argument-type`). construct the +value through a name the `.by` module binds the type to: ```by -P = (name: str, age: int) +# shapes.by +Point = (x: int, y: int) + +def norm(p: (x: int, y: int)) -> int: + return p.x * p.x + p.y * p.y +``` + +```python +# main.py +from shapes import Point, norm -a: P = ("asdf", 1) # transpiled to: a: P = _AnonNamedTuple_xxx("asdf", 1) +norm((3, 4)) # error: [invalid-argument-type] +norm(Point(3, 4)) # fine ``` ## semantics @@ -113,12 +182,8 @@ class _Anon(NamedTuple): age: int ``` -the type checker treats `(name: str, age: int)` as the equivalent -heterogeneous `tuple[str, int]` for assignability and inference. plain tuple -literals can therefore be assigned to or returned from positions annotated -with an anonymous named tuple — at runtime they remain plain tuples, so -field-name access (`x.name`) requires constructing through the synthesized -class explicitly +it is a subclass of the heterogeneous `tuple[str, int]`, so it can be used anywhere that tuple +type is expected ## structural identity @@ -140,7 +205,3 @@ ordered list of types - field defaults aren't yet supported in the surface syntax. if you need defaults, declare a `NamedTuple` class explicitly -- auto-coercion of plain tuple literals only fires on `return`, - annotated assignment, and `list/set/frozenset[anon-NT]` literal sites. - nested positions like `dict[K, anon-NT]` values or call arguments need - the explicit value form `(name=…)` instead From 0f5be3600ed1afb48ad366ee15a69a6acf992d53 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 10 Oct 2026 05:46:07 +1000 Subject: [PATCH 31/47] =?UTF-8?q?write=20a=20module-level=20name=20into=20?= =?UTF-8?q?code=20the=20transpiled=20program=20runs=20only=20where=20it=20?= =?UTF-8?q?holds=20there=20the=20binding=20the=20checker=20resolved:=20eve?= =?UTF-8?q?ry=20binding=20of=20it=20is=20made=20above=20the=20module=20sta?= =?UTF-8?q?tement=20the=20code=20is=20in,=20which=20for=20a=20function=20o?= =?UTF-8?q?r=20a=20method=20is=20the=20`def`=20or=20`class`=20statement=20?= =?UTF-8?q?holding=20it,=20since=20the=20function=20may=20be=20called=20as?= =?UTF-8?q?=20soon=20as=20that=20runs;=20none=20is=20made=20only=20for=20t?= =?UTF-8?q?he=20checker;=20and=20either=20the=20module=20binds=20it=20on?= =?UTF-8?q?=20every=20path=20to=20its=20end,=20with=20no=20`del`=20and=20n?= =?UTF-8?q?o=20binding=20made=20in=20one=20branch=20of=20a=20conditional?= =?UTF-8?q?=20expression=20unbinding=20it,=20or=20every=20binding=20of=20i?= =?UTF-8?q?t=20is=20one=20a=20statement=20at=20the=20top=20of=20the=20modu?= =?UTF-8?q?le=20always=20makes=20once=20it=20completes,=20which=20a=20modu?= =?UTF-8?q?le=20the=20checker=20finds=20never=20ends=20(one=20that=20stops?= =?UTF-8?q?=20with=20`sys.exit()`)=20needs.=20a=20binding=20is=20made=20on?= =?UTF-8?q?ly=20for=20the=20checker=20when=20a=20condition=20deciding=20wh?= =?UTF-8?q?ether=20it=20is=20made=20reads=20`typing.TYPE=5FCHECKING`=20?= =?UTF-8?q?=E2=80=94=20the=20test=20of=20an=20`if`,=20`elif`=20or=20`while?= =?UTF-8?q?`,=20the=20subject,=20patterns=20and=20guards=20of=20a=20`match?= =?UTF-8?q?`=20case,=20an=20expression=20around=20a=20walrus=20=E2=80=94?= =?UTF-8?q?=20or=20when=20its=20value=20reads=20it=20(`Foo=20=3D=20=5FF=20?= =?UTF-8?q?if=20TYPE=5FCHECKING=20else=20None`),=20whichever=20way=20the?= =?UTF-8?q?=20condition=20goes=20(`if=20not=20TYPE=5FCHECKING:`=20and=20`i?= =?UTF-8?q?f=20TYPE=5FCHECKING=20and=20ready:`=20count),=20with=20the=20co?= =?UTF-8?q?nstant=20found=20by=20resolving=20each=20name:=20imported=20und?= =?UTF-8?q?er=20any=20name,=20read=20off=20the=20`typing`=20module,=20held?= =?UTF-8?q?=20by=20an=20assignment,=20a=20walrus=20or=20an=20augmented=20a?= =?UTF-8?q?ssignment,=20or=20a=20module's=20own=20`TYPE=5FCHECKING`,=20whi?= =?UTF-8?q?ch=20the=20checker=20reads=20as=20`True`=20too;=20a=20name=20im?= =?UTF-8?q?ported=20from=20another=20module=20is=20made=20only=20for=20the?= =?UTF-8?q?=20checker=20when=20the=20binding=20it=20imports=20is.=20a=20sp?= =?UTF-8?q?ecialization=20the=20checker=20solved=20that=20only=20refines?= =?UTF-8?q?=20a=20construction=20is=20left=20out=20where=20its=20names=20d?= =?UTF-8?q?o=20not=20hold=20(`self.entries=20=3D=20set()`=20with=20`entrie?= =?UTF-8?q?s:=20set[Decimal]`=20stays=20`set()`=20rather=20than=20raising?= =?UTF-8?q?=20`NameError`),=20a=20soundness=20check=20against=20such=20a?= =?UTF-8?q?=20class=20is=20left=20out,=20a=20plain=20`cast`'s=20type=20is?= =?UTF-8?q?=20written=20as=20a=20string,=20and=20a=20reified=20type=20argu?= =?UTF-8?q?ment,=20an=20`is`=20test=20and=20a=20checked=20cast=20that=20wo?= =?UTF-8?q?uld=20need=20it=20are=20errors=20at=20check=20time=20(`unspecia?= =?UTF-8?q?lized-reified-generic`,=20`erased-type-check`);=20where=20the?= =?UTF-8?q?=20program=20binds=20the=20name=20some=20other=20way,=20in=20th?= =?UTF-8?q?e=20`else:`=20of=20a=20`TYPE=5FCHECKING`=20block,=20the=20error?= =?UTF-8?q?=20says=20the=20class=20it=20binds=20need=20not=20be=20the=20on?= =?UTF-8?q?e=20the=20checker=20resolved.=20`redundant-condition`=20no=20lo?= =?UTF-8?q?nger=20reports=20a=20test=20of=20`typing.TYPE=5FCHECKING`=20imp?= =?UTF-8?q?orted=20under=20another=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/transforms/ast_driver.rs | 5 +- .../src/transforms/auto_quote.rs | 45 +- .../src/transforms/checked_cast.rs | 33 +- .../src/transforms/type_reification.rs | 232 ++++++ crates/by_transforms/src/type_info.rs | 9 + .../tests/type_checking_import_runtime.rs | 315 +++++++ .../basedpython_type_checking_imports.md | 345 ++++++++ .../resources/mdtest/known_constants.md | 51 ++ .../resources/mdtest/redundant_condition.md | 23 +- crates/ty_python_semantic/src/place.rs | 787 +++++++++++++++++- .../ty_python_semantic/src/semantic_model.rs | 17 + .../src/types/infer/builder.rs | 64 +- .../src/types/infer/builder/conditions.rs | 5 + .../src/types/reified_infer.rs | 186 ++++- .../src/types/runtime_spelling.rs | 42 + .../ty_python_semantic/src/types/soundness.rs | 29 +- .../src/types/soundness_sites.rs | 23 +- docs/basedpython/features/checked-cast.md | 7 + .../features/parametric-type-tests.md | 34 + .../features/reified-class-generics.md | 9 +- docs/basedpython/features/soundness.md | 16 +- 21 files changed, 2152 insertions(+), 125 deletions(-) create mode 100644 crates/by_transforms/tests/type_checking_import_runtime.rs create mode 100644 crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index c65b276a23..5e9a7fd73b 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -1445,7 +1445,10 @@ pub(crate) fn run_against_source<'a>( let runtime_union_pass = runtime_union::RuntimeUnionPass::new(written, config.min_version); let generics_pass = generics::GenericPolyfillPass::new(source_ref, written, config.clone()); let soundness_pass = soundness::SoundnessPass::new(source_ref, written, config); - let checked_cast_pass = checked_cast::CheckedCastPass { written }; + let checked_cast_pass = checked_cast::CheckedCastPass { + source: source_ref, + written, + }; let module_api_pass = module_api::ModuleApiPass::new(source_ref); let trailing_lambda_pass = trailing_lambda::TrailingLambdaPass::new(source_ref); let if_let_pass = if_let::IfLetPass::new(source_ref); diff --git a/crates/by_transforms/src/transforms/auto_quote.rs b/crates/by_transforms/src/transforms/auto_quote.rs index f909b96dfa..d294d7445e 100644 --- a/crates/by_transforms/src/transforms/auto_quote.rs +++ b/crates/by_transforms/src/transforms/auto_quote.rs @@ -285,32 +285,33 @@ impl Quoter<'_> { } } - /// wrap `range` in quotes, as one template passing the source through so - /// the lowerings inside it land between the quotes. the delimiter is one the - /// source it wraps does not already use fn quote(&mut self, range: TextRange) { - let text = &self.source[range]; - let delimiter = ["\"", "'", "\"\"\"", "'''"] - .into_iter() - .find(|delimiter| { - !text.contains(delimiter) - && delimiter - .chars() - .next() - .is_none_or(|quote| !text.ends_with(quote)) - }) - .unwrap_or("\""); - self.edits.push(( - range, - vec![ - Fragment::Lit(delimiter.to_owned()), - Fragment::Src(range), - Fragment::Lit(delimiter.to_owned()), - ], - )); + self.edits.push((range, quoted(self.source, range))); } } +/// `range` of `source` wrapped in quotes, as one template passing the source through so +/// the lowerings inside it land between the quotes. the delimiter is one the source it +/// wraps does not already use +pub(crate) fn quoted(source: &str, range: TextRange) -> Vec { + let text = &source[range]; + let delimiter = ["\"", "'", "\"\"\"", "'''"] + .into_iter() + .find(|delimiter| { + !text.contains(delimiter) + && delimiter + .chars() + .next() + .is_none_or(|quote| !text.ends_with(quote)) + }) + .unwrap_or("\""); + vec![ + Fragment::Lit(delimiter.to_owned()), + Fragment::Src(range), + Fragment::Lit(delimiter.to_owned()), + ] +} + #[cfg(test)] mod tests { use crate::config::PythonVersion; diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 6f024cbf29..0b10c04e38 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -115,6 +115,7 @@ impl Helper { } struct CastLower<'a> { + source: &'a str, types: &'a dyn TypeInfo, written: WrittenNames<'a>, /// the name `typing.cast` is written under @@ -126,8 +127,9 @@ struct CastLower<'a> { } impl<'a> CastLower<'a> { - fn new(types: &'a dyn TypeInfo, written: WrittenNames<'a>) -> Self { + fn new(source: &'a str, types: &'a dyn TypeInfo, written: WrittenNames<'a>) -> Self { Self { + source, types, written, cast: written.imported("typing", "cast"), @@ -193,23 +195,27 @@ impl<'a> CastLower<'a> { let value_range = value_arg.range(); // unchecked `typing.cast(type, value)` keeps the exact written type (it // never reaches `isinstance`) and takes the type first + // + // `typing.cast` never reads its type, but python evaluates it all the same, so a + // type naming something that does not hold there the binding the checker resolved — + // a class imported only under `if TYPE_CHECKING:`, or declared further on — is + // written as a string, as python's own spelling of such a cast is let (helper, first, rest) = if helper == Helper::TypingCast { - ( - helper, - Fragment::Src(type_arg.range()), - vec![Fragment::Src(value_range)], - ) + let first = if self.types.reads_names_bound_where_written(type_arg) { + vec![Fragment::Src(type_arg.range())] + } else { + super::auto_quote::quoted(self.source, type_arg.range()) + }; + (helper, first, vec![Fragment::Src(value_range)]) } else { let (helper, rest) = self.target(value_arg, type_arg, helper); - (helper, Fragment::Src(value_range), rest) + (helper, vec![Fragment::Src(value_range)], rest) }; self.used.insert(helper); - let mut fragments = vec![ - Fragment::Lit(helper.open_call(&self.cast)), - first, - Fragment::Lit(", ".to_owned()), - ]; + let mut fragments = vec![Fragment::Lit(helper.open_call(&self.cast))]; + fragments.extend(first); + fragments.push(Fragment::Lit(", ".to_owned())); fragments.extend(rest); fragments.push(Fragment::Lit(")".to_owned())); self.edits.push((whole, fragments)); @@ -248,6 +254,7 @@ impl<'ast> Visitor<'ast> for CastLower<'_> { } pub(crate) struct CheckedCastPass<'src> { + pub(crate) source: &'src str, pub(crate) written: WrittenNames<'src>, } @@ -276,7 +283,7 @@ impl TypeAwarePass for CheckedCastPass<'_> { } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = CastLower::new(types, self.written); + let mut inner = CastLower::new(self.source, types, self.written); for stmt in stmts { inner.visit_stmt(stmt); } diff --git a/crates/by_transforms/src/transforms/type_reification.rs b/crates/by_transforms/src/transforms/type_reification.rs index 974f6becfd..f807e8fa96 100644 --- a/crates/by_transforms/src/transforms/type_reification.rs +++ b/crates/by_transforms/src/transforms/type_reification.rs @@ -653,6 +653,238 @@ mod tests { ); } + /// a class imported only under `if TYPE_CHECKING:` is a name the running program does + /// not have, so a specialization naming it is left out, as an annotated assignment and as + /// a write to a declared attribute + #[test] + fn a_class_imported_only_for_the_checker_is_left_out() { + let out = out(indoc! {" + from typing import TYPE_CHECKING + if TYPE_CHECKING: + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + + def __init__(self) -> None: + self.entries = set() + kept: set[Decimal] = set() + + totals: set[Decimal] = set() + "}); + assert!(!out.contains("set[Decimal]()"), "got: {out}"); + assert_eq!(out.matches("= set()").count(), 3, "got: {out}"); + } + + /// a block on the constant under another name, or read off the module, is a type-checking + /// block all the same, while a block on a condition that holds whenever the program runs + /// is not + #[test] + fn a_class_imported_under_an_aliased_type_checking_block_is_left_out() { + let out = out(indoc! {" + import typing as t + from typing import TYPE_CHECKING as TC + + CHECKING = TC + ALWAYS = True + + if CHECKING: + from decimal import Decimal + if t.TYPE_CHECKING: + from fractions import Fraction + if ALWAYS: + from ipaddress import IPv4Address + + class Ledger: + entries: set[Decimal] + fractions: set[Fraction] + addresses: set[IPv4Address] + + def __init__(self) -> None: + self.entries = set() + self.fractions = set() + self.addresses = set() + "}); + assert!(out.contains("self.entries = set()"), "got: {out}"); + assert!(out.contains("self.fractions = set()"), "got: {out}"); + assert!( + out.contains("self.addresses = set[IPv4Address]()"), + "got: {out}" + ); + } + + /// a condition that reads the constant in any way — here a module's own `TYPE_CHECKING`, + /// which the checker reads as `True` as it does `typing`'s, inside an `or` — may be one the + /// program does not take + #[test] + fn a_class_past_a_condition_that_reads_type_checking_is_left_out() { + let out = out(indoc! {" + import os + + TYPE_CHECKING = False + + if TYPE_CHECKING or os.environ.get('NEVER'): + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + + def __init__(self) -> None: + self.entries = set() + "}); + assert!(out.contains("self.entries = set()"), "got: {out}"); + } + + /// what the running program binds above the statement a read is in is spelled, and what + /// it binds only further down is not: a method's body runs when it is called, which can be + /// before the module reaches a class declared below it (`Ledger()` above `class Later`) + #[test] + fn a_class_declared_above_is_spelled_in_a_method_and_one_below_is_not() { + let out = out(indoc! {" + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + later: set[Later] + + def __init__(self) -> None: + self.entries = set() + self.later = set() + kept: set[Decimal] = set() + + class Later: ... + "}); + assert!(out.contains("self.entries = set[Decimal]()"), "got: {out}"); + assert!(out.contains("self.later = set()"), "got: {out}"); + assert!( + out.contains("kept: set[Decimal] = set[Decimal]()"), + "got: {out}" + ); + } + + /// the module's own statements run in order: one above a class's declaration runs before + /// the class exists, one below it after + #[test] + fn a_class_is_spelled_below_its_declaration_and_not_above() { + let out = out(indoc! {" + early: set[Later] = set() + + class Later: ... + + late: set[Later] = set() + "}); + assert!(out.contains("early: \"set[Later]\" = set()"), "got: {out}"); + assert!( + out.contains("late: set[Later] = set[Later]()"), + "got: {out}" + ); + } + + /// the checker resolves a module-level name from every binding the module makes, so a + /// statement that runs while an earlier binding of the name still holds — the module binds + /// it again further down — may read another class than the one the checker resolved + #[test] + fn a_name_bound_again_further_down_is_left_out_above_it() { + let out = out(indoc! {" + from decimal import Decimal + + kept: set[Decimal] = set() + + from decimal import Decimal + "}); + assert!(out.contains("kept: set[Decimal] = set()"), "got: {out}"); + } + + /// a class the module's global scope does not bind, as its own class, has no spelling at + /// all: a type parameter, a class local to a function, a class nested in another + #[test] + fn a_class_the_module_does_not_bind_is_left_out() { + let out = out(indoc! {" + class Box[T]: + items: set[T] + + def __init__(self): + self.items = set() + + def make(): + class Local: ... + kept: set[Local] = set() + return kept + + class Outer: + class Inner: ... + inners: set[Inner] = set() + "}); + assert!(out.contains("self.items = set()"), "got: {out}"); + assert!(out.contains("kept: set[Local] = set()"), "got: {out}"); + assert!(out.contains("inners: set[Inner] = set()"), "got: {out}"); + } + + /// a `del` unbinds a name by the end of the module, and a binding made in one branch of a + /// conditional expression leaves it unbound on the other + #[test] + fn a_name_the_module_may_have_unbound_is_left_out() { + let out = out(indoc! {" + import os + from decimal import Decimal + from fractions import Fraction as _F + + entries: set[Decimal] = set() + del Decimal + + _ = (Fraction := _F) if os.environ.get(\"UNSET\") else None + fractions: set[Fraction] = set() + "}); + assert!(!out.contains("set[Decimal]()"), "got: {out}"); + assert!(!out.contains("set[Fraction]()"), "got: {out}"); + assert_eq!(out.matches("= set()").count(), 2, "got: {out}"); + } + + /// a function that rebinds a module-level name through `global` can run at any time, so + /// the name may hold whatever it bound + #[test] + fn a_name_a_function_rebinds_through_global_is_left_out() { + let out = out(indoc! {" + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + + def __init__(self) -> None: + self.entries = set() + + def forget() -> None: + global Decimal + del Decimal + "}); + assert!(out.contains("self.entries = set()"), "got: {out}"); + } + + /// a module the checker finds never ends, here one that stops with `sys.exit()`, has no + /// end where every binding is made; a name only statements at its top bind, above the + /// read, is bound wherever the read runs all the same + #[test] + fn a_class_bound_above_is_spelled_in_a_module_that_never_ends() { + let out = out(indoc! {" + import sys + from decimal import Decimal + + class Ledger: + entries: set[Decimal] + + def __init__(self) -> None: + self.entries = set() + + totals: set[Decimal] = set() + sys.exit(0) + "}); + assert!(out.contains("self.entries = set[Decimal]()"), "got: {out}"); + assert!( + out.contains("totals: set[Decimal] = set[Decimal]()"), + "got: {out}" + ); + } + #[test] fn inert_below_39() { let out = out_at("xs = [1, 2]\n", PythonVersion::PY38); diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 16989db14f..da92e77cc4 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -473,6 +473,10 @@ pub(crate) trait TypeInfo { /// annotations`. `None` when ty did not index the name fn is_forward_reference(&self, name: &ExprName) -> Option; + /// whether every name `expr` reads holds, where it is written, the binding the checker + /// resolved when the program runs, so `expr` can be written into code that runs there + fn reads_names_bound_where_written(&self, expr: &Expr) -> bool; + /// how a bare name in a class body is emitted, when it names a member the /// class declares with a visibility keyword — `y = x + 1` after `private x = /// 1`. `None` for any other name @@ -1462,6 +1466,10 @@ impl TypeInfo for SemanticModel<'_> { ty_python_semantic::types::ide_support::repeated_underscore_lowering(self, function) } + fn reads_names_bound_where_written(&self, expr: &Expr) -> bool { + SemanticModel::reads_names_bound_where_written(self, expr) + } + fn cast_check_plan(&self, type_expr: &Expr) -> Option { let ty = type_expr.inferred_type(self)?; ty_python_semantic::types::soundness::cast_check_plan( @@ -1469,6 +1477,7 @@ impl TypeInfo for SemanticModel<'_> { &self.program_environment(), self.file(), ty, + Some(type_expr), ) } diff --git a/crates/by_transforms/tests/type_checking_import_runtime.rs b/crates/by_transforms/tests/type_checking_import_runtime.rs new file mode 100644 index 0000000000..15121a87a0 --- /dev/null +++ b/crates/by_transforms/tests/type_checking_import_runtime.rs @@ -0,0 +1,315 @@ +//! Runtime test for a class imported only under `if TYPE_CHECKING:`, or bound only further +//! down the module. +//! +//! The checker resolves such a name, and the running program does not have it where the +//! lowered code reads it — under `TYPE_CHECKING` that is how an import cycle is broken. A +//! lowering that writes a type into code that runs (a specialization the checker solved, a +//! soundness check's target, a cast's type) must not write that name, or the module raises +//! `NameError` with `by check` clean. + +mod common; + +use std::fs; +use std::path::Path; +use std::process::Command; + +use by_transforms::{Config, PythonVersion, SoundnessPositions, transpile, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; + +const PROGRAM: &str = r#" +from typing import TYPE_CHECKING +from decimal import Decimal as RuntimeDecimal + +if TYPE_CHECKING: + from decimal import Decimal + + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + kept: set[Decimal] = set() + assert kept == set() + + +def first(values: list[Decimal]) -> Decimal: + for value in values: + return value + return RuntimeDecimal(0) + + +def widen(value: Decimal) -> Decimal | int: + return value cast Decimal | int + + +totals: set[Decimal] = set() +ledger = Ledger() +ledger.entries = set() +assert ledger.entries == set() and totals == set() +assert first([RuntimeDecimal(1)]) == RuntimeDecimal(1) +assert widen(RuntimeDecimal(2)) == RuntimeDecimal(2) +print("ok") +"#; + +/// the same reads where the name is bound only under `TYPE_CHECKING` imported under another +/// name, only on a path the program does not take, or only further down the module +const ELSEWHERE_PROGRAM: &str = r#" +import os +from typing import TYPE_CHECKING as CHECKING +from decimal import Decimal as RuntimeDecimal + +if CHECKING: + from decimal import Decimal + +if os.environ.get("BASEDPYTHON_NEVER_SET"): + from fractions import Fraction + + +def first(values: list[Decimal]) -> Decimal: + for value in values: + return value + return RuntimeDecimal(0) + + +def first_fraction(values: list[Fraction]) -> object: + for value in values: + return value + return None + + +early: set[Later] = set() +seen: set[Fraction] = set() + + +class Later: ... + + +class Ledger: + entries: set[Decimal] + fractions: set[Fraction] + + def __init__(self): + self.entries = set() + self.fractions = set() + + +assert first([RuntimeDecimal(1)]) == RuntimeDecimal(1) +assert first_fraction([]) is None +assert early == set() and seen == set() +assert Ledger().entries == set() and Ledger().fractions == set() +print("ok") +"#; + +/// the same reads where the name is bound past a condition that reads `TYPE_CHECKING` in some +/// other way than as the whole test of an `if`, or where its value is chosen by the constant +const CONDITION_PROGRAM: &str = r#" +import typing +from typing import TYPE_CHECKING +from decimal import Decimal as RuntimeDecimal +from fractions import Fraction as RuntimeFraction + +match TYPE_CHECKING: + case True: + from decimal import Decimal as InMatch + case _: + pass + +if TYPE_CHECKING == True: + from decimal import Decimal as InComparison + +if bool(TYPE_CHECKING): + from decimal import Decimal as InCall + +if bool(typing.TYPE_CHECKING): + from decimal import Decimal as InAttribute + +if checking := TYPE_CHECKING: + from decimal import Decimal as InWalrus + +if checking: + from decimal import Decimal as InWalrusAlias + +while TYPE_CHECKING: + from decimal import Decimal as InWhile + break + +RUNNING = not TYPE_CHECKING +if not RUNNING: + from decimal import Decimal as InNegatedAlias + +combined = False +combined |= TYPE_CHECKING +if combined: + from decimal import Decimal as InAugmented + +Fraction = RuntimeFraction if TYPE_CHECKING else None + +_ = (InConditional := RuntimeDecimal) if TYPE_CHECKING else None + + +def parse(text: str) -> dynamic: + return RuntimeFraction(text) + + +class Ledger: + in_match: set[InMatch] + in_comparison: set[InComparison] + in_call: set[InCall] + in_attribute: set[InAttribute] + in_walrus: set[InWalrus] + in_walrus_alias: set[InWalrusAlias] + in_augmented: set[InAugmented] + in_while: set[InWhile] + in_negated_alias: set[InNegatedAlias] + in_conditional: set[InConditional] + + def __init__(self): + self.in_match = set() + self.in_comparison = set() + self.in_call = set() + self.in_attribute = set() + self.in_walrus = set() + self.in_walrus_alias = set() + self.in_augmented = set() + self.in_while = set() + self.in_negated_alias = set() + self.in_conditional = set() + in_ternary: Fraction = parse("1") + assert in_ternary == RuntimeFraction(1) + + +price: Fraction = parse("1/2") +assert price == RuntimeFraction(1, 2) +ledger = Ledger() +assert ledger.in_match == ledger.in_negated_alias == set() +print("ok") +"#; + +#[test] +fn a_class_imported_only_for_the_checker_is_never_evaluated() { + run(PROGRAM); +} + +#[test] +fn a_name_bound_elsewhere_than_where_it_is_read_is_never_evaluated() { + run(ELSEWHERE_PROGRAM); +} + +#[test] +fn a_name_bound_past_a_condition_reading_type_checking_is_never_evaluated() { + run(CONDITION_PROGRAM); +} + +/// a module importing a name another module binds one way for the checker and another way as +/// it runs reads what the program binds, which need not be the class the checker resolved +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_name_imported_from_a_type_checking_block_elsewhere_is_never_evaluated() { + let Some(python) = common::python() else { + eprintln!("skipping type checking import runtime test: no interpreter found"); + return; + }; + let files = [ + ( + "/kinds.by", + "\ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal +else: + Decimal = float + + +def parse(text: str) -> dynamic: + import decimal + + return decimal.Decimal(text) +", + ), + ( + "/main.by", + "\ +from kinds import Decimal, parse + + +def total() -> Decimal: + return parse(\"1.5\") + + +assert str(total()) == \"1.5\" +print(\"ok\") +", + ), + ]; + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("type_checking_import_elsewhere"); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + let config = Config { + min_version: PythonVersion::PY313, + soundness: SoundnessPositions::all(), + ..Config::default() + }; + for (path, _) in files { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = + transpile_typed(&db, file, &config, None).expect("transpile should succeed"); + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + let output = Command::new(&python) + .arg("main.py") + .current_dir(&dir) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- main.py ---\n{}", + String::from_utf8_lossy(&output.stderr), + fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run(program: &str) { + let Some(python) = common::python() else { + eprintln!("skipping type checking import runtime test: no interpreter found"); + return; + }; + let config = Config { + min_version: PythonVersion::PY313, + soundness: SoundnessPositions::all(), + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md b/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md new file mode 100644 index 0000000000..36b5e33b83 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md @@ -0,0 +1,345 @@ +# basedpython: a name that is not bound where the program reads it + +a basedpython lowering that writes a type into code that runs — a specialization the checker solved, +a soundness check, a type test — reads each name in it where that code runs. a binding made under +`if TYPE_CHECKING:` exists for the checker and never runs, which is how an import cycle is broken; a +binding made only on some paths, or only further down the module, is not there yet either. a +lowering leaves such a name out, and where the program needs the type at runtime to do what the +source asks, that is an error here rather than a `NameError` when the program runs + +## A specialization the program only refines is left out + +an empty `set()` declared as a `set[Decimal]` is a set whatever it is specialized as, so the +specialization the checker solved is left out rather than written with a name the program does not +bind + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +totals: set[Decimal] = set() +assert Ledger().entries == totals +``` + +## `TYPE_CHECKING` under another name + +the constant is the same constant whatever name it is imported under, and a block it guards is a +type-checking block all the same + +```by +from typing import TYPE_CHECKING as CHECKING + +if CHECKING: + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +## A block on any condition that reads `TYPE_CHECKING` + +the checker reads the constant as `True` and the program reads it as `False`, so a block whose +condition reads it may be taken by the checker and not by the program, whatever the condition does +with it. a binding made past such a condition is not one the program can be relied on to make, and +this does not ask which way the condition goes: `if not TYPE_CHECKING:` and +`if TYPE_CHECKING and ready:` count all the same + +### A `match` on the constant + +```by +from typing import TYPE_CHECKING + +match TYPE_CHECKING: + case True: + from decimal import Decimal + case _: + pass + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### A comparison with the constant + +```by +from typing import TYPE_CHECKING + +# error: [redundant-boolean-comparison] +if TYPE_CHECKING == True: + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### A call on the constant + +```by +from typing import TYPE_CHECKING + +if bool(TYPE_CHECKING): + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### The constant bound by a walrus + +```by +from typing import TYPE_CHECKING + +if checking := TYPE_CHECKING: + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### A `while` on the constant + +```by +from typing import TYPE_CHECKING + +while TYPE_CHECKING: + from decimal import Decimal + break + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +### A value chosen by the constant + +a conditional expression on the constant binds one value for the checker and another for the +program. here the checker reads `Decimal` as the class, so a value declared as one gets a soundness +check, but the program binds `None`, which the check could not test against, so it is left out + +```by +from typing import TYPE_CHECKING +from decimal import Decimal as RuntimeDecimal + +Decimal = RuntimeDecimal if TYPE_CHECKING else None + +def parse(text: str) -> dynamic: + return RuntimeDecimal(text) + +price: Decimal = parse("1.5") +assert price == RuntimeDecimal("1.5") +``` + +## An import made only on some paths + +a name imported under a condition the program decides as it runs may not be bound when it is read + +```by +import os + +if os.environ.get("BASEDPYTHON_NEVER_SET"): + from decimal import Decimal + +class Ledger: + entries: set[Decimal] + + def __init__(self): + self.entries = set() + +assert Ledger().entries == set() +``` + +## A class declared further down the module + +the module's own statements run in order, so a statement above a class's declaration runs before the +class exists. a method runs later, when the module has declared everything + +```by +early: set[Later] = set() + +class Later: ... + +class Ledger: + entries: set[Later] + + def __init__(self): + self.entries = set() + +assert early == Ledger().entries +``` + +## A name the module unbinds + +a `del` unbinds a name, so by the time a later statement runs the name may be gone + +```by +from decimal import Decimal + +class Ledger: + entries: set[Decimal] + +ledger = Ledger() +del Decimal +ledger.entries = set() +assert ledger.entries == set() +``` + +## A name bound in one branch of a conditional expression + +a binding inside a conditional expression is made only when that branch is taken + +```by +import os +from decimal import Decimal as RuntimeDecimal + +_ = (Decimal := RuntimeDecimal) if os.environ.get("BASEDPYTHON_NEVER_SET") else None +entries: set[Decimal] = set() +assert entries == set() +``` + +## An instance made before a class its declaration names + +a method runs when it is called, which can be before the module reaches a class it names: here +`Ledger()` runs above `class Later` + +```by +class Ledger: + later: set[Later] + + def __init__(self): + self.later = set() + +ledger = Ledger() + +class Later: ... + +assert ledger.later == set() +``` + +## A reified type argument the program does not bind + +a reified type parameter is read at runtime, so a construction or a call that would pass it a class +the program does not bind is reported + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +class Box[reified T]: + def kind(self) -> str: + return T.__name__ + +def kind_of[reified T](value: T) -> str: + return T.__name__ + +class Holder: + box: Box[Decimal] + + def __init__(self): + # error: [unspecialized-reified-generic] + self.box = Box() + +def describe(value: Decimal) -> str: + # error: [unspecialized-reified-generic] + return kind_of(value) +``` + +## A type test or checked cast against a class the program does not bind + +`is` and `cast!` test the value against the class at runtime, so they cannot test against one the +program does not bind + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +def test(value: object) -> bool: + # error: [erased-type-check] "`is Decimal` cannot be checked at runtime: `Decimal` is not bound when it runs" + return value is Decimal + +def check(value: object) -> Decimal: + # error: [erased-type-check] "`Decimal` cannot be checked at runtime: `Decimal` is not bound when it runs" + return value cast! Decimal +``` + +## A class the program binds differently from the checker + +a module may import a class for the checker and bind the name some other way as it runs, in the +`else:` of the same block. the checker reads the block as always taken, so the class it resolves is +the `TYPE_CHECKING` one, and the name the program binds need not be that class. a test against it +would answer for whatever the program bound, so it is refused + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal +else: + Decimal = float + +def test(value: object) -> bool: + # error: [erased-type-check] "`is Decimal` cannot be checked at runtime: the `Decimal` bound when it runs need not be the class the checker resolved" + return value is Decimal + +def check(value: object) -> Decimal: + # error: [erased-type-check] "`Decimal` cannot be checked at runtime: the `Decimal` bound when it runs need not be the class the checker resolved" + return value cast! Decimal +``` + +## An unchecked cast to a class the program does not bind + +a plain `cast` tests nothing, so it is fine: its type is written as a string, which python never +evaluates + +```by +from typing import TYPE_CHECKING +from decimal import Decimal as RuntimeDecimal + +if TYPE_CHECKING: + from decimal import Decimal + +def widen(value: Decimal) -> Decimal | int: + return value cast Decimal | int + +assert widen(RuntimeDecimal(1)) == RuntimeDecimal(1) +``` diff --git a/crates/ty_python_semantic/resources/mdtest/known_constants.md b/crates/ty_python_semantic/resources/mdtest/known_constants.md index fdd396940d..e2752433ce 100644 --- a/crates/ty_python_semantic/resources/mdtest/known_constants.md +++ b/crates/ty_python_semantic/resources/mdtest/known_constants.md @@ -26,6 +26,57 @@ from typing import TYPE_CHECKING as TC reveal_type(TC) # revealed: Literal[True] ``` +### A name assigned the constant, then something else + +A condition reads whatever the name holds where it is tested, so once the alias is reassigned the +branch is decided by the new value: + +```py +from typing import TYPE_CHECKING + +TC = TYPE_CHECKING +TC = False + +if TC: + y = 1 +else: + y = "a" + +reveal_type(y) # revealed: Literal["a"] +``` + +### An alias imported in a class body + +A class body's names are not visible from its methods, so a method testing the same name reads +something else, which nothing here binds: + +```py +class K: + from typing import TYPE_CHECKING as KC + + def m(self): + if KC: # error: [unresolved-reference] + c = 1 + else: + c = "a" + reveal_type(c) # revealed: Literal[1, "a"] +``` + +### An alias imported in an enclosing function + +```py +def outer(): + from typing import TYPE_CHECKING as TC + + def inner(): + if TC: + b = 1 + else: + b = "a" + reveal_type(b) # revealed: Literal[1] + return inner +``` + ### Qualified conditions Qualified references to `TYPE_CHECKING` are evaluated eagerly, allowing narrowing from the diff --git a/crates/ty_python_semantic/resources/mdtest/redundant_condition.md b/crates/ty_python_semantic/resources/mdtest/redundant_condition.md index c762d663ba..f057eb1a12 100644 --- a/crates/ty_python_semantic/resources/mdtest/redundant_condition.md +++ b/crates/ty_python_semantic/resources/mdtest/redundant_condition.md @@ -965,7 +965,7 @@ if BAR >= (3, 14): # no diagnostic reveal_type(TYPE_CHECKINGGGGG) # revealed: Literal[True] -if TYPE_CHECKINGGGGG: # error: [redundant-condition] "This condition is always true" +if TYPE_CHECKINGGGGG: # no diagnostic pass reveal_type(sys_platform) # revealed: Literal["linux"] @@ -1143,6 +1143,27 @@ class PlatformAttributeCycle: pass ``` +## A `TYPE_CHECKING` constant that is not `typing`'s + +ty reads any constant named `TYPE_CHECKING` as true, but only `typing.TYPE_CHECKING` (and its +`typing_extensions` re-export) is a fact about the environment the program is checked in. A module's +own constant of that name, imported under another name, is an ordinary condition: + +`settings.py`: + +```py +TYPE_CHECKING = False +``` + +`main.py`: + +```py +from settings import TYPE_CHECKING as DEBUGGING + +if DEBUGGING: # error: [redundant-condition] "This condition is always true" + pass +``` + ## Environment-dependent assignment guards An assignment can depend on nested conditions or aliases of environment guards. The assigned value diff --git a/crates/ty_python_semantic/src/place.rs b/crates/ty_python_semantic/src/place.rs index 988909202b..40c6fdb81f 100644 --- a/crates/ty_python_semantic/src/place.rs +++ b/crates/ty_python_semantic/src/place.rs @@ -3,6 +3,7 @@ pub(crate) mod definitions; use crate::ProgramEnvironment; use itertools::Either; use ruff_index::IndexSlice; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; use ruff_python_ast::{self as ast, PythonVersion}; use rustc_hash::FxHashMap; use ty_module_resolver::{ @@ -25,7 +26,7 @@ use crate::types::{ }; use crate::{Db, FxIndexSet, FxOrderSet}; use ruff_db::parsed::{ParsedModuleRef, parsed_module}; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextRange}; use ty_python_core::definition::{Definition, DefinitionKind, DefinitionState}; use ty_python_core::narrowing_constraints::ScopedNarrowingConstraint; use ty_python_core::place::{PlaceExpr, ScopedPlaceId}; @@ -33,7 +34,7 @@ use ty_python_core::predicate::{Predicate, ScopedPredicateId}; use ty_python_core::reachability_constraints::{ ReachabilityConstraints, ScopedReachabilityConstraintId, }; -use ty_python_core::scope::ScopeId; +use ty_python_core::scope::{FileScopeId, ScopeId}; use ty_python_core::{ BindingWithConstraints, BindingWithConstraintsIterator, BoundnessAnalysis, DeclarationWithConstraint, DeclarationsIterator, ProgramFile, SemanticIndex, Truthiness, @@ -545,6 +546,648 @@ pub(crate) fn global_symbol<'db>( }) } +/// basedpython: what a name the transpiled program reads holds when the program runs +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeBinding { + /// on every path to the read, a binding the program runs holds the name, and it is the + /// binding the checker resolved + Bound, + /// on some path to the read nothing the program runs has bound the name: it is bound only + /// under `if TYPE_CHECKING:`, only on some paths, or only further on + Unbound, + /// the program binds the name, but a binding the checker resolves never runs — it is made + /// under `if TYPE_CHECKING:` — so the value read need not be what the checker resolved + NotTheCheckers, +} + +impl RuntimeBinding { + /// why a runtime check naming `name` cannot run here, for a diagnostic's message + pub(crate) fn why_not_checkable(self, name: &str) -> String { + match self { + RuntimeBinding::Bound => format!("`{name}` is the class the checker resolved"), + RuntimeBinding::Unbound => format!("`{name}` is not bound when it runs"), + RuntimeBinding::NotTheCheckers => format!( + "the `{name}` bound when it runs need not be the class the checker resolved" + ), + } + } + + /// why `name`, read where a lowering would write it, cannot be read there + pub(crate) fn describe(self, name: &str) -> String { + match self { + RuntimeBinding::Bound => format!("`{name}` holds the binding the checker resolved"), + RuntimeBinding::Unbound => format!( + "`{name}` is not bound when this runs: it is bound only under \ + `if TYPE_CHECKING:`, only on some paths, or further on" + ), + RuntimeBinding::NotTheCheckers => format!( + "the `{name}` bound when this runs need not be the class the checker resolved: \ + the checker reads a binding of `{name}` made under `if TYPE_CHECKING:`, which \ + never runs" + ), + } + } +} + +/// a name the transpiled program reads, and where it reads it +#[derive(Clone, Copy)] +pub(crate) enum RuntimeRead<'a> { + /// a name the source writes, read where it is written + Written(ast::ExprRef<'a>), + /// a module-level name a lowering writes into code at `range` of `scope` + Global { name: &'a str, range: TextRange }, +} + +/// basedpython: what `read` holds when the transpiled program runs — the one question every +/// lowering that writes a name into code that runs asks first. +/// +/// a name the source writes is read where it is written, and holds what ty resolves there. a +/// module-level name a lowering writes is read inside one of the module's own statements, +/// which the program runs in order; that statement may be a `def` or a `class` whose body runs +/// later, but which code runs it, and when, is not known here, so the name is asked about at +/// the point the statement itself runs. it is bound there if every binding of it is made above +/// that statement, and either, by the end of the module, every path has bound it and none has +/// unbound it — a `del`, a binding made only in one branch of a conditional expression — or +/// every binding of it is one a statement at the top of the module always makes once it +/// completes, which a module the checker finds never ends, one that stops with `sys.exit()` +/// say, needs. a binding made under `if TYPE_CHECKING:` never runs, which is how an import +/// cycle is broken +pub(crate) fn runtime_binding<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: ProgramFile<'db>, + read: RuntimeRead<'_>, +) -> RuntimeBinding { + let index = semantic_index(db, file); + let module = parsed_module(db, file.python_file(db)).load(db); + match read { + RuntimeRead::Written(expr) => written_runtime_binding(db, env, file, index, expr), + RuntimeRead::Global { name, range, .. } => { + let global = global_scope(db, file); + let Some(symbol) = place_table(db, global).symbol_id(name) else { + return RuntimeBinding::Unbound; + }; + let Some(statement) = module + .suite() + .iter() + .find(|statement| statement.range().contains_range(range)) + else { + return RuntimeBinding::Unbound; + }; + let use_def = use_def_map(db, global); + match tally_bindings(db, index, use_def.end_of_scope_symbol_bindings(symbol)) + .runtime_binding() + { + RuntimeBinding::Bound => {} + _ if made_by_completed_statements(db, file, &module, name) => {} + other => return other, + } + let bindings = use_def.reachable_symbol_bindings(symbol); + let predicates = bindings.predicates(); + let reachability_constraints = bindings.reachability_constraints(); + let above = bindings.into_iter().all(|binding| { + let DefinitionState::Defined(definition) = binding.binding else { + return true; + }; + evaluate_reachability_with_cache( + db, + None, + reachability_constraints, + predicates, + binding.reachability_constraint, + ) + .is_always_false() + || definition.full_range(db, &module).range().end() <= statement.start() + }); + if above { + RuntimeBinding::Bound + } else { + RuntimeBinding::Unbound + } + } + } +} + +/// whether every binding `file` makes of the module-level `name` is made by a statement at the +/// top of the module that always makes it once it completes — an import, a `class`, a `def`, +/// an assignment — rather than one inside an `if`, a `try` or an expression. the module runs +/// those statements in order, and one that raises stops it, so whatever runs below such a +/// binding runs after it was made +fn made_by_completed_statements<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + module: &ParsedModuleRef, + name: &str, +) -> bool { + bindings_alone(db, file, name).is_some_and(|bindings| { + !bindings.is_empty() + && bindings.into_iter().all(|definition| { + let range = definition.full_range(db, module).range(); + let Some(statement) = module + .suite() + .iter() + .find(|statement| statement.range().contains_range(range)) + else { + return false; + }; + matches!( + (definition.kind(db), statement), + (DefinitionKind::Import(_), ast::Stmt::Import(_)) + | (DefinitionKind::ImportFrom(_), ast::Stmt::ImportFrom(_)) + | (DefinitionKind::Class(_), ast::Stmt::ClassDef(_)) + | (DefinitionKind::Function(_), ast::Stmt::FunctionDef(_)) + | (DefinitionKind::TypeAlias(_), ast::Stmt::TypeAlias(_)) + | (DefinitionKind::Assignment(_), ast::Stmt::Assign(_)) + | ( + DefinitionKind::AnnotatedAssignment(_), + ast::Stmt::AnnAssign(_) + ) + ) + }) + }) +} + +/// every binding `file`'s global scope makes of the module-level `name`, reached by the checker +/// or not, when nothing else changes what the name holds: none is made only for the checker, +/// nothing unbinds the name, and no other scope declares it `global` +fn bindings_alone<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + name: &str, +) -> Option>> { + let global = global_scope(db, file); + let symbol = place_table(db, global).symbol_id(name)?; + if declared_global_elsewhere(db, file, name) { + return None; + } + let use_def = use_def_map(db, global); + let place = ScopedPlaceId::Symbol(symbol); + if use_def.deletions().any(|(_, deleted)| deleted == place) { + return None; + } + let module = parsed_module(db, file.python_file(db)).load(db); + let in_stub = file.file(db).is_stub(db); + let mut bindings = Vec::new(); + for (_, definition, _) in use_def.definitions_with_usage() { + if definition.place(db) != place + || !definition.kind(db).category(in_stub, &module).is_binding() + { + continue; + } + if made_only_for_the_checker(db, definition) { + return None; + } + bindings.push(definition); + } + Some(bindings) +} + +/// whether a scope of `file` other than its global one declares `name` `global`, which is how +/// a function rebinds a module-level name. the module's own bindings of it do not show that +fn declared_global_elsewhere(db: &dyn Db, file: ProgramFile<'_>, name: &str) -> bool { + let global = global_scope(db, file).file_scope_id(db); + semantic_index(db, file).scope_ids().any(|scope| { + let table = place_table(db, scope); + scope.file_scope_id(db) != global + && table + .symbol_id(name) + .is_some_and(|id| table.symbol(id).is_global()) + }) +} + +/// [`runtime_binding`] of a name the source writes, resolved where it is written +fn written_runtime_binding<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: ProgramFile<'db>, + index: &'db SemanticIndex<'db>, + expr: ast::ExprRef<'_>, +) -> RuntimeBinding { + // a name the index never saw (inside a string annotation, or a word basedpython reads + // as syntax) is not one the program reads as a name + let (Some(scope), Some(_)) = (index.try_expression_scope_id(&expr), index.try_use_id(expr)) + else { + return RuntimeBinding::Bound; + }; + let scope = scope.to_scope_id(db, file); + let Some(place) = PlaceExpr::try_from_expr(expr) else { + return RuntimeBinding::Bound; + }; + let mut resolution = + resolve_place_load(db, index, scope, place, PlaceLoadMode::AtExpression(expr)); + loop { + match resolution.next() { + Some(PlaceLoadResolutionStep::Source(source)) => { + let tally = runtime_tally(db, env, index, source); + if !tally.is_empty() { + return tally.runtime_binding(); + } + } + Some(PlaceLoadResolutionStep::MemberResolutionCondition(_)) => {} + Some(PlaceLoadResolutionStep::Exhausted(_)) | None => break, + } + } + // nothing binds it on the way here: either the program binds it only further on, or + // no binding in the program supplies it — a name basedpython resolves itself, which the + // transpiler makes available + match is_forward_reference(db, env, file, expr) { + Some(true) => RuntimeBinding::Unbound, + Some(false) | None => RuntimeBinding::Bound, + } +} + +/// basedpython: whether `definition` may be made only for the checker: the checker reads +/// `typing.TYPE_CHECKING` as `True` and the program reads it as `False`, so a binding whose +/// making, or whose value, depends on the constant need not be one the program makes. +/// +/// a binding depends on the constant when a condition deciding whether it is made reads it — +/// the test of an enclosing `if`, `elif` or `while` and those before it, the subject, patterns +/// and guards of an enclosing `match` case and those before it, or an expression around the +/// binding such as a conditional expression or `and` holding a walrus — +/// or when its value reads it, as `Foo = _F if TYPE_CHECKING else None` does. a condition +/// reads the constant when any name in it resolves to a binding that carries it: the +/// constant's declaration, imported under any name, or a binding whose value reads it. this +/// does not ask which way the condition goes: `if not TYPE_CHECKING:` counts, and so does +/// `if TYPE_CHECKING and x:`, whose body the program may never run either +#[salsa::tracked(returns(copy), cycle_initial=|_, _, _| false, heap_size=ruff_memory_usage::heap_size)] +pub(crate) fn made_only_for_the_checker<'db>(db: &'db dyn Db, definition: Definition<'db>) -> bool { + let file = definition.program_file(db); + let index = semantic_index(db, file); + let module = parsed_module(db, file.python_file(db)).load(db); + let range = definition.full_range(db, &module).range(); + if index.is_in_type_checking_block(definition.file_scope(db), range) { + return true; + } + let reader = TypeCheckingReader { db, file }; + statements_check_only(&reader, module.suite(), range) + || carries_type_checking(db, definition) + || match definition.kind(db) { + // a binding imported from another module is made only for the checker when the + // binding it imports is + DefinitionKind::Import(_) + | DefinitionKind::ImportFrom(_) + | DefinitionKind::StarImport(_) => { + resolved_definitions(db, definition).any(|resolved| match resolved { + ResolvedTarget::Definition(resolved) => { + resolved != definition && made_only_for_the_checker(db, resolved) + } + ResolvedTarget::Module(_) => false, + }) + } + _ => false, + } +} + +/// whether `range`, inside one of `body`'s statements, is reached only past a condition that +/// reads the `TYPE_CHECKING` constant +fn statements_check_only( + reader: &TypeCheckingReader<'_>, + body: &[ast::Stmt], + range: TextRange, +) -> bool { + let Some(statement) = body + .iter() + .find(|statement| statement.range().contains_range(range)) + else { + return false; + }; + let holds = |body: &[ast::Stmt]| body.iter().any(|s| s.range().contains_range(range)); + // the conditions decided before `range` is reached, and the body holding it, if any + let (guards, inner): (Vec, Option<&[ast::Stmt]>) = match statement { + ast::Stmt::If(node) => { + let clauses = std::iter::once((Some(&*node.test), &node.body[..], node.test.start())) + .chain( + node.elif_else_clauses + .iter() + .map(|clause| (clause.test.as_ref(), &clause.body[..], clause.start())), + ) + .collect::>(); + let taken = clauses + .iter() + .rposition(|(_, _, start)| *start <= range.start()) + .unwrap_or_default(); + let guards = clauses[..=taken] + .iter() + .filter_map(|(test, _, _)| test.map(Guard::Expr)) + .collect(); + let body = clauses[taken].1; + (guards, holds(body).then_some(body)) + } + ast::Stmt::While(node) => ( + vec![Guard::Expr(&node.test)], + [&node.body[..], &node.orelse[..]] + .into_iter() + .find(|body| holds(body)), + ), + // the checker never takes the body of a `for` or an `except` clause to run for + // certain, so a binding made only there is one it already finds possibly unbound + ast::Stmt::For(node) => ( + Vec::new(), + [&node.body[..], &node.orelse[..]] + .into_iter() + .find(|body| holds(body)), + ), + ast::Stmt::Try(node) => ( + Vec::new(), + std::iter::once(&node.body[..]) + .chain(node.handlers.iter().map(|handler| match handler { + ast::ExceptHandler::ExceptHandler(handler) => &handler.body[..], + })) + .chain([&node.orelse[..], &node.finalbody[..]]) + .find(|body| holds(body)), + ), + ast::Stmt::Match(node) => { + // a case is tried only once the subject is evaluated and every case before it + // has failed to match + let taken = node + .cases + .iter() + .position(|case| case.range().contains_range(range)); + let tried = taken.map_or(0, |taken| taken + 1); + let guards = std::iter::once(Guard::Expr(&node.subject)) + .chain(node.cases[..tried].iter().flat_map(|case| { + std::iter::once(Guard::Pattern(&case.pattern)) + .chain(case.guard.as_deref().map(Guard::Expr)) + })) + .collect(); + let body = taken.map(|taken| &node.cases[taken].body[..]); + (guards, body.filter(|body| holds(body))) + } + ast::Stmt::With(node) => (Vec::new(), holds(&node.body).then_some(&node.body[..])), + ast::Stmt::FunctionDef(node) => (Vec::new(), holds(&node.body).then_some(&node.body[..])), + ast::Stmt::ClassDef(node) => (Vec::new(), holds(&node.body).then_some(&node.body[..])), + _ => (Vec::new(), None), + }; + if guards.into_iter().any(|guard| reader.reads(guard)) { + return true; + } + match inner { + Some(body) => statements_check_only(reader, body, range), + // the binding is made in the statement itself, by an expression it may stand inside + None => { + let mut around = AroundBinding { + reader, + range, + found: false, + }; + source_order::walk_stmt(&mut around, statement); + around.found + } + } +} + +/// a condition deciding whether a binding is made +#[derive(Clone, Copy)] +enum Guard<'a> { + Expr(&'a ast::Expr), + Pattern(&'a ast::Pattern), +} + +/// finds an expression standing around a binding's range that reads the `TYPE_CHECKING` +/// constant, without entering the statement's nested bodies +struct AroundBinding<'r, 'db> { + reader: &'r TypeCheckingReader<'db>, + range: TextRange, + found: bool, +} + +impl<'a> SourceOrderVisitor<'a> for AroundBinding<'_, '_> { + fn visit_body(&mut self, _body: &'a [ast::Stmt]) {} + + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if self.found || !expr.range().contains_range(self.range) || expr.range() == self.range { + return; + } + if self.reader.reads(Guard::Expr(expr)) { + self.found = true; + } else { + source_order::walk_expr(self, expr); + } + } +} + +/// answers whether an expression of `file` reads the `TYPE_CHECKING` constant +struct TypeCheckingReader<'db> { + db: &'db dyn Db, + file: ProgramFile<'db>, +} + +impl<'db> TypeCheckingReader<'db> { + fn reads(&self, guard: Guard<'_>) -> bool { + let mut visitor = ReadsTypeChecking { + reader: self, + found: false, + }; + match guard { + Guard::Expr(expr) => visitor.visit_expr(expr), + Guard::Pattern(pattern) => visitor.visit_pattern(pattern), + } + visitor.found + } + + /// the bindings `expr` reads: those of a name that reach it, or the module members an + /// attribute of a module names + fn definitions(&self, expr: &ast::Expr) -> Vec> { + match expr { + ast::Expr::Name(_) => self.reaching_definitions(expr), + ast::Expr::Attribute(attribute) => self + .definitions(&attribute.value) + .into_iter() + .flat_map(|definition| resolved_definitions(self.db, definition)) + .collect::>() + .into_iter() + .flat_map(|module| match module { + ResolvedTarget::Module(module) => { + crate::types::definition_resolution::find_symbol_in_scope( + self.db, + global_scope(self.db, module), + attribute.attr.as_str(), + ) + .into_iter() + .collect() + } + ResolvedTarget::Definition(_) => Vec::new(), + }) + .collect(), + _ => Vec::new(), + } + } + + /// the bindings of a name the program reads that reach it + fn reaching_definitions(&self, expr: &ast::Expr) -> Vec> { + let db = self.db; + let index = semantic_index(db, self.file); + let expr = ast::ExprRef::from(expr); + let (Some(scope), Some(_)) = (index.try_expression_scope_id(&expr), index.try_use_id(expr)) + else { + return Vec::new(); + }; + let Some(place) = PlaceExpr::try_from_expr(expr) else { + return Vec::new(); + }; + let mut resolution = resolve_place_load( + db, + index, + scope.to_scope_id(db, self.file), + place, + PlaceLoadMode::AtExpression(expr), + ); + loop { + match resolution.next() { + Some(PlaceLoadResolutionStep::Source(PlaceLoadSource { + kind: PlaceLoadSourceKind::Bindings(bindings), + .. + })) => { + let predicates = bindings.predicates(); + let reachability_constraints = bindings.reachability_constraints(); + let definitions: Vec> = bindings + .filter(|binding| { + !evaluate_reachability_with_cache( + db, + None, + reachability_constraints, + predicates, + binding.reachability_constraint, + ) + .is_always_false() + }) + .filter_map(|binding| match binding.binding { + DefinitionState::Defined(definition) => Some(definition), + DefinitionState::Undefined | DefinitionState::Deleted => None, + }) + .collect(); + if !definitions.is_empty() { + return definitions; + } + } + Some( + PlaceLoadResolutionStep::Source(_) + | PlaceLoadResolutionStep::MemberResolutionCondition(_), + ) => {} + Some(PlaceLoadResolutionStep::Exhausted(_)) | None => return Vec::new(), + } + } + } +} + +struct ReadsTypeChecking<'r, 'db> { + reader: &'r TypeCheckingReader<'db>, + found: bool, +} + +impl<'a> SourceOrderVisitor<'a> for ReadsTypeChecking<'_, '_> { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if self.found { + return; + } + if matches!(expr, ast::Expr::Name(_) | ast::Expr::Attribute(_)) + && self + .reader + .definitions(expr) + .into_iter() + .any(|definition| carries_type_checking(self.reader.db, definition)) + { + self.found = true; + return; + } + source_order::walk_expr(self, expr); + } +} + +/// what an import resolves to: a binding in the module it imports from, or a module +enum ResolvedTarget<'db> { + Definition(Definition<'db>), + Module(ProgramFile<'db>), +} + +/// `definition`, or the bindings an import resolves to, through every alias +fn resolved_definitions<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> impl Iterator> + 'db { + let env = ProgramEnvironment::from_file(definition.program_file(db)); + crate::types::definition_resolution::resolve_definition( + db, + &env, + definition, + None, + crate::types::definition_resolution::ImportAliasResolution::ResolveAliases, + ) + .into_iter() + .filter_map(|resolved| match resolved { + crate::types::definition_resolution::ResolvedDefinition::Definition(definition) => { + Some(ResolvedTarget::Definition(definition)) + } + crate::types::definition_resolution::ResolvedDefinition::Module(module) => { + Some(ResolvedTarget::Module(module)) + } + crate::types::definition_resolution::ResolvedDefinition::FileWithRange(_) => None, + }) +} + +/// whether the value `definition` binds carries the `TYPE_CHECKING` constant: it is the +/// constant's declaration, reached through any import of it, or a binding computed from an +/// expression that reads the constant. +/// +/// ty reads every binding named `TYPE_CHECKING` as `True`, the constant's own declaration in +/// `typing` among them, and a module that cannot import `typing` declares its own as `False` +#[salsa::tracked(returns(copy), cycle_initial=|_, _, _| false, heap_size=ruff_memory_usage::heap_size)] +fn carries_type_checking<'db>(db: &'db dyn Db, definition: Definition<'db>) -> bool { + resolved_definitions(db, definition).any(|resolved| { + let ResolvedTarget::Definition(resolved) = resolved else { + return false; + }; + let file = resolved.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + let reader = TypeCheckingReader { db, file }; + let reads = |expr: &ast::Expr| reader.reads(Guard::Expr(expr)); + let names_the_constant = |target: &ast::Expr| { + target + .as_name_expr() + .is_some_and(|name| name.id == "TYPE_CHECKING") + }; + match resolved.kind(db) { + DefinitionKind::Assignment(assignment) => { + names_the_constant(assignment.target(&module)) || reads(assignment.value(&module)) + } + DefinitionKind::AnnotatedAssignment(assignment) => { + names_the_constant(assignment.target(&module)) + || assignment.value(&module).is_some_and(reads) + } + DefinitionKind::NamedExpression(named) => reads(&named.node(&module).value), + DefinitionKind::AugmentedAssignment(assignment) => { + reads(&assignment.node(&module).value) + } + _ => false, + } + }) +} + +/// basedpython: whether `definition` is the declaration of `typing.TYPE_CHECKING` (or of +/// `typing_extensions.TYPE_CHECKING`, which re-exports it) +pub(crate) fn declares_typing_type_checking<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> bool { + let file = definition.program_file(db); + let is_typing = file_to_module(db, file.resolver_file(db)).is_some_and(|module| { + module.is_known(db, KnownModule::Typing) + || module.is_known(db, KnownModule::TypingExtensions) + }); + if !is_typing { + return false; + } + let module = parsed_module(db, file.python_file(db)).load(db); + let target = match definition.kind(db) { + DefinitionKind::Assignment(assignment) => assignment.target(&module), + DefinitionKind::AnnotatedAssignment(assignment) => assignment.target(&module), + _ => return false, + }; + target + .as_name_expr() + .is_some_and(|name| name.id == "TYPE_CHECKING") +} + /// Infers the public type of an imported symbol. /// /// If `requires_explicit_reexport` is [`None`], it will be inferred from the file's source type. @@ -1781,7 +2424,6 @@ pub(crate) fn is_forward_reference<'db>( let index = semantic_index(db, file); let scope = index.try_expression_scope_id(&expr)?.to_scope_id(db, file); index.try_use_id(expr)?; - let module = parsed_module(db, file.python_file(db)).load(db); let mut resolution = resolve_place_load( db, @@ -1793,7 +2435,7 @@ pub(crate) fn is_forward_reference<'db>( let bound_where_written = loop { match resolution.next() { Some(PlaceLoadResolutionStep::Source(source)) => { - match runtime_definedness(db, env, index, &module, source) { + match runtime_definedness(db, env, index, source) { RuntimeDefinedness::Defined => break true, RuntimeDefinedness::PossiblyUndefined => break false, RuntimeDefinedness::Undefined => {} @@ -1856,6 +2498,39 @@ fn lexical_binding_exists<'db>(db: &'db dyn Db, source: PlaceLoadSource<'db>) -> } } +/// what the bindings of one source of a place load hold once the program is running +#[derive(Default, Clone, Copy)] +struct BindingTally { + /// a binding the program runs holds a value on some path + runs: bool, + /// on some path nothing holds a value + unbound: bool, + /// a binding the checker sees is made under `if TYPE_CHECKING:`, and never runs + checker_only: bool, +} + +impl BindingTally { + fn is_empty(self) -> bool { + !self.runs && !self.checker_only + } + + fn definedness(self) -> RuntimeDefinedness { + match (self.runs, self.unbound || self.checker_only) { + (true, false) => RuntimeDefinedness::Defined, + (true, true) => RuntimeDefinedness::PossiblyUndefined, + (false, _) => RuntimeDefinedness::Undefined, + } + } + + fn runtime_binding(self) -> RuntimeBinding { + match (self.runs, self.unbound, self.checker_only) { + (true, false, false) => RuntimeBinding::Bound, + (true, _, true) => RuntimeBinding::NotTheCheckers, + _ => RuntimeBinding::Unbound, + } + } +} + /// Whether one source of a place load holds a value once the program is running. enum RuntimeDefinedness { Defined, @@ -1869,33 +2544,33 @@ fn runtime_definedness<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, index: &'db SemanticIndex<'db>, - module: &ParsedModuleRef, source: PlaceLoadSource<'db>, ) -> RuntimeDefinedness { + runtime_tally(db, env, index, source).definedness() +} + +fn runtime_tally<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + index: &'db SemanticIndex<'db>, + source: PlaceLoadSource<'db>, +) -> BindingTally { match source.kind { - PlaceLoadSourceKind::Bindings(bindings) => { - bindings_runtime_definedness(db, index, module, bindings) - } + PlaceLoadSourceKind::Bindings(bindings) => tally_bindings(db, index, bindings), // a lazy scope reads the owner's value when it runs, by which point the owner has made // every binding it is going to PlaceLoadSourceKind::DefinitionsFromOwningScope { scope, id } => { - bindings_runtime_definedness( - db, - index, - module, - use_def_map(db, scope).end_of_scope_bindings(id), - ) + tally_bindings(db, index, use_def_map(db, scope).end_of_scope_bindings(id)) } PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::ExplicitGlobalSymbol { file, name }) => { let global = global_scope(db, file); match place_table(db, global).symbol_id(&name) { - Some(symbol) => bindings_runtime_definedness( + Some(symbol) => tally_bindings( db, index, - module, use_def_map(db, global).end_of_scope_symbol_bindings(symbol), ), - None => RuntimeDefinedness::Undefined, + None => BindingTally::default(), } } PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::Builtin(name)) => { @@ -1908,30 +2583,30 @@ fn runtime_definedness<'db>( defined_if(module_type_implicit_global_symbol(db, file, &name)) } PlaceLoadSourceKind::Observed(_) - | PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::DunderClass(_)) => { - RuntimeDefinedness::Defined - } + | PlaceLoadSourceKind::Implicit(ImplicitPlaceLoad::DunderClass(_)) => BindingTally { + runs: true, + ..BindingTally::default() + }, } } /// an implicit source holds a value only where its lookup finds one -fn defined_if(place: PlaceAndQualifiers<'_>) -> RuntimeDefinedness { - if place.place.is_definitely_bound() { - RuntimeDefinedness::Defined - } else { - RuntimeDefinedness::Undefined +fn defined_if(place: PlaceAndQualifiers<'_>) -> BindingTally { + BindingTally { + runs: place.place.is_definitely_bound(), + ..BindingTally::default() } } -fn bindings_runtime_definedness<'db>( +fn tally_bindings<'db>( db: &'db dyn Db, index: &'db SemanticIndex<'db>, - module: &ParsedModuleRef, bindings: BindingWithConstraintsIterator<'_, 'db>, -) -> RuntimeDefinedness { +) -> BindingTally { let predicates = bindings.predicates(); let reachability_constraints = bindings.reachability_constraints(); - let (mut defined, mut undefined) = (false, false); + let mut tally = BindingTally::default(); + let mut checker_only_place = None; for binding in bindings { if evaluate_reachability_with_cache( db, @@ -1945,22 +2620,52 @@ fn bindings_runtime_definedness<'db>( continue; } match binding.binding { - DefinitionState::Defined(definition) - if !index.is_in_type_checking_block( - definition.file_scope(db), - definition.full_range(db, module).range(), - ) => - { - defined = true; + DefinitionState::Defined(definition) if made_only_for_the_checker(db, definition) => { + tally.checker_only = true; + checker_only_place = Some((definition.file_scope(db), definition.place(db))); } - _ => undefined = true, + DefinitionState::Defined(_) => tally.runs = true, + DefinitionState::Undefined | DefinitionState::Deleted => tally.unbound = true, } } - match (defined, undefined) { - (true, false) => RuntimeDefinedness::Defined, - (true, true) => RuntimeDefinedness::PossiblyUndefined, - (false, _) => RuntimeDefinedness::Undefined, + // the checker reads every `if TYPE_CHECKING:` block as taken, so a binding the program + // makes in its `else:` instead is one it never sees reach anything. the program still + // makes it, so the name holds something when it runs — not what the checker resolved + if !tally.runs + && let Some((scope, place)) = checker_only_place + { + tally.runs = binds_where_the_checker_does_not_look(db, index, scope, place); } + tally +} + +/// whether `scope` binds `place` anywhere the checker does not vouch for as unmade: outside +/// every branch made only for the checker, or in one the checker never reaches. the checker +/// reads every `if TYPE_CHECKING:` block as taken, so a binding in its `else:` is one the +/// checker sees reach nothing, and one the program makes +fn binds_where_the_checker_does_not_look<'db>( + db: &'db dyn Db, + index: &'db SemanticIndex<'db>, + scope: FileScopeId, + place: ScopedPlaceId, +) -> bool { + let bindings = index.use_def_map(scope).reachable_bindings(place); + let predicates = bindings.predicates(); + let reachability_constraints = bindings.reachability_constraints(); + bindings.into_iter().any(|binding| { + let DefinitionState::Defined(definition) = binding.binding else { + return false; + }; + !made_only_for_the_checker(db, definition) + || evaluate_reachability_with_cache( + db, + None, + reachability_constraints, + predicates, + binding.reachability_constraint, + ) + .is_always_false() + }) } /// The type declared for the place a binding writes to. diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index d92b66c145..f9d430b8b1 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -23,6 +23,7 @@ use crate::types::AnonNamedTupleClass; use crate::types::ide_support::{ImportAliasResolution, definition_for_name}; use crate::types::implicit_names::implicit_name; use crate::types::list_members::{all_members, all_reachable_members}; +use crate::types::runtime_spelling::RuntimeSite; use crate::types::{ ClassPatternPositionalResult, ClassPatternPositionalSource, CycleDetector, ProgramEnvironment, SpecialFormType, Type, TypeQualifiers, binding_type, class_pattern_positional_result, @@ -259,6 +260,7 @@ impl<'db> SemanticModel<'db> { function, positional, keywords, + Some(RuntimeSite::at(call)), ) } @@ -759,6 +761,20 @@ impl<'db> SemanticModel<'db> { AnonNamedTupleClass::of(self.db, &self.program_environment(), ty) } + /// basedpython: whether every name `expr` reads holds, where it is written, the binding the + /// checker resolved when the program runs — so writing `expr` into code that runs there + /// evaluates what the checker saw. a class imported only under `if TYPE_CHECKING:`, bound + /// only on some paths or only further on does not + pub fn reads_names_bound_where_written(&self, expr: &ast::Expr) -> bool { + crate::types::reified_infer::written_unbound_read( + self.db, + &self.program_environment(), + self.file(), + expr, + ) + .is_none() + } + /// could any conversion apply at this call at all? /// /// Binding the call a second time to find out is the expensive part, and @@ -1083,6 +1099,7 @@ impl<'db> SemanticModel<'db> { self.file(), class_literal, constructed, + Some(RuntimeSite::at(call)), ) } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 29e6ff80c3..3b46689c7e 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -146,6 +146,7 @@ use crate::types::reified_infer::{self, ErasedTargetReason, ReifiedInferenceErro use crate::types::repeated_underscore::{ LoweredParameters, WrittenNames, lower_repeated_underscores, parameter_slots, }; +use crate::types::runtime_spelling::RuntimeSite; use crate::types::set_theoretic::RecursivelyDefined; use crate::types::signatures::{CallableSignature, ReturnCallableTypeVarScope}; use crate::types::soundness::{ @@ -11234,6 +11235,47 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + /// a checked cast with no runtime check the checker can spell tests its value against + /// the target as written, so a target reading a name that does not hold there the binding + /// the checker resolved — a class imported only under `if TYPE_CHECKING:` — would raise + /// `NameError`, or test against another class. a cast that needs no check (a widening) is + /// a plain `typing.cast`, whose target is written as a string, and is left alone + fn report_cast_target_unbound_at_runtime( + &mut self, + type_arg: &ast::Expr, + value_ty: Type<'db>, + target: Type<'db>, + ) { + let env = self.program_environment(); + let db = self.db(); + if cast_is_redundant(db, env, value_ty, target) + || crate::types::soundness::cast_check_plan( + db, + env, + self.file(), + target, + Some(type_arg), + ) + .is_some() + { + return; + } + let Some((name, binding)) = + crate::types::reified_infer::written_unbound_read(db, env, self.file(), type_arg) + else { + return; + }; + let Some(builder) = self.context.report_lint(&ERASED_TYPE_CHECK, type_arg) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{}` cannot be checked at runtime: {}", + target.display(db, env), + binding.why_not_checkable(&name) + )); + diagnostic.info(binding.describe(&name)); + } + /// Report a plain `cast` that is not a widening, and say whether it was one. /// /// The unsuffixed `cast` reinterprets a value without looking at it, so the @@ -11473,6 +11515,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // claim; the plain `cast` never reaches a check in the first place if cast_kind.verifies_at_runtime() { self.report_erased_cast_argument(type_arg, value_ty, target); + self.report_cast_target_unbound_at_runtime(type_arg, value_ty, target); self.record_checked_cast_spellings(value_ty, target, type_arg); } // an unsound plain `cast` is already the sharper report — a @@ -12978,6 +13021,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { callable_type, function, &call_arguments, + Some(RuntimeSite::at(call_expression)), ) .err() .map(Some) @@ -13010,6 +13054,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ty.display(self.db(), env), )); } + Some(ReifiedInferenceError::NotAtRuntime(read, binding)) => { + diagnostic.info(binding.describe(&read)); + } Some(ReifiedInferenceError::NoBinding) => { diagnostic.info(format_args!( "the specialization cannot be inferred from this call — \ @@ -13182,6 +13229,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.file(), class_literal, return_ty, + Some(RuntimeSite::at(call_expression)), ) else { return; }; @@ -13205,6 +13253,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { spelling — construct a specialization, as in `{name}[int](...)`", ty.display(db, env), )), + ReifiedInferenceError::NotAtRuntime(read, binding) => { + diagnostic.info(binding.describe(&read)); + } ReifiedInferenceError::NoBinding => diagnostic.info(format_args!( "the specialization cannot be inferred from this call — construct \ one, as in `{name}[int](...)`" @@ -15898,7 +15949,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.context.record_spelling(spelling); } if let Some(spelling) = - crate::types::soundness::cast_check_plan(db, env, self.file(), target) + crate::types::soundness::cast_check_plan(db, env, self.file(), target, Some(type_arg)) .and_then(|check| check.spelling()) { self.context.record_spelling(&spelling); @@ -16746,12 +16797,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { for spelling in plan.spellings() { self.context.record_spelling(spelling); } - if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = plan { + if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = &plan { let source = ruff_db::source::source_text(db, self.file()); self.report_erased_type_check( TextRange::new(left.start(), right.end()), &source[right.range()], - reason, + reason.clone(), ); } else { // a target with no runtime test has already been rejected, and @@ -16824,6 +16875,13 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return; }; match reason { + ErasedTargetReason::NotAtRuntime(name, binding) => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: {}", + binding.why_not_checkable(&name) + )); + diagnostic.info(binding.describe(&name)); + } ErasedTargetReason::Protocol => { let mut diagnostic = builder.into_diagnostic(format_args!( "`is {target}` cannot be checked at runtime: a protocol's instances don't \ diff --git a/crates/ty_python_semantic/src/types/infer/builder/conditions.rs b/crates/ty_python_semantic/src/types/infer/builder/conditions.rs index 640049dd0b..9339e6193c 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/conditions.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/conditions.rs @@ -220,6 +220,11 @@ fn definition_is_environment_derived_inner<'db>( return false; } let module = parsed_module(db, definition.python_file(db)).load(db); + // the declaration of `typing.TYPE_CHECKING` is the fact itself — typeshed declares it + // without a value — so a name imported from it under another name stands for it too + if crate::place::declares_typing_type_checking(db, definition) { + return true; + } // an import is followed by name resolution before it ever reaches here, so what is left is // the value a binding was written with let value: &ast::Expr = match definition.kind(db) { diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 63ebb00850..9eabcec10d 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -26,7 +26,7 @@ use ruff_text_size::{Ranged, TextRange}; use rustc_hash::FxHashMap; use crate::Db; -use crate::place::{builtins_symbol, global_symbol}; +use crate::place::{RuntimeBinding, RuntimeRead, builtins_symbol, global_symbol, runtime_binding}; use crate::types::ProgramEnvironment; use crate::types::call::{Argument, CallArguments}; use crate::types::class::{ClassLiteral, ClassType, GenericAlias}; @@ -36,7 +36,7 @@ use crate::types::generics::{Specialization, combine_use_site_projections}; use crate::types::instance::Protocol; use crate::types::literal::LiteralValueTypeKind; use crate::types::protocol_class::ReifiedMember; -use crate::types::runtime_spelling::RuntimeSpelling; +use crate::types::runtime_spelling::{RuntimeSite, RuntimeSpelling}; use crate::types::tuple::Tuple; use crate::types::typevar::TypeVarKind; use crate::types::variance::TypeVarVariance; @@ -48,6 +48,9 @@ pub(crate) enum ReifiedInferenceError<'db> { Unsolved(Name), /// solved, but the solution has no runtime spelling at this call site Unspellable(Name, Type<'db>), + /// solved and spelled, but the spelling reads a name that does not hold, where the call + /// runs, the binding the checker resolved + NotAtRuntime(Name, RuntimeBinding), /// the callable's specialization could not be derived at all (no unique /// matching overload, or the arguments do not bind) NoBinding, @@ -65,6 +68,7 @@ pub(crate) fn inferred_call_type_arguments<'db>( callee: Type<'db>, function: FunctionType<'db>, arguments: &CallArguments<'_, 'db>, + site: Option, ) -> Result, ReifiedInferenceError<'db>> { let bindings = callee .try_call(db, env, arguments) @@ -75,7 +79,7 @@ pub(crate) fn inferred_call_type_arguments<'db>( .ok_or(ReifiedInferenceError::NoBinding)? .1 .merged_specialization(db, env); - rendered_type_arguments(db, env, file, function, specialization) + rendered_type_arguments(db, env, file, function, specialization, site) } /// [`inferred_call_type_arguments`] for callers outside the `types` module: @@ -85,6 +89,7 @@ pub(crate) fn inferred_call_type_arguments<'db>( /// keyword-variadic pack contributes fields, since a subscript takes no /// keywords. any failure collapses to `None` — the checker reports those, the /// caller just skips injection +#[expect(clippy::too_many_arguments)] pub(crate) fn injectable_call_specialization<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -93,6 +98,7 @@ pub(crate) fn injectable_call_specialization<'db>( function: FunctionType<'db>, positional: Vec>, keywords: Vec<(&str, Type<'db>)>, + site: Option, ) -> Option { let arguments: CallArguments<'_, 'db> = positional .into_iter() @@ -104,7 +110,7 @@ pub(crate) fn injectable_call_specialization<'db>( ) .collect(); let rendered = - inferred_call_type_arguments(db, env, file, callee, function, &arguments).ok()?; + inferred_call_type_arguments(db, env, file, callee, function, &arguments, site).ok()?; // an empty prefix means everything defaults — the bare call is already // correct and nothing is injected if rendered.is_empty() { @@ -146,6 +152,7 @@ fn rendered_type_arguments<'db>( file: File, function: FunctionType<'db>, specialization: Option>, + site: Option, ) -> Result, ReifiedInferenceError<'db>> { let signature = function.signature(db); let generic_context = signature @@ -204,14 +211,21 @@ fn rendered_type_arguments<'db>( .value .ok_or_else(|| ReifiedInferenceError::Unsolved(parameter.name.clone()))?; let promoted = ty.promote_in(db, env, file); - parameter + let text = parameter .kind .spelling(db, env, file, promoted) - .map(|text| TypeArgument { - text, - keyword: parameter.kind == ParameterKind::KeywordPack, - }) - .ok_or_else(|| ReifiedInferenceError::Unspellable(parameter.name.clone(), promoted)) + .ok_or_else(|| { + ReifiedInferenceError::Unspellable(parameter.name.clone(), promoted) + })?; + if let Some(site) = site + && let Some((name, binding)) = text.unbound_read(db, env, file, site) + { + return Err(ReifiedInferenceError::NotAtRuntime(name, binding)); + } + Ok(TypeArgument { + text, + keyword: parameter.kind == ParameterKind::KeywordPack, + }) }) // a variadic or pack that absorbed nothing spells as nothing — it // occupies no slot in the injected list, exactly as the wrapper binds it @@ -612,6 +626,7 @@ fn constructor_specialization<'db>( file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, + site: Option, ) -> Result> { let Type::NominalInstance(instance) = constructed.promote_in(db, env, file) else { return Err(ReifiedInferenceError::NoBinding); @@ -623,7 +638,13 @@ fn constructor_specialization<'db>( if origin != class_literal { return Err(ReifiedInferenceError::NoBinding); } - spell_specialization_arguments(db, env, file, origin, alias.specialization(db)) + let spelling = spell_specialization_arguments(db, env, file, origin, alias.specialization(db))?; + if let Some(site) = site + && let Some((name, binding)) = spelling.unbound_read(db, env, file, site) + { + return Err(ReifiedInferenceError::NotAtRuntime(name, binding)); + } + Ok(spelling) } /// The bracketed type-argument spelling to inject at a bare constructor call @@ -639,8 +660,9 @@ pub(crate) fn constructor_specialization_display<'db>( file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, + site: Option, ) -> Option { - constructor_specialization(db, env, file, class_literal, constructed).ok() + constructor_specialization(db, env, file, class_literal, constructed, site).ok() } /// Why a construction of the reified generic class `class_literal` cannot say @@ -658,8 +680,9 @@ pub(crate) fn reified_construction_error<'db>( file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, + site: Option, ) -> Option> { - constructor_specialization(db, env, file, class_literal, constructed).err() + constructor_specialization(db, env, file, class_literal, constructed, site).err() } /// the class's bare name, provided that name resolves — in the module's @@ -667,6 +690,9 @@ pub(crate) fn reified_construction_error<'db>( /// evaluates to the intended type object. a module-level `private` class is /// emitted under a name of the transpiler's, so its name is kept apart for the /// transpiler to write ([`RuntimeSpelling::global`]) +/// +/// whether the name holds that class where the expression is written is the +/// writer's to ask ([`RuntimeSpelling::unbound_read`]) fn spell_class_literal<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -930,7 +956,7 @@ pub enum ProtocolMemberCheck { } /// why a type test's target cannot be checked at runtime -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum ErasedTargetReason { /// the target says nothing a runtime check could look for — `Any` and /// `Unknown` admit every value, so the test has no content @@ -959,6 +985,10 @@ pub enum ErasedTargetReason { /// the target has no runtime spelling at all — a type the checker can name /// but the emitted python cannot evaluate Unspellable, + /// the target names something that does not hold, where the test runs, the binding + /// the checker resolved — a class imported only under `if TYPE_CHECKING:`, or declared + /// further on + NotAtRuntime(Name, RuntimeBinding), /// a builtin collection (`list` / `dict` / `set` / `frozenset` / `tuple`) /// erases its type arguments — its C-level instances reject /// `__orig_class__` entirely @@ -1269,7 +1299,7 @@ pub(crate) fn type_test_plan<'db>( target: Type<'db>, target_node: Option<&ast::Expr>, ) -> ParametricIsPlan { - type_test_plan_seen( + let plan = type_test_plan_seen( db, env, file, @@ -1277,7 +1307,18 @@ pub(crate) fn type_test_plan<'db>( target, target_node, &mut Vec::new(), - ) + ); + // each spelling the plan rebuilds is written where the test is: one for a union's arm + // has no node of its own to have been asked about + if let Some(site) = target_node.map(RuntimeSite::at) + && let Some((name, binding)) = plan + .spellings() + .into_iter() + .find_map(|spelling| spelling.unbound_read(db, env, file, site)) + { + return ParametricIsPlan::ErasedTarget(ErasedTargetReason::NotAtRuntime(name, binding)); + } + plan } /// [`type_test_plan`] carrying the aliases already opened on the way here. @@ -1476,14 +1517,14 @@ fn literal_target_plan<'db>( // an enum member is a singleton, so identity is exact — and it is the // comparison `Enum.__eq__` performs anyway LiteralValueTypeKind::Enum(member) => { - match written_or_spelled(target_node, false, || { + match written_or_spelled(db, env, file, target_node, false, || { spell_class_literal(db, env, file, member.enum_class(db)).map(|mut class| { class.push_str(&format!(".{}", member.name(db))); class }) }) { - Some(spelling) => ParametricIsPlan::Identity(spelling), - None => unspellable, + Ok(spelling) => ParametricIsPlan::Identity(spelling), + Err(reason) => ParametricIsPlan::ErasedTarget(reason), } } // `LiteralString` is the *property* of having been written as a literal, @@ -1511,17 +1552,85 @@ fn literal_target_plan<'db>( /// or a class imported under another name. Only a plain dotted name qualifies, /// and only a [`Probe`](ParametricIsPlan::Probe) also accepts a subscript, /// whose value is the specialization it unwinds. -fn written_or_spelled( +/// +/// the written node is evaluated where it stands, and a rebuilt spelling where the test is +/// written, so each has to read names that hold there the bindings the checker resolved +/// ([`runtime_binding`]): a class imported under `if TYPE_CHECKING:` is one the checker +/// resolves and the program does not have +fn written_or_spelled<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, target_node: Option<&ast::Expr>, subscript_evaluates: bool, spell: impl FnOnce() -> Option, -) -> Option { +) -> Result { + let site = target_node.map(RuntimeSite::at); + let mut unbound = None; if let Some(spelling) = spell() { - return Some(TargetSpelling::Rebuilt(spelling)); + match site.and_then(|site| spelling.unbound_read(db, env, file, site)) { + None => return Ok(TargetSpelling::Rebuilt(spelling)), + Some(read) => unbound = Some(read), + } + } + let Some(node) = target_node.filter(|node| evaluates_to_its_target(node, subscript_evaluates)) + else { + return Err( + unbound.map_or(ErasedTargetReason::Unspellable, |(name, binding)| { + ErasedTargetReason::NotAtRuntime(name, binding) + }), + ); + }; + match written_unbound_read(db, env, file, node) { + None => Ok(TargetSpelling::Written), + Some((name, binding)) => Err(ErasedTargetReason::NotAtRuntime(name, binding)), } - target_node - .filter(|node| evaluates_to_its_target(node, subscript_evaluates)) - .map(|_| TargetSpelling::Written) +} + +/// the first name the written expression `node` reads that does not hold, where `node` is +/// written, the binding the checker resolved — with what it holds instead. the same question +/// [`RuntimeSpelling::unbound_read`] asks of a spelling the checker writes, asked of one the +/// source wrote +pub(crate) fn written_unbound_read<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + node: &ast::Expr, +) -> Option<(Name, RuntimeBinding)> { + struct Names<'a, 'db> { + db: &'db dyn Db, + env: &'a ProgramEnvironment<'db>, + file: File, + found: Option<(Name, RuntimeBinding)>, + } + impl<'ast> ast::visitor::Visitor<'ast> for Names<'_, '_> { + fn visit_expr(&mut self, expr: &'ast ast::Expr) { + if self.found.is_some() { + return; + } + if let ast::Expr::Name(name) = expr { + let binding = runtime_binding( + self.db, + self.env, + self.db.program_file(self.file), + RuntimeRead::Written(ast::ExprRef::from(expr)), + ); + if binding != RuntimeBinding::Bound { + self.found = Some((name.id.clone(), binding)); + } + return; + } + ast::visitor::walk_expr(self, expr); + } + } + let mut names = Names { + db, + env, + file, + found: None, + }; + ast::visitor::Visitor::visit_expr(&mut names, node); + names.found } /// whether the runtime value of `node` is the thing the type it names denotes @@ -1596,11 +1705,11 @@ fn class_target_plan<'db>( .iter() .all(Type::is_dynamic) => { - match written_or_spelled(target_node, false, || { + match written_or_spelled(db, env, file, target_node, false, || { spell_class_literal(db, env, file, ClassLiteral::Static(alias.origin(db))) }) { - Some(spelling) => ParametricIsPlan::Isinstance(spelling), - None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + Ok(spelling) => ParametricIsPlan::Isinstance(spelling), + Err(reason) => ParametricIsPlan::ErasedTarget(reason), } } // a specialization keeps the engine it always had: reified cells, @@ -1613,15 +1722,15 @@ fn class_target_plan<'db>( // registry rather than by the class hierarchy, so it is checkable // even though a conforming type is not a subclass if let Some((members, declared_in)) = conformance_members(db, file, class) { - return match written_or_spelled(target_node, false, || { + return match written_or_spelled(db, env, file, target_node, false, || { spell_class(db, env, file, class) }) { - Some(target) => ParametricIsPlan::Conformance { + Ok(target) => ParametricIsPlan::Conformance { target, members, declared_in, }, - None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + Err(reason) => ParametricIsPlan::ErasedTarget(reason), }; } if let Some(protocol) = class.into_protocol_class(db) { @@ -1643,9 +1752,11 @@ fn class_target_plan<'db>( }; } } - match written_or_spelled(target_node, false, || spell_class(db, env, file, class)) { - Some(spelling) => ParametricIsPlan::Isinstance(spelling), - None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + match written_or_spelled(db, env, file, target_node, false, || { + spell_class(db, env, file, class) + }) { + Ok(spelling) => ParametricIsPlan::Isinstance(spelling), + Err(reason) => ParametricIsPlan::ErasedTarget(reason), } } } @@ -1944,10 +2055,11 @@ fn classify_value<'db>( // runtime probe (user generic) or an erased-target error (builtin) // the source's own spelling is preferred: a name it wrote is in scope, // and the runtime probe unwraps a `TypeAliasType` for itself - let Some(target) = written_or_spelled(rhs_node, true, || { + let target = match written_or_spelled(db, env, file, rhs_node, true, || { spell_class(db, env, file, ClassType::Generic(rhs_alias)) - }) else { - return ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); + }) { + Ok(target) => target, + Err(reason) => return ParametricIsPlan::ErasedTarget(reason), }; ParametricIsPlan::Probe { target, diff --git a/crates/ty_python_semantic/src/types/runtime_spelling.rs b/crates/ty_python_semantic/src/types/runtime_spelling.rs index 1a2dc66c8f..e3cc06334e 100644 --- a/crates/ty_python_semantic/src/types/runtime_spelling.rs +++ b/crates/ty_python_semantic/src/types/runtime_spelling.rs @@ -15,10 +15,29 @@ use ruff_db::files::File; use ruff_python_ast::name::Name; +use ruff_text_size::{Ranged, TextRange}; use crate::Db; +use crate::place::{RuntimeBinding, RuntimeRead, runtime_binding}; +use crate::types::ProgramEnvironment; use crate::types::visibility::private_symbols; +/// where a lowering writes a spelling into code that runs, which decides what each name it +/// reads holds there +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RuntimeSite { + range: TextRange, +} + +impl RuntimeSite { + /// the code `node` is written as + pub(crate) fn at(node: &impl Ranged) -> Self { + Self { + range: node.range(), + } + } +} + /// a python expression the checker spells for the transpiled program, which evaluates in /// the module it was spelled for to the runtime object of a type (`list[int]`) or to a /// value (`Color.RED`) @@ -74,6 +93,29 @@ impl RuntimeSpelling { }) } + /// the first module-level name the spelling reads that does not hold, where `site` runs, + /// the binding the checker resolved — with what it holds instead + pub(crate) fn unbound_read<'db>( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + site: RuntimeSite, + ) -> Option<(Name, RuntimeBinding)> { + self.global_reads().find_map(|name| { + let binding = runtime_binding( + db, + env, + db.program_file(file), + RuntimeRead::Global { + name, + range: site.range, + }, + ); + (binding != RuntimeBinding::Bound).then(|| (name.clone(), binding)) + }) + } + pub(crate) fn push_str(&mut self, text: &str) { if text.is_empty() { return; diff --git a/crates/ty_python_semantic/src/types/soundness.rs b/crates/ty_python_semantic/src/types/soundness.rs index 7e2fac204e..2dc18b0b3a 100644 --- a/crates/ty_python_semantic/src/types/soundness.rs +++ b/crates/ty_python_semantic/src/types/soundness.rs @@ -12,6 +12,7 @@ //! what second argument does that check take? (`runtime_check_target`) use ruff_db::files::File; +use ruff_python_ast as ast; use crate::Db; use crate::place::{Place, explicit_global_symbol}; @@ -21,7 +22,7 @@ use crate::types::literal::LiteralValueTypeKind; use crate::types::reified_infer::{ ArgVariance, parametric_soundness_spelling, protocol_structural_members, }; -use crate::types::runtime_spelling::RuntimeSpelling; +use crate::types::runtime_spelling::{RuntimeSite, RuntimeSpelling}; use crate::types::signatures::CallableSignature; use crate::types::visitor::any_over_type; use crate::types::{ClassLiteral, ClassType, FunctionType, KnownClass, Type}; @@ -203,11 +204,32 @@ fn python_string_literal(value: &str) -> String { /// protocol with no faithful check degrades to [`CastCheck::Unchecked`]. This /// keeps the emitted code from ever running `isinstance` against a subscripted /// or non-`@runtime_checkable` protocol, which raises at runtime. +/// +/// `written_at` is the cast's written target, where the check runs: a check whose target +/// reads a name that does not hold there the binding the checker resolved has no plan pub fn cast_check_plan<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, + written_at: Option<&ast::Expr>, +) -> Option { + let plan = cast_check_plan_anywhere(db, env, file, ty)?; + if let Some(site) = written_at.map(RuntimeSite::at) + && plan + .spelling() + .is_some_and(|spelling| spelling.unbound_read(db, env, file, site).is_some()) + { + return None; + } + Some(plan) +} + +fn cast_check_plan_anywhere<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + ty: Type<'db>, ) -> Option { if let Type::ProtocolInstance(instance) = ty { if let Protocol::FromClass(protocol_class) = instance.inner @@ -236,7 +258,7 @@ pub fn cast_target_is_unverifiable_protocol<'db>( target: Type<'db>, ) -> bool { matches!( - cast_check_plan(db, env, file, target), + cast_check_plan_anywhere(db, env, file, target), Some(CastCheck::Unchecked) ) } @@ -494,7 +516,8 @@ fn target<'db>( /// resolve the runtime name for `literal` as seen from module scope in /// `file`: either the module binds the class's name to this exact class /// (definition or import), or the name is unbound and the class is a -/// builtin so the bare name reaches it +/// builtin so the bare name reaches it. whether the name holds that binding +/// where the check runs is the site's to ask ([`RuntimeSpelling::unbound_read`]) fn class_target<'db>( db: &'db dyn Db, file: File, diff --git a/crates/ty_python_semantic/src/types/soundness_sites.rs b/crates/ty_python_semantic/src/types/soundness_sites.rs index 2348b99000..cac3283c25 100644 --- a/crates/ty_python_semantic/src/types/soundness_sites.rs +++ b/crates/ty_python_semantic/src/types/soundness_sites.rs @@ -12,6 +12,7 @@ use ruff_text_size::{Ranged, TextRange}; use crate::Db; use crate::semantic_model::{HasType, SemanticModel}; +use crate::types::runtime_spelling::RuntimeSite; use crate::types::soundness::{ ArgSelector, CheckKind, CheckTarget, call_result_is_typevar_derived, is_specialized_generic_instance, parameter_check_plan, parameter_runtime_check_plan, @@ -270,8 +271,23 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { .filter(|plan| !matches!(plan, CheckKind::Isinstance(CheckTarget::NoneType))) } + /// whether every name `plan`'s target reads holds, where `site` runs, the binding the + /// checker resolved. a check that would read one that does not — a class imported only + /// under `if TYPE_CHECKING:`, or declared further on — is left out, as a check with no + /// faithful target is + fn runs_at(&self, plan: &CheckKind, site: Option) -> bool { + site.is_some_and(|site| { + plan.spelling() + .unbound_read(self.db(), &self.env(), self.model.file(), site) + .is_none() + }) + } + /// check the value `expr` produces against `plan` fn check_value(&mut self, expr: &'ast Expr, plan: CheckKind) { + if !self.runs_at(&plan, Some(RuntimeSite::at(expr))) { + return; + } self.sites.push(SoundnessSite::Value { expr, plan }); } @@ -285,6 +301,9 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { let Some(plan) = self.check_plan(target_expr) else { return; }; + if !self.runs_at(&plan, Some(RuntimeSite::at(iterable))) { + return; + } self.sites.push(SoundnessSite::Elements { iterable, is_async, @@ -358,7 +377,9 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { .chain(¶ms.kwonlyargs) { let parameter = &pwd.parameter; - if let Some(plan) = self.parameter_plan(parameter) { + if let Some(plan) = self.parameter_plan(parameter) + && self.runs_at(&plan, Some(RuntimeSite::at(func))) + { guards.push((parameter, plan)); } } diff --git a/docs/basedpython/features/checked-cast.md b/docs/basedpython/features/checked-cast.md index 3d9d509be5..2204fffa51 100644 --- a/docs/basedpython/features/checked-cast.md +++ b/docs/basedpython/features/checked-cast.md @@ -76,6 +76,13 @@ both forms: - skip the check entirely when it is provably redundant. if the value is already the target, the probe would always pass, so the cast degrades to the same plain `typing.cast` the unsuffixed `cast` emits +- refuse a target naming a class the program does not have where the cast runs + (`erased-type-check`), as a [type test](parametric-type-tests.md#the-erased-type-check-error) + does: one imported only under `if TYPE_CHECKING:` (or past any other condition + that reads the constant), imported only on some paths, + or bound in the `else:` of a `TYPE_CHECKING` block, where the name need not be + the class the checker resolved. an unchecked `cast` tests nothing, and writes + such a type as a string - evaluate the value exactly once, even when it has side effects (`g() cast! int` calls `g()` once) - are basedpython-only: a `.py` file using either produces a parse error. they diff --git a/docs/basedpython/features/parametric-type-tests.md b/docs/basedpython/features/parametric-type-tests.md index 8308412118..350674c0ee 100644 --- a/docs/basedpython/features/parametric-type-tests.md +++ b/docs/basedpython/features/parametric-type-tests.md @@ -182,6 +182,40 @@ def g(x): return x is array.array # ok — an ordinary isinstance ``` +and a target naming a class the program does not have where the test runs, since the test reads +it there: one imported only under `if TYPE_CHECKING:` (or past any other condition that reads the +constant, such as `match TYPE_CHECKING:` or `if TYPE_CHECKING and ready:`), or imported only on some +paths + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +def f(x) -> bool: + return x is Decimal # error: `Decimal` is not bound when it runs +``` + +a module can import a class for the checker and bind the name some other way as it runs, in the +`else:` of the same block. the checker reads the block as always taken, so the class it resolves is +the `TYPE_CHECKING` one, and the name the test would read need not be that class — so that is +refused too: + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal +else: + Decimal = float + +def f(x) -> bool: + return x is Decimal # error: the `Decimal` bound when it runs need not be the class the checker resolved +``` + +import the class as the program runs to test against it + ## `===` is unaffected the `===` / `!==` identity operators keep python semantics; only the `is` / diff --git a/docs/basedpython/features/reified-class-generics.md b/docs/basedpython/features/reified-class-generics.md index 25774332c5..bd388e7976 100644 --- a/docs/basedpython/features/reified-class-generics.md +++ b/docs/basedpython/features/reified-class-generics.md @@ -137,7 +137,14 @@ class Holder: where nothing solves it, the bare construction is an error (`unspecialized-reified-generic`) rather than an instance that would fail the -first time a method asked it a question +first time a method asked it a question. that includes a type argument the +running program does not have where the construction runs: a class imported only +under `if TYPE_CHECKING:` (or past any other condition that reads the constant), +imported only on some paths, unbound by a `del`, or declared further down the +module than the statement the construction is in — +for one in a function or a method, the `def` or `class` statement at the top of +the module that holds it, which can be called as soon as that runs. write the +specialization out there (`Box[Later]()`) where you know it is bound ## a reified parameter is invariant diff --git a/docs/basedpython/features/soundness.md b/docs/basedpython/features/soundness.md index 5b2490b667..aeab8bf48d 100644 --- a/docs/basedpython/features/soundness.md +++ b/docs/basedpython/features/soundness.md @@ -106,8 +106,20 @@ no check is emitted when there is no faithful shallow runtime test — protocols, callables, unsolved type parameters, and `object` (which every value satisfies) are all skipped, as is a type whose name doesn't resolve at module scope (a class shadowed by a local binding, or one not in scope where -the check would run). a pure-`None` result is skipped too: validating a value -that carries no data guards nothing +the check would run). a class the program does not have where the check runs +is skipped the same way: one bound only past a condition that reads +`TYPE_CHECKING` (how an import cycle is broken) — an `if`, `while` or `match` +on the constant under any name or inside any expression, or a value a +conditional expression on it chooses — one imported only on some paths, +unbound by a `del`, rebound from a function through `global`, or declared +further down the module than the check, and one bound in the `else:` of a +`TYPE_CHECKING` block, where the name the check would read need not be the +class the checker resolved. a pure-`None` result is skipped too: validating a +value that carries no data guards nothing + +a check in a function or a method runs whenever it is called, so there the +class has to be bound above the `def` or `class` statement at the top of the +module that holds the function, which can be called as soon as that runs ## deep checks for user generics From 5b12c7bc1fe0bf0a8c2dff25c8a8335faef1976b Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:45:40 +1000 Subject: [PATCH 32/47] make a soundness check in a function on a class the module binds further down, or only on some paths, once the class is bound, by looking the name up as the check runs (`for cut in _soundness_once_bound(("Cut",), self.cuts, _soundness_iter, lambda: Cut):`), where every binding the module makes of the name binds that one class, none only for the checker, and nothing unbinds or rebinds it through `global`; such a check was left out. the lookup is keyed by the name the class is emitted under (`"_Kept"` for a `private class Kept`), a check at the module's own level keeps the rule that the class is bound above it, and `by compile` leaves a function making such a check to its interpreted definition --- crates/by_irbuild/src/lib.rs | 13 +- crates/by_irbuild/src/tests.rs | 18 + crates/by_transforms/src/runtime.rs | 1 + .../by_transforms/src/runtime/_by_runtime.py | 14 + .../by_transforms/src/transforms/soundness.rs | 339 +++++++++++++----- crates/by_transforms/src/type_info.rs | 3 + .../tests/type_checking_import_runtime.rs | 63 +++- crates/ty_python_semantic/src/place.rs | 21 ++ .../src/types/runtime_spelling.rs | 18 + .../src/types/soundness_sites.rs | 108 ++++-- docs/basedpython/features/soundness.md | 33 +- 11 files changed, 503 insertions(+), 128 deletions(-) diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index fc4f1e9e77..9cf5006408 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -143,6 +143,7 @@ use ruff_python_stdlib::identifiers::is_identifier; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::ProgramEnvironment; use ty_python_semantic::types::soundness::{CheckKind, CheckTarget}; +use ty_python_semantic::types::soundness_sites::SiteCheck; use ty_python_semantic::types::{KnownClass, SpecialFormType, Type, TypeDefinition, TypeQualifier}; use ty_python_semantic::{HasType, SemanticModel}; @@ -15645,8 +15646,8 @@ impl Lowering<'_, '_> { /// with: an unboxed `int` came through `PyLong_Check`, and an emitted class's /// instance through a type test. the narrowing that made the value raised the /// check's own `TypeError` if it failed, so nothing is left to ask - fn sound_target(&mut self, ty: &RType, plan: &CheckKind) -> Lowered> { - let CheckKind::Isinstance(target) = plan else { + fn sound_target(&mut self, ty: &RType, check: &SiteCheck) -> Lowered> { + let CheckKind::Isinstance(target) = &check.plan else { return Err(Decline::new( "a soundness check against a generic class's type arguments is not lowered", )); @@ -15654,6 +15655,14 @@ impl Lowering<'_, '_> { if self.proves(ty, target) { return Ok(None); } + // the interpreted build looks such a check's names up as it runs, and makes it only + // once they are bound + if !check.once_bound.is_empty() { + return Err(Decline::new( + "a soundness check on a class the module may not have bound yet when it runs \ + is not lowered", + )); + } self.resolve_sound_target(target).map(Some) } diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index 4e6a24ca83..8484b01126 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -451,6 +451,24 @@ def unwrap(v: Any) -> Cell[int]: assert!(reason.contains("type arguments"), "{reason}"); } +#[test] +fn a_soundness_check_on_a_class_declared_further_down_declines() { + // the function may run before the module binds `Later`, so the interpreted build looks + // it up as the check runs and makes the check only once it is bound, which a compiled + // module has no lowering for + let reason = decline( + "\ +from typing import Any + +def unwrap(v: Any) -> Later: + return v + +class Later: ... +", + ); + assert!(reason.contains("may not have bound yet"), "{reason}"); +} + #[test] fn a_gradual_parameter_is_an_object() { // `object` assumes nothing, so no check is needed and nothing declines diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs index df1d8397af..c285b5ff29 100644 --- a/crates/by_transforms/src/runtime.rs +++ b/crates/by_transforms/src/runtime.rs @@ -73,6 +73,7 @@ helpers! { SOUNDNESS_PARAMETRIC = "_soundness_parametric", SOUNDNESS_ITER_P = "_soundness_iter_p", SOUNDNESS_AITER_P = "_soundness_aiter_p", + SOUNDNESS_ONCE_BOUND = "_soundness_once_bound", CHECKED_CAST = "_checked_cast", TRY_CAST = "_try_cast", CHECKED_CAST_PRED = "_checked_cast_pred", diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py index ad663b078b..78ca777e53 100644 --- a/crates/by_transforms/src/runtime/_by_runtime.py +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -32,6 +32,20 @@ async def _soundness_aiter(_it, _t): yield _soundness_check(_x, _t) +# a check in a function on a class the module binds further down, or only on some +# paths. every binding the module makes of each of `_names` binds the class the check +# names, so once they are all bound `_target` builds the target the checker named, and +# `_check` is made against it; before then the value passes unchecked. the names are +# looked up in the globals of the module the check is written in, which the lambda +# `_target` carries +def _soundness_once_bound(_names, _v, _check, _target, *_rest): + _globals = _target.__globals__ + for _name in _names: + if _name not in _globals: + return _v + return _check(_v, _target(), *_rest) + + # --- checked casts --------------------------------------------------------- # `value cast! T` verifies at run time and raises on a mismatch; `value cast? T` # yields None instead. the predicate forms serve any target the parametric diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index e4c9330990..86e1aac129 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -34,6 +34,13 @@ //! no check. A check whose target is exactly `type(None)` is dropped too: //! validating a `None` result guards no data flowing onward. //! +//! A check is made only where every name its target reads holds the class the +//! checker resolved. In a function, which runs whenever it is called, a name +//! the module binds to one class and nothing else — a class declared further +//! down — is looked up as the check runs instead: +//! `_soundness_once_bound(("Cut",), expr, _soundness_check, lambda: Cut)` makes +//! the check once `Cut` is bound and passes the value unchecked before. +//! //! When the target is a *user-defined* generic specialization (`A[int]`), the //! check deepens to `_soundness_parametric(expr, A[int], variances)`, which //! validates the base class *and* — when the value carries its reified @@ -60,9 +67,7 @@ //! //! Known gaps, deliberate for now: no checks on `await` results, unpacking //! targets, `*args`/`**kwargs` spreads, or arguments to non-function callees -//! (class constructors, overloaded functions); a check argument naming a class -//! defined later in the module can raise `NameError` if the checked line runs -//! at import time before the class body. +//! (class constructors, overloaded functions). use std::cell::Cell; use std::collections::HashMap; @@ -82,7 +87,10 @@ use super::repeated_underscore::WrittenNames; use super::source_util::{PrologueStatement, body_prologue, needs_parentheses_as_argument}; use crate::Config; use crate::config::SoundnessPositions; -use crate::type_info::{SoundnessCheck, TypeInfo}; +use crate::type_info::{SiteCheck, SoundnessCheck, TypeInfo}; + +/// the helper a check is made through when its target reads a name that may not be bound yet +const ONCE_BOUND: &str = crate::runtime::SOUNDNESS_ONCE_BOUND.name(); // deep check for a user-generic-specialized target: validates the base class // always, and the reified type arguments when the value carries them @@ -96,22 +104,19 @@ use crate::type_info::{SoundnessCheck, TypeInfo}; /// decided on, with each checked parameter named as python binds it enum Site<'ast> { /// the value `expr` produces is checked as it is produced - Value { - expr: &'ast Expr, - plan: SoundnessCheck, - }, + Value { expr: &'ast Expr, check: SiteCheck }, /// each element drawn from `iterable` is checked as it is drawn Elements { iterable: &'ast Expr, is_async: bool, - plan: SoundnessCheck, + check: SiteCheck, }, /// `function`'s own parameters are checked where its body begins Parameters { function: &'ast StmtFunctionDef, /// each checked parameter by the name python binds it to, which for a repeated /// `_` is not the one the source spells - guards: Vec<(Name, SoundnessCheck)>, + guards: Vec<(Name, SiteCheck)>, }, } @@ -127,24 +132,24 @@ fn sites<'ast>( .soundness_sites(suite, positions) .into_iter() .map(|site| match site { - SoundnessSite::Value { expr, plan } => Site::Value { expr, plan }, + SoundnessSite::Value { expr, check } => Site::Value { expr, check }, SoundnessSite::Elements { iterable, is_async, - plan, + check, } => Site::Elements { iterable, is_async, - plan, + check, }, SoundnessSite::Parameters { function, guards } => Site::Parameters { function, guards: guards .into_iter() - .map(|(parameter, plan)| { + .map(|(parameter, check)| { ( crate::python_parameter_name(&function.parameters, parameter, written), - plan, + check, ) }) .collect(), @@ -182,25 +187,25 @@ impl SiteKey { /// same values against the same types, under the same `--soundness` positions #[derive(Debug, Default)] pub struct SoundnessSites { - values: HashMap, - elements: HashMap, - parameters: HashMap>, + values: HashMap, + elements: HashMap, + parameters: HashMap>, } impl SoundnessSites { /// the check made on the value `expr` produces, as it is produced - pub fn value(&self, expr: &Expr) -> Option<&SoundnessCheck> { + pub fn value(&self, expr: &Expr) -> Option<&SiteCheck> { self.values.get(&SiteKey::of(expr)?) } /// the check made on each element drawn from `iterable`, as it is drawn - pub fn elements(&self, iterable: &Expr) -> Option<&SoundnessCheck> { + pub fn elements(&self, iterable: &Expr) -> Option<&SiteCheck> { self.elements.get(&SiteKey::of(iterable)?) } /// the checks made on `function`'s own parameters where its body begins, by the /// name python binds each parameter to, in the order the parameters are written - pub fn parameters(&self, function: &StmtFunctionDef) -> &[(String, SoundnessCheck)] { + pub fn parameters(&self, function: &StmtFunctionDef) -> &[(String, SiteCheck)] { SiteKey::of(function) .and_then(|key| self.parameters.get(&key)) .map_or(&[], Vec::as_slice) @@ -218,21 +223,23 @@ pub fn soundness_sites( let mut sites = SoundnessSites::default(); for site in self::sites(model, written, suite, positions) { match site { - Site::Value { expr, plan } => { + Site::Value { expr, check } => { if let Some(key) = SiteKey::of(expr) { - sites.values.insert(key, plan); + sites.values.insert(key, check); } } - Site::Elements { iterable, plan, .. } => { + Site::Elements { + iterable, check, .. + } => { if let Some(key) = SiteKey::of(iterable) { - sites.elements.insert(key, plan); + sites.elements.insert(key, check); } } Site::Parameters { function, guards } => { if let Some(key) = SiteKey::of(function) { let guards = guards .into_iter() - .map(|(name, plan)| (name.to_string(), plan)) + .map(|(name, check)| (name.to_string(), check)) .collect(); sites.parameters.insert(key, guards); } @@ -274,6 +281,9 @@ struct Rendered<'src> { used_aiter: bool, used_iter_p: bool, used_aiter_p: bool, + /// any check made once the names it reads are bound — pulls in + /// `_soundness_once_bound` + used_once_bound: bool, /// any deep parametric check emitted — pulls in `_soundness_parametric` /// and the `_parametric_is` probe it reuses used_parametric: bool, @@ -290,6 +300,7 @@ impl<'src> Rendered<'src> { used_aiter: false, used_iter_p: false, used_aiter_p: false, + used_once_bound: false, used_parametric: false, } } @@ -299,9 +310,9 @@ impl<'src> Rendered<'src> { self.written.spelled(&target.spelling()) } - /// wrap `expr` in `helper(, )`. `trailing` - /// carries its own leading `, ` (e.g. `", str"` or `", A[int], (0,)"`) - fn wrap_call(&mut self, expr: &Expr, helper: &str, trailing: &str) { + /// wrap `expr` in `helper()`. `leading` carries its own + /// trailing `, `, and `trailing` its own leading `, ` (e.g. `", str"`) + fn wrap_call(&mut self, expr: &Expr, helper: &str, leading: &str, trailing: &str) { let (open, close) = if needs_parentheses_as_argument(expr) { ("(", ")") } else { @@ -309,90 +320,90 @@ impl<'src> Rendered<'src> { }; self.wraps.push(( expr.range(), - format!("{helper}({open}"), + format!("{helper}({leading}{open}"), format!("{close}{trailing})"), )); } - /// wrap `expr` in the scalar check (`_soundness_check` / - /// `_soundness_parametric`) named by `plan` - fn wrap_check(&mut self, expr: &Expr, plan: &SoundnessCheck) { - match plan { - SoundnessCheck::Isinstance(target) => { - let target = self.target(target); - self.wrap_call(expr, "_soundness_check", &format!(", {target}")); - } + /// the helper a check against `check`'s target is made with, and the text its value + /// argument is preceded and followed by. `helpers` names the plain helper and the + /// parametric one, which also takes the variances + fn checking( + &mut self, + check: &SiteCheck, + helpers: [&'static str; 2], + ) -> (String, String, String) { + let (helper, target, rest) = match &check.plan { + SoundnessCheck::Isinstance(target) => (helpers[0], self.target(target), String::new()), SoundnessCheck::Parametric { alias, variances } => { self.used_parametric = true; - let alias = self.written.spelled(alias); - self.wrap_call( - expr, - "_soundness_parametric", - &format!(", {alias}, {}", variance_tuple(variances)), - ); + ( + helpers[1], + self.written.spelled(alias), + format!(", {}", variance_tuple(variances)), + ) } + }; + if check.once_bound.is_empty() { + return ( + helper.to_owned(), + String::new(), + format!(", {target}{rest}"), + ); } + // the target is read through a lambda, once every name it reads is bound + self.used_once_bound = true; + let names: Vec = check + .once_bound + .iter() + .map(|read| format!("\"{}\"", self.written.spelled(read))) + .collect(); + ( + ONCE_BOUND.to_owned(), + format!("({},), ", names.join(", ")), + format!(", {helper}, lambda: {target}{rest}"), + ) + } + + /// wrap `expr` in the scalar check (`_soundness_check` / + /// `_soundness_parametric`) named by `check` + fn wrap_check(&mut self, expr: &Expr, check: &SiteCheck) { + let (helper, leading, trailing) = + self.checking(check, ["_soundness_check", "_soundness_parametric"]); + self.wrap_call(expr, &helper, &leading, &trailing); } /// wrap an iterable in a validating generator whose form (shallow vs /// parametric) follows the element's check plan - fn wrap_iteration(&mut self, iterable: &Expr, is_async: bool, plan: &SoundnessCheck) { - let (helper, trailing) = match plan { - SoundnessCheck::Isinstance(target) => { - let helper = if is_async { - self.used_aiter = true; - "_soundness_aiter" - } else { - self.used_iter = true; - "_soundness_iter" - }; - (helper, format!(", {}", self.target(target))) - } - SoundnessCheck::Parametric { alias, variances } => { - self.used_parametric = true; - let helper = if is_async { - self.used_aiter_p = true; - "_soundness_aiter_p" - } else { - self.used_iter_p = true; - "_soundness_iter_p" - }; - ( - helper, - format!( - ", {}, {}", - self.written.spelled(alias), - variance_tuple(variances) - ), - ) - } + fn wrap_iteration(&mut self, iterable: &Expr, is_async: bool, check: &SiteCheck) { + let helpers = if is_async { + ["_soundness_aiter", "_soundness_aiter_p"] + } else { + ["_soundness_iter", "_soundness_iter_p"] }; - self.wrap_call(iterable, helper, &trailing); + match (&check.plan, is_async) { + (SoundnessCheck::Isinstance(_), false) => self.used_iter = true, + (SoundnessCheck::Isinstance(_), true) => self.used_aiter = true, + (SoundnessCheck::Parametric { .. }, false) => self.used_iter_p = true, + (SoundnessCheck::Parametric { .. }, true) => self.used_aiter_p = true, + } + let (helper, leading, trailing) = self.checking(check, helpers); + self.wrap_call(iterable, &helper, &leading, &trailing); } /// the guard statements that validate each of `checks`, in order - fn guard_stmts(&mut self, checks: &[(Name, SoundnessCheck)]) -> Vec { + fn guard_stmts(&mut self, checks: &[(Name, SiteCheck)]) -> Vec { checks .iter() - .map(|(name, plan)| self.guard_stmt(name, plan)) + .map(|(name, check)| self.guard_stmt(name, check)) .collect() } - /// the guard statement that validates parameter `name` against `plan` - fn guard_stmt(&mut self, name: &str, plan: &SoundnessCheck) -> String { - match plan { - SoundnessCheck::Isinstance(target) => { - format!("_soundness_check({name}, {})", self.target(target)) - } - SoundnessCheck::Parametric { alias, variances } => { - self.used_parametric = true; - format!( - "_soundness_parametric({name}, {}, {})", - self.written.spelled(alias), - variance_tuple(variances) - ) - } - } + /// the guard statement that validates parameter `name` against `check` + fn guard_stmt(&mut self, name: &str, check: &SiteCheck) -> String { + let (helper, leading, trailing) = + self.checking(check, ["_soundness_check", "_soundness_parametric"]); + format!("{helper}({leading}{name}{trailing})") } /// insert the entry guards at the top of `func`'s body, after any docstring @@ -400,7 +411,7 @@ impl<'src> Rendered<'src> { &mut self, source: &str, func: &StmtFunctionDef, - checks: &[(Name, SoundnessCheck)], + checks: &[(Name, SiteCheck)], ) { let guards: Vec = self .guard_stmts(checks) @@ -462,12 +473,12 @@ impl TypeAwarePass for SoundnessPass<'_> { let mut checks = Vec::new(); for site in &sites { match site { - Site::Value { expr, plan } => rendered.wrap_check(expr, plan), + Site::Value { expr, check } => rendered.wrap_check(expr, check), Site::Elements { iterable, is_async, - plan, - } => rendered.wrap_iteration(iterable, *is_async, plan), + check, + } => rendered.wrap_iteration(iterable, *is_async, check), Site::Parameters { function, guards } => { checks.push(RerenderedCheck::Guards { function: function.range(), @@ -487,6 +498,7 @@ impl TypeAwarePass for SoundnessPass<'_> { inner.used_aiter |= rendered.used_aiter; inner.used_iter_p |= rendered.used_iter_p; inner.used_aiter_p |= rendered.used_aiter_p; + inner.used_once_bound |= rendered.used_once_bound; inner.used_parametric |= rendered.used_parametric; if !checks.is_empty() { rerendered.push((idx, checks)); @@ -495,12 +507,12 @@ impl TypeAwarePass for SoundnessPass<'_> { } for site in &sites { match site { - Site::Value { expr, plan } => inner.wrap_check(expr, plan), + Site::Value { expr, check } => inner.wrap_check(expr, check), Site::Elements { iterable, is_async, - plan, - } => inner.wrap_iteration(iterable, *is_async, plan), + check, + } => inner.wrap_iteration(iterable, *is_async, check), Site::Parameters { function, guards } => { inner.insert_param_guards(self.source, function, guards); } @@ -535,6 +547,9 @@ impl TypeAwarePass for SoundnessPass<'_> { if inner.used_aiter_p { ctx.runtime.insert(crate::runtime::SOUNDNESS_AITER_P); } + if inner.used_once_bound { + ctx.runtime.insert(crate::runtime::SOUNDNESS_ONCE_BOUND); + } ctx.template_edits .extend(inner.wraps.into_iter().map(|(range, before, after)| { ( @@ -706,6 +721,8 @@ impl Transformer for Placing { #[cfg(test)] mod tests { + use indoc::indoc; + use crate::{Config, transpile}; fn soundness_config() -> Config { @@ -922,6 +939,130 @@ mod tests { ); } + /// a function may be called before or after the module reaches a class declared below + /// it, so a check in one looks the class up as it runs. a check at the module's own level + /// runs in order, where the class is not there yet, and is left out + #[test] + fn a_class_declared_below_is_looked_up_where_the_check_runs() { + let out = check_with( + indoc! {" + def t[T]() -> T: ... + + class Holder: + later: list[Later] + + def first(self) -> Later: + return self.later[0] + + def each(self) -> None: + for later in self.later: + pass + + def take(value: Later) -> None: ... + + early: Later = t() + + class Later: ... + "}, + crate::SoundnessPositions::all(), + ); + assert!( + out.contains( + r#"return _soundness_once_bound(("Later",), self.later[0], _soundness_check, lambda: Later)"# + ), + "got:\n{out}" + ); + assert!( + out.contains( + r#"for later in _soundness_once_bound(("Later",), self.later, _soundness_iter, lambda: Later):"# + ), + "got:\n{out}" + ); + assert!( + out.contains( + r#"_soundness_once_bound(("Later",), value, _soundness_check, lambda: Later)"# + ), + "got:\n{out}" + ); + assert!(out.contains(r#"early: "Later" = t()"#), "got:\n{out}"); + assert!(out.contains("def _soundness_once_bound"), "got:\n{out}"); + } + + /// the lookup finds whatever the name holds, so it is made only where every binding of + /// the name binds the one class: not where the module binds it to something else first, + /// binds a second class under it, binds it only for the checker, unbinds it, or rebinds + /// it from a function through `global` + #[test] + fn a_name_bound_to_anything_but_the_class_is_not_looked_up() { + let out = check(indoc! {" + from typing import TYPE_CHECKING + + class Other: ... + + def t[T]() -> T: ... + + def first() -> Later: + return t() + + def second() -> Twice: + return t() + + def third() -> Checked: + return t() + + def fourth() -> Gone: + return t() + + def fifth() -> Rebound: + return t() + + def rebind() -> None: + global Rebound + Rebound = None + + Later = None + + class Later: ... + + Twice = Other + + class Twice: ... + + if TYPE_CHECKING: + from decimal import Decimal as Checked + else: + class Checked: ... + + class Gone: ... + + del Gone + + class Rebound: ... + "}); + assert!(!out.contains("_soundness_once_bound("), "got:\n{out}"); + assert!(!out.contains("_soundness_check(t()"), "got:\n{out}"); + } + + /// a `private` class is emitted under another name, and the lookup is keyed by the name + /// the module emits it under + #[test] + fn a_private_class_is_looked_up_by_the_name_it_is_emitted_under() { + let out = check(indoc! {" + def t[T]() -> T: ... + + def make() -> Kept: + return t() + + private class Kept: ... + "}); + assert!( + out.contains( + r#"_soundness_once_bound(("_Kept",), t(), _soundness_check, lambda: _Kept)"# + ), + "got:\n{out}" + ); + } + #[test] fn shadowed_builtin_skips_check() { let out = check("str = 1\ndef t[T]() -> T: ...\ndef f():\n a: str = t()\n"); diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index da92e77cc4..867fd1c09c 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -848,6 +848,9 @@ pub(crate) trait TypeInfo { /// re-export of the ty-side check plan so transforms name a single type pub(crate) use ty_python_semantic::types::soundness::CheckKind as SoundnessCheck; +/// re-export of the ty-side check a soundness site makes, with the names it waits on +pub(crate) use ty_python_semantic::types::soundness_sites::SiteCheck; + /// re-export of the ty-side checked-cast plan (a superset of [`SoundnessCheck`] /// that also carries protocol-structural and unchecked cases) pub(crate) use ty_python_semantic::types::soundness::CastCheck; diff --git a/crates/by_transforms/tests/type_checking_import_runtime.rs b/crates/by_transforms/tests/type_checking_import_runtime.rs index 15121a87a0..3b0cd2d0b3 100644 --- a/crates/by_transforms/tests/type_checking_import_runtime.rs +++ b/crates/by_transforms/tests/type_checking_import_runtime.rs @@ -5,7 +5,7 @@ //! lowered code reads it — under `TYPE_CHECKING` that is how an import cycle is broken. A //! lowering that writes a type into code that runs (a specialization the checker solved, a //! soundness check's target, a cast's type) must not write that name, or the module raises -//! `NameError` with `by check` clean. +//! `NameError` with `by check` clean, unless it looks the name up as it runs. mod common; @@ -188,6 +188,62 @@ assert ledger.in_match == ledger.in_negated_alias == set() print("ok") "#; +/// a check in a function on a class the module declares further down: the function can run +/// before the class exists, when the value passes unchecked, and after, when it is checked +const LATER_PROGRAM: &str = r#" +def loose(value: object) -> dynamic: + return value + + +def first(values: list[dynamic]) -> Later: + return values[0] + + +def make(value: object) -> Kept: + return loose(value) + + +assert first([1]) == 1 + + +class Later: + def __init__(self, size: int): + self.size = size + + +private class Kept: ... + + +class Holder: + later: list[Later] + + def __init__(self, later: list[dynamic]): + self.later = later + + def total(self) -> int: + total = 0 + for later in self.later: + total += later.size + return total + + +def refused(run) -> bool: + try: + run() + except TypeError: + return True + return False + + +assert Holder([Later(1), Later(2)]).total() == 3 +assert refused(lambda: Holder([1]).total()) +assert refused(lambda: first([1])) +assert first([Later(3)]).size == 3 +assert refused(lambda: make(1)) +assert make(Kept()) is not None +print("ok") +"#; + #[test] fn a_class_imported_only_for_the_checker_is_never_evaluated() { run(PROGRAM); @@ -203,6 +259,11 @@ fn a_name_bound_past_a_condition_reading_type_checking_is_never_evaluated() { run(CONDITION_PROGRAM); } +#[test] +fn a_class_declared_further_down_is_checked_once_it_is_bound() { + run(LATER_PROGRAM); +} + /// a module importing a name another module binds one way for the checker and another way as /// it runs reads what the program binds, which need not be the class the checker resolved #[test] diff --git a/crates/ty_python_semantic/src/place.rs b/crates/ty_python_semantic/src/place.rs index 40c6fdb81f..f4016ce88b 100644 --- a/crates/ty_python_semantic/src/place.rs +++ b/crates/ty_python_semantic/src/place.rs @@ -668,6 +668,27 @@ pub(crate) fn runtime_binding<'db>( } } +/// basedpython: whether every binding `file` makes of the module-level `name` binds one and +/// the same class, so that whenever the program finds the name bound it holds that class. a +/// name that is not yet bound, or bound only on some paths, passes: what this answers is what +/// the name holds once it is there +pub(crate) fn binds_one_class_wherever_bound<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + name: &str, +) -> bool { + let Some(bindings) = bindings_alone(db, file, name) else { + return false; + }; + let mut classes = bindings + .into_iter() + .map(|definition| binding_type(db, definition)); + let Some(Type::ClassLiteral(class)) = classes.next() else { + return false; + }; + classes.all(|bound| bound == Type::ClassLiteral(class)) +} + /// whether every binding `file` makes of the module-level `name` is made by a statement at the /// top of the module that always makes it once it completes — an import, a `class`, a `def`, /// an assignment — rather than one inside an `if`, a `try` or an expression. the module runs diff --git a/crates/ty_python_semantic/src/types/runtime_spelling.rs b/crates/ty_python_semantic/src/types/runtime_spelling.rs index e3cc06334e..fdf8378f47 100644 --- a/crates/ty_python_semantic/src/types/runtime_spelling.rs +++ b/crates/ty_python_semantic/src/types/runtime_spelling.rs @@ -36,6 +36,10 @@ impl RuntimeSite { range: node.range(), } } + + pub(crate) fn range(self) -> TextRange { + self.range + } } /// a python expression the checker spells for the transpiled program, which evaluates in @@ -93,6 +97,20 @@ impl RuntimeSpelling { }) } + /// every binding of the module's global scope the spelling reads, each with the spelling + /// of that read alone + pub(crate) fn global_read_spellings(&self) -> impl Iterator { + self.parts.iter().filter_map(|part| match part { + SpellingPart::Global(name) | SpellingPart::ModulePrivate(name) => Some(( + name, + Self { + parts: vec![part.clone()], + }, + )), + SpellingPart::Text(_) => None, + }) + } + /// the first module-level name the spelling reads that does not hold, where `site` runs, /// the binding the checker resolved — with what it holds instead pub(crate) fn unbound_read<'db>( diff --git a/crates/ty_python_semantic/src/types/soundness_sites.rs b/crates/ty_python_semantic/src/types/soundness_sites.rs index cac3283c25..3954350d8f 100644 --- a/crates/ty_python_semantic/src/types/soundness_sites.rs +++ b/crates/ty_python_semantic/src/types/soundness_sites.rs @@ -9,10 +9,13 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Comprehension, Expr, ExprCall, Parameter, Stmt, StmtFunctionDef, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; +use ty_python_core::scope::ScopeKind; +use ty_python_core::semantic_index; use crate::Db; +use crate::place::{RuntimeBinding, RuntimeRead, binds_one_class_wherever_bound, runtime_binding}; use crate::semantic_model::{HasType, SemanticModel}; -use crate::types::runtime_spelling::RuntimeSite; +use crate::types::runtime_spelling::{RuntimeSite, RuntimeSpelling}; use crate::types::soundness::{ ArgSelector, CheckKind, CheckTarget, call_result_is_typevar_derived, is_specialized_generic_instance, parameter_check_plan, parameter_runtime_check_plan, @@ -129,18 +132,18 @@ enum ArgSlot<'a> { #[derive(Debug, Clone)] pub enum SoundnessSite<'ast> { /// the value `expr` produces is checked as it is produced - Value { expr: &'ast Expr, plan: CheckKind }, + Value { expr: &'ast Expr, check: SiteCheck }, /// each element drawn from `iterable` is checked as it is drawn Elements { iterable: &'ast Expr, is_async: bool, - plan: CheckKind, + check: SiteCheck, }, /// `function`'s own parameters are checked where its body begins, each in the order /// it is written Parameters { function: &'ast StmtFunctionDef, - guards: Vec<(&'ast Parameter, CheckKind)>, + guards: Vec<(&'ast Parameter, SiteCheck)>, }, } @@ -148,12 +151,27 @@ impl SoundnessSite<'_> { /// every check the site makes pub(crate) fn plans(&self) -> Vec<&CheckKind> { match self { - Self::Value { plan, .. } | Self::Elements { plan, .. } => vec![plan], - Self::Parameters { guards, .. } => guards.iter().map(|(_, plan)| plan).collect(), + Self::Value { check, .. } | Self::Elements { check, .. } => vec![&check.plan], + Self::Parameters { guards, .. } => { + guards.iter().map(|(_, check)| &check.plan).collect() + } } } } +/// a check a site makes, and the module-level names it reads that the program may not have +/// bound yet when it runs +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SiteCheck { + pub plan: CheckKind, + /// each name the check's target reads that is bound only further down the module, or + /// only on some paths, as a spelling of its own. every binding the module makes of one + /// binds the class the checker resolved, so once all of them are bound the target is the + /// one the checker named; the check is made only then, and the value passes unchecked + /// before. empty for a check whose names are bound wherever it runs + pub once_bound: Vec, +} + /// where the checks go, decided once over a syntax tree, and what is not a place /// for one struct Soundness<'a, 'db, 'ast> { @@ -271,24 +289,65 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { .filter(|plan| !matches!(plan, CheckKind::Isinstance(CheckTarget::NoneType))) } - /// whether every name `plan`'s target reads holds, where `site` runs, the binding the - /// checker resolved. a check that would read one that does not — a class imported only - /// under `if TYPE_CHECKING:`, or declared further on — is left out, as a check with no - /// faithful target is - fn runs_at(&self, plan: &CheckKind, site: Option) -> bool { - site.is_some_and(|site| { - plan.spelling() - .unbound_read(self.db(), &self.env(), self.model.file(), site) - .is_none() + /// the check `plan` makes at `site`, if it can make one there: every name its target + /// reads must hold, where `site` runs, the binding the checker resolved. a check that would + /// read one that does not — a class imported only under `if TYPE_CHECKING:`, or declared + /// further on — is left out, as a check with no faithful target is. + /// + /// code in a function runs whenever the function is called, which can be before or after + /// the module binds a class declared further down. there a name the module binds only to + /// one class, and never to anything else, is looked up as the check runs instead + /// ([`SiteCheck::once_bound`]), and the check is made once it is bound + fn site_check( + &self, + plan: CheckKind, + site: RuntimeSite, + runs_later: bool, + ) -> Option { + let db = self.db(); + let file = self.model.file(); + let program_file = db.program_file(file); + let mut once_bound = Vec::new(); + for (name, read) in plan.spelling().global_read_spellings() { + match runtime_binding( + db, + &self.env(), + program_file, + RuntimeRead::Global { + name, + range: site.range(), + }, + ) { + RuntimeBinding::Bound => {} + RuntimeBinding::Unbound | RuntimeBinding::NotTheCheckers + if runs_later && binds_one_class_wherever_bound(db, program_file, name) => + { + once_bound.push(read); + } + RuntimeBinding::Unbound | RuntimeBinding::NotTheCheckers => return None, + } + } + Some(SiteCheck { plan, once_bound }) + } + + /// whether `expr` runs only once a function or a lambda holding it is called, rather than + /// as the module or a class body reaches it + fn runs_later(&self, expr: &Expr) -> bool { + let index = semantic_index(self.db(), self.db().program_file(self.model.file())); + index.try_expression_scope_id(expr).is_some_and(|scope| { + index + .ancestor_scopes(scope) + .any(|(_, scope)| matches!(scope.kind(), ScopeKind::Function | ScopeKind::Lambda)) }) } /// check the value `expr` produces against `plan` fn check_value(&mut self, expr: &'ast Expr, plan: CheckKind) { - if !self.runs_at(&plan, Some(RuntimeSite::at(expr))) { + let Some(check) = self.site_check(plan, RuntimeSite::at(expr), self.runs_later(expr)) + else { return; - } - self.sites.push(SoundnessSite::Value { expr, plan }); + }; + self.sites.push(SoundnessSite::Value { expr, check }); } /// check each element drawn from the iterable of a `for` / comprehension @@ -301,13 +360,15 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { let Some(plan) = self.check_plan(target_expr) else { return; }; - if !self.runs_at(&plan, Some(RuntimeSite::at(iterable))) { + let Some(check) = + self.site_check(plan, RuntimeSite::at(iterable), self.runs_later(iterable)) + else { return; - } + }; self.sites.push(SoundnessSite::Elements { iterable, is_async, - plan, + check, }); } @@ -377,10 +438,11 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { .chain(¶ms.kwonlyargs) { let parameter = &pwd.parameter; + // the guards run where the function's body begins, when it is called if let Some(plan) = self.parameter_plan(parameter) - && self.runs_at(&plan, Some(RuntimeSite::at(func))) + && let Some(check) = self.site_check(plan, RuntimeSite::at(func), true) { - guards.push((parameter, plan)); + guards.push((parameter, check)); } } if !guards.is_empty() { diff --git a/docs/basedpython/features/soundness.md b/docs/basedpython/features/soundness.md index aeab8bf48d..088946f8c4 100644 --- a/docs/basedpython/features/soundness.md +++ b/docs/basedpython/features/soundness.md @@ -117,9 +117,36 @@ further down the module than the check, and one bound in the `else:` of a class the checker resolved. a pure-`None` result is skipped too: validating a value that carries no data guards nothing -a check in a function or a method runs whenever it is called, so there the -class has to be bound above the `def` or `class` statement at the top of the -module that holds the function, which can be called as soon as that runs +a check in a function or a method runs whenever it is called, which can be +before or after the module reaches a class declared further down. where every +binding the module makes of the name binds that one class — none binds anything +else, none is made only for the checker, nothing unbinds it, and no function +rebinds it through `global` — the check looks the name up as it runs, and is +made once the name is bound; until then the value passes unchecked. where that +does not hold, the class has to be bound above the `def` or `class` statement +at the top of the module that holds the function, which can be called as soon +as that runs: + +```by +class Modifier: + cuts: list[Cut] + + def total(self) -> int: + total = 0 + for cut in self.cuts: + total += cut.size + return total + +class Cut: + size: int +``` + +→ `for cut in _soundness_once_bound(("Cut",), self.cuts, _soundness_iter, lambda: Cut):` + +the lookup is by the name the class is emitted under, so a `private class Cut` +is looked up as `"_Cut"`. a check at the module's own level runs in order, so +it is made only below the class it names. `by compile` leaves a function with +such a check to its interpreted definition ## deep checks for user generics From c309ebaa7108fa144befceeab60aaed4addb6268 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:31:42 +1000 Subject: [PATCH 33/47] show a parameter default that is re-evaluated per call as a marker whose `repr` is the default as written, so `inspect.signature` reports `geysers: int = ` rather than one anonymous `object()` shared by every such parameter; a required parameter after a defaulted one shows as ``. the guard tests the marker's class (`if type(geysers) is _MISSING:`), and a marker handed back as an argument, as `BoundArguments.apply_defaults` hands it, is still the argument not given. a structural protocol test against a method whose unannotated parameter has such a default raises rather than read the marker's type as the parameter's, and no longer reads a required parameter after a default as optional --- crates/by_transforms/src/runtime.rs | 1 + .../by_transforms/src/runtime/_by_runtime.py | 71 +++- .../src/transforms/ast_driver.rs | 30 +- .../by_transforms/src/transforms/coalesce.rs | 32 +- .../src/transforms/decorator_keyword.rs | 15 +- .../src/transforms/init_method.rs | 75 ++-- .../src/transforms/mutable_defaults.rs | 393 +++++++++++++----- .../src/transforms/repeated_underscore.rs | 16 +- .../by_transforms/src/transforms/rerender.rs | 4 +- .../src/transforms/type_reification.rs | 2 +- .../src/transforms/unique_loop_bindings.rs | 20 +- .../tests/mutable_defaults_runtime.rs | 260 ++++++++++++ .../tests/parametric_is_runtime.rs | 71 ++++ docs/basedpython/features/init-method.md | 4 +- docs/basedpython/features/mutable-defaults.md | 72 ++++ .../features/parametric-type-tests.md | 23 + docs/basedpython/features/trailing-lambdas.md | 5 +- 17 files changed, 911 insertions(+), 183 deletions(-) diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs index c285b5ff29..7e0c258a8a 100644 --- a/crates/by_transforms/src/runtime.rs +++ b/crates/by_transforms/src/runtime.rs @@ -103,6 +103,7 @@ helpers! { OPTIONAL = "Optional", FORCE_UNWRAP = "_force_unwrap", MAIN_ARGS = "_by_main_args", + DEFAULT = "_by_default", RAISES = "_by_raises", LAZY_MODULE = "_lazy_module", LAZY_ATTR = "_lazy_attr", diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py index 78ca777e53..f73412db3e 100644 --- a/crates/by_transforms/src/runtime/_by_runtime.py +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -104,6 +104,36 @@ def _force_unwrap(_v): return _v +# --- parameter defaults ---------------------------------------------------- +# a default re-evaluated per call has no value until a call evaluates it, so the +# signature holds a marker in its place and the body tests for it. each default has +# a marker of its own, bound at module level under a name of the transpiler's, and +# the body's guard tests for that one object. its `repr` is the default as +# basedpython reads it, which is what `inspect.signature` shows; `` stands +# for a parameter python cannot declare required where it stands. a marker handed +# back as an argument, as `apply_defaults` hands it, is the argument not given +# +# a marker reduces to the name it is bound under, so a pickled one is read back from +# that name, and a copy of one — which `copy` makes through the same reduction — is +# the marker itself. a tool that copies or pickles a signature's defaults hands back +# the object the guard tests for + + +class _by_default: + def __init__(self, source, module, name): + self._by_source = source + self.__module__ = module + self._by_name = name + + def __repr__(self): + if self._by_source is None: + return "" + return f"" + + def __reduce__(self): + return self._by_name + + # --- effects, entry points and loop capture -------------------------------- @@ -943,6 +973,27 @@ def _by_method_matches(klass, name, params, ret): hints = typing.get_type_hints(method) except Exception: return False + import sys + + # a marker is the object its module binds under the name it carries. no attribute + # of its own says so — a class of the program's may define any of them — and a + # value is read without assuming it has any of them + def marker_source(value): + try: + module = sys.modules.get(value.__module__) + if module is None or getattr(module, value._by_name) is not value: + return False, None + return True, value._by_source + except Exception: + return False, None + + # a required parameter after a default carries a marker in place of a default + def has_default(p): + if p.default is inspect.Parameter.empty: + return False + marker, source = marker_source(p.default) + return not (marker and source is None) + positional = [ p for p in signature.parameters.values() if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) @@ -954,7 +1005,7 @@ def _by_method_matches(klass, name, params, ret): # extra positional parameters the protocol doesn't supply must be optional, # else a caller matching the protocol would fail to provide them for p in positional[len(params):]: - if p.default is inspect.Parameter.empty: + if not has_default(p): return False # likewise any required keyword-only parameter would break a protocol call for p in signature.parameters.values(): @@ -963,11 +1014,23 @@ def _by_method_matches(klass, name, params, ret): for (expected, variance), p in zip(params, positional): if p.name in hints: actual = hints[p.name] - elif p.default is not inspect.Parameter.empty: + elif not has_default(p): + return False + elif marker_source(p.default)[0]: + if variance == 3: + continue + # the default is evaluated per call, so the signature holds a marker + # and nothing at runtime records the type the checker read off the + # default. the marker's own type would answer for it, wrongly + raise TypeError( + "cannot check `" + klass.__qualname__ + "." + name + "` against a " + + "parameterized protocol: parameter `" + p.name + "` has no " + + "annotation, and its default is evaluated per call, so nothing " + + "records its type at runtime. annotate it" + ) + else: # a reified default gives the parameter's inferred type at runtime actual = type(p.default) - else: - return False if not _by_variance_ok(actual, expected, variance): return False if ret is not None: diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 5e9a7fd73b..fe8a2af6fa 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -406,10 +406,10 @@ fn is_wrapper(frags: &[Fragment], start: usize, end: usize) -> bool { /// /// A [`SubPatch::Relocating`] edit is the one thing that does *not* travel with /// that source. It is the stand-in a pass leaves at the span it moved a -/// construct away from — `_MISSING` where the default was written — and a -/// passthrough of exactly that span, from outside the template's own range, is -/// the place the construct moved *to*. Materializing the stand-in there too -/// would put `_MISSING` at both ends and the construct at neither, which is how +/// construct away from — the `_MISSING` marker where the default was written — +/// and a passthrough of exactly that span, from outside the template's own range, +/// is the place the construct moved *to*. Materializing the stand-in there too +/// would put the marker at both ends and the construct at neither, which is how /// a `decorator def` whose header template re-emits the whole signature came to /// emit `if tags is _MISSING: tags = _MISSING`. fn template_claimees( @@ -1128,7 +1128,7 @@ pub(crate) fn merge_into_own_imports(body: &mut String, imports: Vec) -> /// Coalesce repeated `from import X` lines into a single /// `from import X, Y, ...` line. Preserves any non-matching -/// lines (e.g. `import foo`, `_MISSING = object()`) in their original +/// lines (e.g. `import foo`, a synthesized class) in their original /// order. Names within a merged line are sorted and deduped fn merge_from_imports(lines: Vec) -> (Vec, Vec) { // preserve first-seen module order so tests that depend on specific @@ -1231,6 +1231,13 @@ pub(crate) fn run_against_source<'a>( // tokens go when the syntax tree is taken out of the parse let accessor_value_ranges = properties::collect_value_ranges(&parsed.syntax().body, parsed.tokens()); + // a default's marker shows the default on one line and without its comments, which + // only the tokens tell apart from the text of a string + let default_markers = mutable_defaults::DefaultMarkers::collect( + &parsed.syntax().body, + parsed.tokens(), + source_ref, + ); // a statement an AST pass rewrote is re-emitted in the source's own indentation, // since the lines it passes through keep theirs let indentation = ruff_python_codegen::Stylist::from_tokens(parsed.tokens(), source_ref) @@ -1357,12 +1364,17 @@ pub(crate) fn run_against_source<'a>( let dedent_string_pass = dedent_string::DedentString::new(source_ref); let super_keyword_pass = super_keyword::SuperKeyword::new(written); let postfix_await_pass = postfix_await::PostfixAwait::new(source_ref); - let mutable_defaults_pass = - mutable_defaults::MutableDefaultsPass::new(source_ref, written, config.is_stub); + let mutable_defaults_pass = mutable_defaults::MutableDefaultsPass::new( + source_ref, + written, + &default_markers, + config.is_stub, + ); let unique_loop_bindings_pass = unique_loop_bindings::UniqueLoopBindingsPass::new(source_ref, config.unique_loop_bindings); let auto_quote_pass = auto_quote::AutoQuote::new(source_ref); - let init_method_pass = init_method::InitMethod::new(source_ref, written, config.clone()); + let init_method_pass = + init_method::InitMethod::new(source_ref, written, &default_markers, config.clone()); let properties_pass = properties::PropertiesPass::new( source_ref, written, @@ -1730,7 +1742,7 @@ pub(crate) fn run_against_source<'a>( // …); receiver spans pass through as `Src` fragments, so sibling // lowerings inside compose &grapheme_string_pass, - // mutable defaults → `_MISSING` sentinel swap + body-prologue guard; + // mutable defaults → a `_MISSING` marker per default + body-prologue guard; // narrow edits, so the function body's own lowerings still apply &mutable_defaults_pass, // per-iteration loop bindings: a closure made in a loop body is applied diff --git a/crates/by_transforms/src/transforms/coalesce.rs b/crates/by_transforms/src/transforms/coalesce.rs index 2ce5303108..ce0ef0f1bb 100644 --- a/crates/by_transforms/src/transforms/coalesce.rs +++ b/crates/by_transforms/src/transforms/coalesce.rs @@ -225,6 +225,16 @@ mod tests { ); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + #[test] fn basic_coalesce() { check("x = a ?? b\n", "x = a if a is not None else b\n"); @@ -319,7 +329,7 @@ mod tests { /// which printed the signature again over this edit and left the program refused #[test] fn coalesce_in_a_parameter_default() { - check( + check_beside_runtime( indoc::indoc! {" g: int? = None @@ -328,7 +338,8 @@ mod tests { "}, indoc::indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"g ?? 1\", __name__, \"_MISSING\") g: int | None = None def f(y: int = _MISSING) -> int: @@ -342,28 +353,20 @@ mod tests { /// the same default with the body written on the clause's line #[test] fn coalesce_in_a_parameter_default_with_an_inline_body() { - check( + check_beside_runtime( indoc::indoc! {" g: int? = None def f(y: int = g ?? 1) -> int: return y "}, - indoc::indoc! {" - from typing import Any - _MISSING: Any = object() - g: int | None = None - - def f(y: int = _MISSING) -> int: \n if y is _MISSING: - y = g if g is not None else 1 - return y - "}, + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"g ?? 1\", __name__, \"_MISSING\")\ng: int | None = None\n\ndef f(y: int = _MISSING) -> int: \n if y is _MISSING:\n y = g if g is not None else 1\n return y\n", ); } /// and a chain longer than one link, which recurses in the same edit #[test] fn a_chained_coalesce_in_a_parameter_default() { - check( + check_beside_runtime( indoc::indoc! {" g: int? = None h: int? = None @@ -373,7 +376,8 @@ mod tests { "}, indoc::indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"g ?? h ?? 1\", __name__, \"_MISSING\") g: int | None = None h: int | None = None diff --git a/crates/by_transforms/src/transforms/decorator_keyword.rs b/crates/by_transforms/src/transforms/decorator_keyword.rs index 8b550a498f..e0948c38ad 100644 --- a/crates/by_transforms/src/transforms/decorator_keyword.rs +++ b/crates/by_transforms/src/transforms/decorator_keyword.rs @@ -535,6 +535,16 @@ mod tests { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + /// the options are passed as `@d(option=...)`, so a repeated `_` among them would /// be keyword-only, and the name only a keyword reaches it by is not the author's #[test] @@ -837,14 +847,15 @@ mod tests { /// and `@d` raised `TypeError: object of type 'object' has no len()` #[test] fn a_mutable_option_default_reaches_the_guard_and_not_the_signature() { - check( + check_beside_runtime( indoc! {" decorator def d(fn: (...) -> object, tags: list[str] = []) -> int: return len(tags) "}, indoc! {" from typing import Any, Callable, overload - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") @overload def d(fn: Callable[..., object], *, tags: list[str] = ...) -> int: ... @overload diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index 7db32a13b6..6f03e7ad62 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -31,7 +31,9 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::callable::lower_type_expr_full; -use super::mutable_defaults::{parameter_guards, undeclarable_error}; +use super::mutable_defaults::{ + DefaultMarkers, define_markers, parameter_guards, undeclarable_error, +}; use super::repeated_underscore::WrittenNames; use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; use super::type_expr_walker::RootKind; @@ -42,14 +44,21 @@ use crate::type_info::TypeInfo; pub(crate) struct InitMethod<'src> { source: &'src str, written: WrittenNames<'src>, + markers: &'src DefaultMarkers, config: Config, } impl<'src> InitMethod<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + markers: &'src DefaultMarkers, + config: Config, + ) -> Self { Self { source, written, + markers, config, } } @@ -68,15 +77,13 @@ impl TypeAwarePass for InitMethod<'_> { templates: RefCell::new(Vec::new()), relocating: RefCell::new(Vec::new()), errors: RefCell::new(Vec::new()), - needs_missing: RefCell::new(false), + markers: self.markers, + definitions: RefCell::new(Vec::new()), }; for stmt in stmts { state.visit_stmt(stmt); } - if state.needs_missing.into_inner() { - ctx.required_imports - .extend(super::mutable_defaults::sentinel_definition(self.written)); - } + define_markers(self.written, state.definitions.into_inner(), ctx); ctx.text_edits.extend(state.edits.into_inner()); ctx.template_edits.extend(state.templates.into_inner()); ctx.relocating_edits.extend(state.relocating.into_inner()); @@ -145,8 +152,9 @@ struct State<'src> { /// the `_MISSING` substitutions, whose defaults the guards re-evaluate relocating: RefCell)>>, errors: RefCell>, - /// whether the `_MISSING` sentinel reached the output - needs_missing: RefCell, + markers: &'src DefaultMarkers, + /// the lines binding the `_MISSING` markers the generated signatures name + definitions: RefCell>, } impl State<'_> { @@ -383,12 +391,13 @@ impl State<'_> { sentinels, written, guards, + definitions, undeclarable, } = parameter_guards( self.source, func, self.written, - &super::mutable_defaults::sentinel_name(self.written), + self.markers, self.types, self.config.is_stub, ); @@ -417,6 +426,7 @@ impl State<'_> { // are written whether or not the body gains anything self.relocating.borrow_mut().extend(sentinels); self.templates.borrow_mut().extend(written); + self.definitions.borrow_mut().extend(definitions); let pos = func.range.end(); if guards.is_empty() { // no passthrough to carry, so emit plain text: a template @@ -425,12 +435,6 @@ impl State<'_> { self.push(TextRange::new(pos, pos), literal_text(&frags)); return; } - if guards - .iter() - .any(super::mutable_defaults::Guard::uses_sentinel) - { - *self.needs_missing.borrow_mut() = true; - } self.templates .borrow_mut() .push((TextRange::new(pos, pos), frags)); @@ -456,6 +460,16 @@ mod tests { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + #[test] fn a_declaring_parameter_lowers_its_annotation_once() { // the parser synthesizes `self.a: T = a` from the parameter, with the @@ -811,7 +825,7 @@ mod tests { /// stored as the sentinel #[test] fn a_declaration_follows_the_guard_for_its_own_default() { - check( + check_beside_runtime( indoc! {" class A: init(let a: list[int] = []): @@ -820,7 +834,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class A: def __init__(self, a: list[int] = _MISSING): \"\"\"doc\"\"\" @@ -857,20 +872,12 @@ mod tests { /// write at the offset after the docstring, in the order they run #[test] fn two_passes_write_under_a_docstring_on_the_clause_line() { - check( + check_beside_runtime( indoc! {" class A: init(let a: list[int] = []): \"\"\"doc\"\"\" "}, - indoc! {" - from typing import Any - _MISSING: Any = object() - class A: - def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\" - if a is _MISSING: - a = [] - self.a: list[int] = a - "}, + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\nclass A:\n def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\"\n if a is _MISSING:\n a = []\n self.a: list[int] = a\n", ); } @@ -879,20 +886,12 @@ mod tests { /// that follows it #[test] fn a_docstring_followed_by_statements_on_the_clause_line_is_given_a_line_of_its_own() { - check( + check_beside_runtime( indoc! {" class A: init(let a: list[int] = []): \"\"\"doc\"\"\"; print(a) "}, - indoc! {" - from typing import Any - _MISSING: Any = object() - class A: - def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\"; \n if a is _MISSING: - a = [] - \n self.a: list[int] = a - print(a) - "}, + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\nclass A:\n def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\"; \n if a is _MISSING:\n a = []\n \n self.a: list[int] = a\n print(a)\n", ); } diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index a9afa2c25d..62d649ee76 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -1,14 +1,26 @@ //! Text-edit pass: replaces non-scalar default arguments with a `_MISSING` -//! sentinel and injects a guard at the top of each function body. +//! marker and injects a guard at the top of each function body. //! -//! def f(x=[]): → def f(x=_MISSING): -//! ... if x is _MISSING: -//! x = [] -//! ... +//! ```text +//! def f(x=[]): → _MISSING: Any = _by_default("[]", __name__, "_MISSING") +//! ... def f(x=_MISSING): +//! if x is _MISSING: +//! x = [] +//! ... +//! ``` //! //! Only number, bool, None, string, and ellipsis literals (and unary +/- //! on a number) are kept as-is; everything else is re-evaluated per call. //! +//! Such a default has no value until a call evaluates it, so nothing truthful can +//! stand in the signature as its value. The marker standing there instead carries +//! the default's text, which its `repr` shows, so `inspect.signature` reports +//! `x=` rather than an anonymous object. Each default has a marker of +//! its own, bound at module level ([`DefaultMarkers`]), and its guard tests for that +//! one object, so a marker handed back as an argument — `BoundArguments.apply_defaults` +//! does — is the argument not given, as leaving it out is, while another default's +//! marker is a value like any other. +//! //! The same sentinel machinery lowers basedpython's relaxed parameter order — //! a `def` may declare a required parameter after a defaulted one (so a //! trailing lambda can bind the last parameter while earlier parameters keep @@ -44,7 +56,11 @@ //! not have read yet. A dropped default is silent when the reading is wrong, so the //! python → basedpython direction leaves every written default where it is. +use std::cell::RefCell; +use std::collections::HashMap; + use ruff_python_ast::helpers::is_immutable_scalar_default; +use ruff_python_ast::token::{TokenKind, Tokens}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, ParameterWithDefault, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -52,11 +68,12 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::repeated_underscore::{WrittenNames, reads_underscore, rebound_underscore}; use super::source_util::{ - PrologueStatement, body_prologue, first_body_statement, standalone_expression, + PrologueStatement, body_prologue, first_body_statement, python_string_literal, + standalone_expression, }; use crate::type_info::TypeInfo; -/// what a `_MISSING`-sentinel guard does when the argument was not supplied +/// what a `_MISSING`-marker guard does when the argument was not supplied pub(crate) enum Guard { /// re-evaluate the written default per call (mutable defaults). the default /// is carried as its source *range*, not its text: the guard re-emits it @@ -65,14 +82,14 @@ pub(crate) enum Guard { /// rather than being dropped with the signature they came from Reevaluate { name: String, - sentinel: String, + marker: String, default: Vec, }, /// raise — the parameter is required, its sentinel default only exists /// because python rejects a required parameter after a defaulted one Required { name: String, - sentinel: String, + marker: String, function: String, /// the name the builtin `TypeError` is written under type_error: String, @@ -84,37 +101,27 @@ pub(crate) enum Guard { Rebind { parameter: String }, } -impl Guard { - /// whether the guard tests for the `_MISSING` sentinel, which the output then defines - pub(crate) fn uses_sentinel(&self) -> bool { - match self { - Guard::Reevaluate { .. } | Guard::Required { .. } => true, - Guard::Rebind { .. } => false, - } - } -} - impl PrologueStatement for Guard { fn push(&self, frags: &mut Vec, base: &str) { match self { Guard::Reevaluate { name, - sentinel, + marker, default, } => { frags.push(Fragment::Lit(format!( - "if {name} is {sentinel}:\n{base} {name} = " + "if {name} is {marker}:\n{base} {name} = " ))); frags.extend(default.iter().cloned()); } Guard::Required { name, - sentinel, + marker, function, type_error, } => { frags.push(Fragment::Lit(format!( - "if {name} is {sentinel}:\n{base} raise {type_error}(\"{function}() missing required argument: '{name}'\")" + "if {name} is {marker}:\n{base} raise {type_error}(\"{function}() missing required argument: '{name}'\")" ))); } Guard::Rebind { parameter } => { @@ -127,13 +134,15 @@ impl PrologueStatement for Guard { struct MutableDefaults<'src> { source: &'src str, written: WrittenNames<'src>, + markers: &'src DefaultMarkers, types: &'src dyn TypeInfo, edits: Vec<(TextRange, Vec)>, /// the `_MISSING` substitutions, whose defaults the guards re-evaluate relocating: Vec<(TextRange, Vec)>, /// the guard suites, anchored at the body statement they precede guards: Vec<(TextSize, Vec)>, - used: bool, + /// the lines binding the markers the module's signatures name + definitions: Vec, /// functions whose body starts with parser-synthesized statements, so a /// guard has no source position to anchor to unanchored: Vec, @@ -143,41 +152,150 @@ struct MutableDefaults<'src> { undeclarable: Vec<(String, String)>, } -/// replace a default with the sentinel. a template rather than plain text so -/// it *absorbs* the zero-width insertions a lowering anchored to the default's -/// first token left behind (`_force_unwrap(`), which a plain-text replacement -/// leaves stranded in the signature. the guard re-emits them -fn sentinel_edit(default: TextRange, sentinel: &str) -> (TextRange, Vec) { - (default, vec![Fragment::Lit(sentinel.to_owned())]) +/// replace a default with its marker. a template rather than plain text so it +/// *absorbs* the zero-width insertions a lowering anchored to the default's first +/// token left behind (`_force_unwrap(`), which a plain-text replacement leaves +/// stranded in the signature. the guard re-emits them +fn sentinel_edit(default: TextRange, marker: &str) -> (TextRange, Vec) { + (default, vec![Fragment::Lit(marker.to_owned())]) } -/// The name the sentinel goes into the output under. -/// -/// It stands for "no argument was given", so it has to be a value nothing else can be — -/// which a name the module binds itself is not. Taken past whatever the module spells, so a -/// module that writes `_MISSING` of its own keeps it and the sentinel goes somewhere else. -pub(crate) fn sentinel_name(written: WrittenNames) -> String { - written.fresh("_MISSING") +/// what a marker stands in for +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum MarkerSite { + /// the default at this range, re-evaluated per call + Default(TextRange), + /// the parameter at this range, required after a defaulted one + Required(TextRange), } -/// The lines the output needs before it can use the sentinel: the sentinel itself, and the -/// import its annotation names. +/// The marker standing in the signature for each default re-evaluated per call, and for +/// each required parameter after a defaulted one: one object per site, bound at module +/// level, which the site's own guard tests for by identity. /// -/// The sentinel stands where the source wrote an annotated default, so an unannotated -/// `object()` there is a default the parameter cannot hold and a checker reading the output -/// says so. Declaring it `Any` is how typeshed declares its own sentinels, and it costs the -/// output nothing: the value never reaches the body, whose guard replaces it before anything -/// reads it. -pub(crate) fn sentinel_definition(written: WrittenNames) -> [String; 2] { - [ - written.import_from("typing", &["Any"]), - format!( - "{}: {} = {}()", - sentinel_name(written), +/// A marker stands for "no argument was given" at exactly one parameter. Shared between +/// sites, one function's marker read off its signature would be an argument not given to +/// any other, so each site gets its own, under a name nothing else in the module has — +/// `_MISSING`, `_MISSING2`, … in the order the lowerings ask for them, which is the same +/// for every transpile of the module. The guard's test is then the one identity test the +/// sentinel always was, at the same cost per call. +/// +/// The text a marker shows is the default as basedpython reads it — after the +/// lowerings that run ahead of this pass, such as the qualification of an enum member — +/// on one line: the comments in it are dropped, and a line break between two of its +/// tokens becomes a space, or nothing just inside a bracket. A string literal keeps its +/// own text, a triple-quoted one its own line breaks. +#[derive(Debug, Default)] +pub(crate) struct DefaultMarkers { + /// the text each default's marker shows, by the default's range + texts: HashMap, + /// each site named so far, with its name + named: RefCell>, +} + +impl DefaultMarkers { + /// the text of every parameter default `module` writes, read off `tokens`, the tokens of + /// `source` + pub(crate) fn collect(module: &[Stmt], tokens: &Tokens, source: &str) -> Self { + struct Defaults<'a> { + tokens: &'a Tokens, + source: &'a str, + texts: HashMap, + } + + impl<'a> Visitor<'a> for Defaults<'a> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::FunctionDef(f) = stmt { + for default in f + .parameters + .iter_non_variadic_params() + .filter_map(|parameter| parameter.default.as_deref()) + { + self.texts.insert( + default.range(), + one_line(self.tokens, self.source, default.range()), + ); + } + } + walk_stmt(self, stmt); + } + } + + let mut defaults = Defaults { + tokens, + source, + texts: HashMap::new(), + }; + defaults.visit_body(module); + Self { + texts: defaults.texts, + named: RefCell::default(), + } + } + + /// the name `site`'s marker is bound under, and the line binding it + fn marker(&self, written: WrittenNames, site: MarkerSite, source: &str) -> (String, String) { + let mut named = self.named.borrow_mut(); + let name = if let Some((_, name)) = named.iter().find(|(named, _)| *named == site) { + name.clone() + } else { + let name = written.fresh_outside("_MISSING", |candidate| { + named.iter().any(|(_, name)| name == candidate) + }); + named.push((site, name.clone())); + name + }; + let text = match site { + MarkerSite::Default(range) => python_string_literal( + self.texts + .get(&range) + .map_or(&source[range], String::as_str), + ), + MarkerSite::Required(_) => "None".to_owned(), + }; + let definition = format!( + "{name}: {} = {}({text}, __name__, \"{name}\")", written.imported("typing", "Any"), - written.builtin("object") - ), - ] + crate::runtime::DEFAULT.name() + ); + (name, definition) + } +} + +/// the text of `range`, an expression in `source`, on one line and without its comments +fn one_line(tokens: &Tokens, source: &str, range: TextRange) -> String { + let mut text = String::new(); + let mut previous: Option<(TokenKind, TextSize)> = None; + let mut broken = false; + for token in tokens.in_range(range) { + match token.kind() { + TokenKind::Comment => {} + TokenKind::NonLogicalNewline | TokenKind::Newline => broken = true, + kind => { + if let Some((before, end)) = previous { + let gap = &source[TextRange::new(end, token.start())]; + // a backslash continuation is a line break too + if broken || gap.contains(['\n', '\r']) { + let opens = matches!( + before, + TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace + ); + let closes = + matches!(kind, TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace); + if !opens && !closes { + text.push(' '); + } + } else { + text.push_str(gap); + } + } + text.push_str(&source[token.range()]); + previous = Some((kind, token.end())); + broken = false; + } + } + } + text } /// A default written into the signature at the end of the parameter it belongs to. @@ -208,6 +326,9 @@ pub(crate) struct ParameterGuards { /// relocate nothing; they add text at the end of a parameter pub(crate) written: Vec<(TextRange, Vec)>, pub(crate) guards: Vec, + /// the lines binding the markers the signature names, which the module defines ahead + /// of everything that reads them + pub(crate) definitions: Vec, /// in a stub, the parameters python cannot declare where they stand: a /// required one after a defaulted one. python rejects the shape, and the /// sentinel default a module gets for it would declare the parameter @@ -223,14 +344,20 @@ pub(crate) fn parameter_guards( source: &str, f: &StmtFunctionDef, written_names: WrittenNames, - sentinel: &str, + markers: &DefaultMarkers, types: &dyn TypeInfo, is_stub: bool, ) -> ParameterGuards { let mut sentinels = Vec::new(); let mut written = Vec::new(); let mut guards = Vec::new(); + let mut definitions = Vec::new(); let mut undeclarable = Vec::new(); + let mut marker = |site| { + let (name, definition) = markers.marker(written_names, site, source); + definitions.push(definition); + name + }; let params = f.parameters.as_ref(); // positional parameters: swap non-scalar defaults for the sentinel, and give // basedpython's required-after-defaulted parameters a sentinel default plus @@ -242,11 +369,12 @@ pub(crate) fn parameter_guards( Some(d) => { seen_default = true; if !is_stub && !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) { - sentinels.push(sentinel_edit(d.range(), sentinel)); + let marker = marker(MarkerSite::Default(d.range())); + sentinels.push(sentinel_edit(d.range(), &marker)); guards.push(Guard::Reevaluate { name: crate::python_parameter_name(params, &pw.parameter, written_names) .to_string(), - sentinel: sentinel.to_owned(), + marker, default: standalone_expression(source, d.range()), }); } @@ -263,11 +391,12 @@ pub(crate) fn parameter_guards( ); } None if seen_default => { - written.push(written_default(pw, sentinel)); + let marker = marker(MarkerSite::Required(pw.range())); + written.push(written_default(pw, &marker)); guards.push(Guard::Required { name: crate::python_parameter_name(params, &pw.parameter, written_names) .to_string(), - sentinel: sentinel.to_owned(), + marker, function: f.name.id.to_string(), type_error: written_names.builtin("TypeError"), }); @@ -278,11 +407,12 @@ pub(crate) fn parameter_guards( for pw in ¶ms.kwonlyargs { match pw.default.as_deref() { Some(d) if !is_stub && !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) => { - sentinels.push(sentinel_edit(d.range(), sentinel)); + let marker = marker(MarkerSite::Default(d.range())); + sentinels.push(sentinel_edit(d.range(), &marker)); guards.push(Guard::Reevaluate { name: crate::python_parameter_name(params, &pw.parameter, written_names) .to_string(), - sentinel: sentinel.to_owned(), + marker, default: standalone_expression(source, d.range()), }); } @@ -306,6 +436,7 @@ pub(crate) fn parameter_guards( sentinels, written, guards, + definitions, undeclarable, } } @@ -378,15 +509,17 @@ impl MutableDefaults<'_> { sentinels, written, guards, + definitions, undeclarable, } = parameter_guards( self.source, f, self.written, - &sentinel_name(self.written), + self.markers, self.types, self.is_stub, ); + self.definitions.extend(definitions); self.undeclarable.extend( undeclarable .into_iter() @@ -397,7 +530,6 @@ impl MutableDefaults<'_> { if guards.is_empty() { return; } - self.used |= guards.iter().any(Guard::uses_sentinel); // the guards go at the start of the first body statement the source // actually wrote @@ -423,29 +555,60 @@ impl<'ast> Visitor<'ast> for MutableDefaults<'_> { pub(crate) struct MutableDefaultsPass<'src> { source: &'src str, written: WrittenNames<'src>, + markers: &'src DefaultMarkers, is_stub: bool, } impl<'src> MutableDefaultsPass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, is_stub: bool) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + markers: &'src DefaultMarkers, + is_stub: bool, + ) -> Self { Self { source, written, + markers, is_stub, } } } +/// Define `definitions`, the lines binding markers a module's signatures name, ahead of +/// everything that reads them, with the import their annotation names and the runtime +/// class they are made of. +/// +/// A marker stands where the source wrote an annotated default, so a marker of a class of +/// its own there is a default the parameter cannot hold and a checker reading the output +/// says so. Declaring each marker `Any` is how typeshed declares its own sentinels, and it +/// costs the output nothing: a marker never reaches the body, whose guard replaces it +/// before anything reads it. +pub(crate) fn define_markers( + written: WrittenNames, + definitions: Vec, + ctx: &mut PassContext, +) { + if definitions.is_empty() { + return; + } + ctx.runtime.insert(crate::runtime::DEFAULT); + ctx.required_imports + .push(written.import_from("typing", &["Any"])); + ctx.required_imports.extend(definitions); +} + impl TypeAwarePass for MutableDefaultsPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = MutableDefaults { source: self.source, written: self.written, + markers: self.markers, types, edits: Vec::new(), relocating: Vec::new(), guards: Vec::new(), - used: false, + definitions: Vec::new(), unanchored: Vec::new(), is_stub: self.is_stub, undeclarable: Vec::new(), @@ -468,10 +631,7 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { )); return; } - if inner.used { - ctx.required_imports - .extend(sentinel_definition(self.written)); - } + define_markers(self.written, inner.definitions, ctx); ctx.template_edits.extend(inner.edits); ctx.relocating_edits.extend(inner.relocating); ctx.statement_inserts.extend(inner.guards); @@ -488,9 +648,13 @@ mod tests { check_at(crate::Config::test_default().min_version, input, expected); } + /// the runtime is taken from a module written beside the output, as a build writes it, + /// so an expectation shows the import of the helper that makes the markers rather than + /// its definition fn check_at(min_version: crate::PythonVersion, input: &str, expected: &str) { let config = crate::Config { min_version, + runtime_module: Some("_by_runtime".to_owned()), ..crate::Config::test_default() }; assert_eq!( @@ -587,7 +751,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class A: def f(self, a = _MISSING): if a is _MISSING: @@ -643,7 +808,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -657,7 +823,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -678,7 +845,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(None, __name__, \"_MISSING\") class S: def __init__(self, first: int = 1, second: str = _MISSING): if second is _MISSING: @@ -701,7 +869,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -735,7 +904,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") class S: def __init__(self, items: list[int] = _MISSING) -> None: if items is _MISSING: @@ -754,7 +924,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") def f(x=_MISSING): if x is _MISSING: x = [] @@ -772,7 +943,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"{}\", __name__, \"_MISSING\") def f(x=_MISSING): if x is _MISSING: x = {} @@ -790,7 +962,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"{1, 2}\", __name__, \"_MISSING\") def f(x=_MISSING): if x is _MISSING: x = {1, 2} @@ -844,11 +1017,13 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() - def f(x=_MISSING, y=_MISSING): + from _by_runtime import _by_default + _MISSING2: Any = _by_default(\"{}\", __name__, \"_MISSING2\") + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") + def f(x=_MISSING, y=_MISSING2): if x is _MISSING: x = [] - if y is _MISSING: + if y is _MISSING2: y = {} pass "}, @@ -865,7 +1040,8 @@ mod tests { "#}, indoc! {r#" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default("[]", __name__, "_MISSING") def f(x=_MISSING): """doc""" if x is _MISSING: @@ -883,7 +1059,7 @@ mod tests { fn a_docstring_on_the_def_line_is_given_a_line_of_its_own() { check( "def f(x=[]): \"\"\"doc\"\"\"\n", - "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"\n if x is _MISSING:\n x = []\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"\n if x is _MISSING:\n x = []\n", ); } @@ -897,7 +1073,7 @@ mod tests { fn a_docstring_followed_by_statements_on_the_def_line_is_given_a_line_of_its_own() { check( "def f(x=[]): \"\"\"doc\"\"\"; x.append(1); return x\n", - "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", ); } @@ -907,7 +1083,7 @@ mod tests { fn statements_on_the_def_line_without_a_docstring_take_one_break() { check( "def f(x=[]): x.append(1); return x\n", - "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n x.append(1); return x\n", ); } @@ -917,7 +1093,7 @@ mod tests { fn a_method_body_on_the_clause_line_is_broken_at_its_own_indentation() { check( "class C:\n def m(self, x=[]): \"\"\"doc\"\"\"; x.append(1); return x\n", - "from typing import Any\n_MISSING: Any = object()\nclass C:\n def m(self, x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\nclass C:\n def m(self, x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", ); } @@ -941,13 +1117,15 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING2: Any = _by_default(\"{}\", __name__, \"_MISSING2\") + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") def f(x=_MISSING): if x is _MISSING: x = [] pass - def g(y=_MISSING): - if y is _MISSING: + def g(y=_MISSING2): + if y is _MISSING2: y = {} pass "}, @@ -963,7 +1141,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(None, __name__, \"_MISSING\") def f(x=1, a=_MISSING): if a is _MISSING: raise TypeError(\"f() missing required argument: 'a'\") @@ -981,7 +1160,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(None, __name__, \"_MISSING\") def f(x: int = 1, a: int = _MISSING): if a is _MISSING: raise TypeError(\"f() missing required argument: 'a'\") @@ -1001,11 +1181,13 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() - def f(x=_MISSING, a=_MISSING): + from _by_runtime import _by_default + _MISSING2: Any = _by_default(None, __name__, \"_MISSING2\") + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") + def f(x=_MISSING, a=_MISSING2): if x is _MISSING: x = [] - if a is _MISSING: + if a is _MISSING2: raise TypeError(\"f() missing required argument: 'a'\") print(x, a) "}, @@ -1038,7 +1220,8 @@ mod tests { "#}, indoc! {r#" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default("f\"asdf{data}\"", __name__, "_MISSING") data = "fdsa" def f(a=_MISSING): if a is _MISSING: @@ -1060,7 +1243,8 @@ mod tests { "#}, indoc! {r#" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default("t\"asdf{data}\"", __name__, "_MISSING") data = "fdsa" def f(a=_MISSING): if a is _MISSING: @@ -1085,7 +1269,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"a + 1\", __name__, \"_MISSING\") def f(a, b = _MISSING): if b is _MISSING: b = a + 1 @@ -1108,7 +1293,7 @@ mod tests { a: int = [] ) -> int: ... "}, - "from typing import Any\n_MISSING: Any = object()\ndef f(\n a: int = _MISSING\n) -> int: \n if a is _MISSING:\n a = []\n ...\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(\n a: int = _MISSING\n) -> int: \n if a is _MISSING:\n a = []\n ...\n", ); } @@ -1116,7 +1301,7 @@ mod tests { fn inline_ellipsis_body() { check( "def f(x=[]): ...", - "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n ...", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\")\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n ...", ); } @@ -1133,7 +1318,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[1 is int]\", __name__, \"_MISSING\") def f(x = _MISSING): if x is _MISSING: x = [True] @@ -1155,7 +1341,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"1 is int\", __name__, \"_MISSING\") def f(x = _MISSING): if x is _MISSING: x = True @@ -1201,7 +1388,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default + _MISSING: Any = _by_default(\"[]\", __name__, \"_MISSING\") def f(xs: list[int] = _MISSING) -> int: if xs is _MISSING: xs = [] @@ -1280,7 +1468,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING2: Any = object() + from _by_runtime import _by_default + _MISSING2: Any = _by_default(\"[]\", __name__, \"_MISSING2\") _MISSING = 5 def f(xs: list[int] = _MISSING2) -> int: diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index edda10f9d0..01f094e555 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -568,7 +568,7 @@ impl<'src> WrittenNames<'src> { } /// [`Self::fresh`], also passing over every name `taken` answers for - fn fresh_outside(self, stem: &str, taken: impl Fn(&str) -> bool) -> String { + pub(crate) fn fresh_outside(self, stem: &str, taken: impl Fn(&str) -> bool) -> String { self.names .fresh_outside(stem, |name| self.lowering_binds(name) || taken(name)) } @@ -1100,6 +1100,16 @@ mod tests { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + fn refused(input: &str) -> String { transpile(input, &Config::test_default()).unwrap_err() } @@ -1302,9 +1312,9 @@ mod tests { /// that re-evaluates `f`'s mutable default there #[test] fn a_numbered_name_skips_a_name_the_module_binds() { - check( + check_beside_runtime( "_2: list[int] = [9]\n\ndef f(_: int, _: int, x: list[int] = _2) -> list[int]:\n return x\n\ndef g(_: int, _: int) -> list[int]:\n return _2\n", - "from typing import Any\n_MISSING: Any = object()\n_2: list[int] = [9]\n\ndef f(_: int, _3: int, /, x: list[int] = _MISSING) -> list[int]:\n if x is _MISSING:\n x = _2\n return x\n\ndef g(_: int, _3: int, /) -> list[int]:\n return _2\n", + "from typing import Any\nfrom _by_runtime import _by_default\n_MISSING: Any = _by_default(\"_2\", __name__, \"_MISSING\")\n_2: list[int] = [9]\n\ndef f(_: int, _3: int, /, x: list[int] = _MISSING) -> list[int]:\n if x is _MISSING:\n x = _2\n return x\n\ndef g(_: int, _3: int, /) -> list[int]:\n return _2\n", ); } diff --git a/crates/by_transforms/src/transforms/rerender.rs b/crates/by_transforms/src/transforms/rerender.rs index 8a563e9174..fff310b77c 100644 --- a/crates/by_transforms/src/transforms/rerender.rs +++ b/crates/by_transforms/src/transforms/rerender.rs @@ -1848,9 +1848,9 @@ mod tests { def f(_: list[int] = [], _: list[int] = []) -> int: return len(_) "}); - assert!(out.contains("if _ is _MISSING:"), "got:\n{out}"); + assert!(out.contains("if _ is _MISSING"), "got:\n{out}"); assert!( - out.contains("if _2 is _MISSING:\n _2 = []"), + out.contains("if _2 is _MISSING2:\n _2 = []"), "got:\n{out}" ); } diff --git a/crates/by_transforms/src/transforms/type_reification.rs b/crates/by_transforms/src/transforms/type_reification.rs index f807e8fa96..badd81be1d 100644 --- a/crates/by_transforms/src/transforms/type_reification.rs +++ b/crates/by_transforms/src/transforms/type_reification.rs @@ -524,7 +524,7 @@ mod tests { "}); assert!( out.contains("def f(x=_MISSING):"), - "default must stay a bare sentinel: {out}" + "default must stay a bare marker: {out}" ); assert!( !out.contains("](_MISSING"), diff --git a/crates/by_transforms/src/transforms/unique_loop_bindings.rs b/crates/by_transforms/src/transforms/unique_loop_bindings.rs index 2249b480e3..c142ddd5e8 100644 --- a/crates/by_transforms/src/transforms/unique_loop_bindings.rs +++ b/crates/by_transforms/src/transforms/unique_loop_bindings.rs @@ -574,6 +574,16 @@ mod tests { ); } + /// the runtime imported from a module written beside the output, as a build writes it, + /// so an expectation names the helpers it uses rather than spelling out their definitions + fn check_beside_runtime(input: &str, expected: &str) { + let config = Config { + runtime_module: Some("_by_runtime".to_owned()), + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + /// the decorated form emits the rebind runtime; the tests below assert /// everything else. exactly the runtime's text is removed, so whatever else /// the preamble holds is still asserted @@ -646,7 +656,7 @@ mod tests { /// of a loop binding there is a closure read and has to be rebound with it #[test] fn a_default_reading_the_target_is_rebound() { - check_body( + check_beside_runtime( indoc! {" def build(): for i in items: @@ -655,7 +665,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default, _by_loop_bind + _MISSING: Any = _by_default(\"i\", __name__, \"_MISSING\") def build(): for i in items: @_by_loop_bind(i=i) @@ -818,7 +829,7 @@ mod tests { /// binding the rebind froze #[test] fn a_closure_in_a_parameter_default_is_left_to_the_body_guard() { - check_body( + check_beside_runtime( indoc! {" def build(): for i in items: @@ -827,7 +838,8 @@ mod tests { "}, indoc! {" from typing import Any - _MISSING: Any = object() + from _by_runtime import _by_default, _by_loop_bind + _MISSING: Any = _by_default(\"lambda: i\", __name__, \"_MISSING\") def build(): for i in items: @_by_loop_bind(i=i) diff --git a/crates/by_transforms/tests/mutable_defaults_runtime.rs b/crates/by_transforms/tests/mutable_defaults_runtime.rs index bbbdcde00f..1c66cded4a 100644 --- a/crates/by_transforms/tests/mutable_defaults_runtime.rs +++ b/crates/by_transforms/tests/mutable_defaults_runtime.rs @@ -144,6 +144,194 @@ assert Holder().xs == [1, 2], "so is one of a bodyless `init`" print("ok") "#; +/// a default that names a value stands in the signature as a marker whose `repr` is the +/// default as written — there is no value to put there, because the call reads the name as +/// it runs. handed back, as `apply_defaults` hands it, the marker means the argument was +/// not given +const NAME_DEFAULT: &str = r#" +import inspect + +EARLY = 2 + +def f(x: int = EARLY) -> int: + return x + +signature = inspect.signature(f) +assert repr(signature.parameters["x"].default) == "", repr(signature.parameters["x"].default) +assert str(signature) == "(x: int = ) -> int", str(signature) + +EARLY = 3 +bound = signature.bind() +bound.apply_defaults() +assert f(*bound.args, **bound.kwargs) == 3, "the default handed back is read as the call runs" +assert f() == 3, "and so is one left out" + +print("ok") +"#; + +/// a call is evaluated afresh by every call that leaves its parameter out, and by every +/// call handed the marker +const CALL_DEFAULT: &str = r#" +import inspect + +def make() -> list[int]: + return [] + +def f(items: list[int] = make()) -> list[int]: + items.append(1) + return items + +signature = inspect.signature(f) +assert repr(signature.parameters["items"].default) == "", repr(signature.parameters["items"].default) + +bound = signature.bind() +bound.apply_defaults() +assert f(*bound.args, **bound.kwargs) == [1], "the marker handed back is a fresh call" +assert f(*bound.args, **bound.kwargs) == [1], "every time" + +print("ok") +"#; + +/// a marker copied or pickled — a tool that keeps a signature's defaults may do either — +/// comes back as the very marker, and still means the argument was not given +const COPIED_MARKER: &str = r#" +import copy +import inspect +import pickle + +def f(items: list[int] = []) -> list[int]: + items.append(1) + return items + +default = inspect.signature(f).parameters["items"].default +assert copy.copy(default) === default, "a copy is the marker" +assert copy.deepcopy(default) === default, "so is a deep copy" +assert pickle.loads(pickle.dumps(default)) === default, "and a pickled one" +assert f(**{"items": pickle.loads(pickle.dumps(default))}) == [1], "which is the default" + +print("ok") +"#; + +/// each default has a marker of its own: one handed to another parameter is a value there, +/// as any other object would be +const ANOTHER_DEFAULTS_MARKER: &str = r#" +import inspect + +def f(items: list[int] = []) -> list[int]: + return items + +def g(value: object = []) -> object: + return value + +marker = inspect.signature(f).parameters["items"].default +assert g(**{"value": marker}) === marker, "another default's marker is a value" +assert g() == [], "and the parameter's own default still applies" + +print("ok") +"#; + +/// a marker shows its default on one line, without the comments written in it +const MULTI_LINE_MARKER: &str = r#" +import inspect + +def f(xs: list[int] = [ + 1, # one + 2, +], ys: list[int] = ( + [3] + + [4] +)) -> list[int]: + return xs + ys + +parameters = inspect.signature(f).parameters +assert repr(parameters["xs"].default) == "", repr(parameters["xs"].default) +assert repr(parameters["ys"].default) == "", repr(parameters["ys"].default) +assert f() == [1, 2, 3, 4] + +print("ok") +"#; + +/// a default that reads an earlier parameter has no value until the call that supplies +/// that parameter — the marker is all there is to show +const LATE_BOUND_DEFAULT: &str = r#" +import inspect + +def f(a: int, b: int = a + 1) -> int: + return b + +signature = inspect.signature(f) +assert repr(signature.parameters["b"].default) == "", repr(signature.parameters["b"].default) + +bound = signature.bind(5) +bound.apply_defaults() +assert f(*bound.args, **bound.kwargs) == 6, "the default is computed from the argument given" + +print("ok") +"#; + +/// the constructor an `init(let ...)` declares shows its defaults the same way +const INIT_FIELD_DEFAULT: &str = r#" +import inspect + +EARLY = 2 + +class Base: + init(let geysers: int = EARLY) + +default = inspect.signature(Base.__init__).parameters["geysers"].default +assert repr(default) == "", repr(default) + +signature = inspect.signature(Base) + +bound = signature.bind() +bound.apply_defaults() +assert Base(*bound.args, **bound.kwargs).geysers == 2, "the marker handed back is the default" + +EARLY = 5 +assert Base(*bound.args, **bound.kwargs).geysers == 5, "read as the call runs" +assert Base().geysers == 5, "and so is one left out" + +print("ok") +"#; + +/// a parameter python would reject — required, after a defaulted one — stands in the +/// signature with a marker that says it is required, and handed back it is the missing +/// argument it stands for +const REQUIRED_AFTER_DEFAULT: &str = r#" +import inspect + +def f(x: int = 1, a: int) -> int: + return x + a + +signature = inspect.signature(f) +assert repr(signature.parameters["a"].default) == "", repr(signature.parameters["a"].default) + +bound = signature.bind() +bound.apply_defaults() +try: + f(*bound.args, **bound.kwargs) +except TypeError as error: + assert "missing required argument: 'a'" in str(error), str(error) +else: + raise AssertionError("a required argument handed back as its marker was accepted") + +print("ok") +"#; + +/// a scalar default is python's own, so the signature holds its value +const SCALAR_DEFAULT: &str = r#" +import inspect + +def f(x: int = 3, s: str = "a") -> int: + return x + +parameters = inspect.signature(f).parameters +assert parameters["x"].default == 3, repr(parameters["x"].default) +assert parameters["s"].default == "a", repr(parameters["s"].default) + +print("ok") +"#; + fn run(python: &str, program: &str) { let config = Config { min_version: PythonVersion::PY313, @@ -244,3 +432,75 @@ fn a_leading_bom_stays_ahead_of_the_preamble() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } } + +#[test] +fn a_name_default_shows_as_written_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, NAME_DEFAULT); +} + +#[test] +fn a_call_default_shows_as_written_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, CALL_DEFAULT); +} + +#[test] +fn a_late_bound_default_shows_as_written_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, LATE_BOUND_DEFAULT); +} + +#[test] +fn an_init_field_default_shows_as_written_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, INIT_FIELD_DEFAULT); +} + +#[test] +fn a_required_parameter_after_a_default_shows_as_required_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, REQUIRED_AFTER_DEFAULT); +} + +#[test] +fn a_scalar_default_shows_its_value_in_the_signature() { + let Some(python) = python() else { + return; + }; + run(&python, SCALAR_DEFAULT); +} + +#[test] +fn a_copied_or_pickled_marker_is_still_the_default() { + let Some(python) = python() else { + return; + }; + run(&python, COPIED_MARKER); +} + +#[test] +fn another_defaults_marker_is_a_value() { + let Some(python) = python() else { + return; + }; + run(&python, ANOTHER_DEFAULTS_MARKER); +} + +#[test] +fn a_marker_shows_its_default_on_one_line() { + let Some(python) = python() else { + return; + }; + run(&python, MULTI_LINE_MARKER); +} diff --git a/crates/by_transforms/tests/parametric_is_runtime.rs b/crates/by_transforms/tests/parametric_is_runtime.rs index c9fe2c9ecb..a82a1c7e71 100644 --- a/crates/by_transforms/tests/parametric_is_runtime.rs +++ b/crates/by_transforms/tests/parametric_is_runtime.rs @@ -142,6 +142,26 @@ assert (FeedBool() is Feed[int]) is False, "param contravariant: bool does not a assert (FeedBool() is Feed[bool]) is True, "param matches bool" assert (C() is Feed[bool]) is False, "value without the method fails" +# a default re-evaluated per call has no value before the call, so nothing at runtime +# records the type the checker reads off it — the marker standing in the signature +# is not that type. the check refuses to answer rather than answer from the marker +class FeedLate: + def f(self, other = C()): + pass + +def refused(test) -> bool: + try: + test() + except TypeError as error: + return "per call" in str(error) + return False + +def feeds_bool(value: object) -> bool: + return value is Feed[bool] + +assert refused(lambda: FeedLate() is Feed[C]), "the check refuses the type the default would give" +assert refused(lambda: feeds_bool(FeedLate())), "and one it would not" + # an impl with an extra *required* parameter the protocol doesn't supply can't # satisfy it — a protocol-shaped call would fail to provide that argument class ExtraRequired: @@ -150,6 +170,57 @@ class ExtraRequired: assert (ExtraRequired() is Feed[bool]) is False, "extra required parameter fails" +# so does a required one after a default, which python cannot declare and which stands +# in the signature as a marker in place of a default +class ExtraRequiredAfterDefault: + def f(self, other = True, extra): + pass + +assert feeds_bool(ExtraRequiredAfterDefault()) is False, "a required parameter after a default fails" + +# a required parameter after a default the protocol does supply is required, not a +# default evaluated per call: an unannotated one says nothing about its type, so the +# check answers `False` rather than refusing +class FeedSecond[in out T](Protocol): + def f(self, a: int, other: T): ... + +class ReqAfterDefault: + def f(self, a = 1, other): + pass + +def feeds_second_bool(value: object) -> bool: + return value is FeedSecond[bool] + +assert feeds_second_bool(ReqAfterDefault()) is False, "a required parameter is not a late-bound default" + +# a class of the program's own may carry every attribute a marker has, and a default +# that is one of its instances is still a value, read by its type: a marker is only the +# object its module binds under the name it carries. written in plain python, so the +# default stands in the signature as written +_plain = {"__name__": "__main__"} +exec(""" +class Tagged: + _by_marker = True + _by_name = "Tagged" + +class Forged: + _by_name = "Forged" + _by_source = None + +class TakesTagged: + def f(self, other=Tagged()): + pass + +class TakesForged: + def f(self, other: bool = True, extra=Forged()): + pass +""", _plain) +Tagged = _plain["Tagged"] +Forged = _plain["Forged"] + +assert feeds_bool(_plain["TakesTagged"]()) is False, "a value with a marker's attributes is a value" +assert feeds_bool(_plain["TakesForged"]()) is True, "and a default, not a required parameter's marker" + # a method return is covariant: an impl returning `bool` satisfies a protocol # asking for `int` (bool <: int), but not one asking for `str` class Get[in out T](Protocol): diff --git a/docs/basedpython/features/init-method.md b/docs/basedpython/features/init-method.md index 11b3a3b254..e1c2052ca7 100644 --- a/docs/basedpython/features/init-method.md +++ b/docs/basedpython/features/init-method.md @@ -60,10 +60,10 @@ class Fight: ```python class Fight: - def __init__(self, seen: set[int] = _MISSING, last: int = _MISSING): + def __init__(self, seen: set[int] = _MISSING, last: int = _MISSING2): if seen is _MISSING: seen = set() - if last is _MISSING: + if last is _MISSING2: raise TypeError("__init__() missing required argument: 'last'") self.seen: set[int] = seen self.last: int = last diff --git a/docs/basedpython/features/mutable-defaults.md b/docs/basedpython/features/mutable-defaults.md index fef170d0bb..ee82f9800e 100644 --- a/docs/basedpython/features/mutable-defaults.md +++ b/docs/basedpython/features/mutable-defaults.md @@ -20,6 +20,78 @@ everything else is rewritten so the default expression runs at call time. a useful side effect: `def g(a, b=a + 1)` becomes valid late-bound default syntax, with `b` computed fresh from the actual `a` at each call +a default that names a value is read when the call runs, too, not where the +`def` stands: + +```by +LIMIT = 2 + +def cap(n=LIMIT): + return n + +LIMIT = 3 +cap() # 3, where python's own default would still be 2 +``` + +## what `inspect.signature` shows + +a default re-evaluated per call has no value until a call evaluates it, so the +signature holds a marker in its place, and the marker's `repr` is the default as +basedpython reads it: + +```by +import inspect + +LIMIT = 2 + +def f(a: int, b: list[int] = [], c: int = LIMIT, d: int = 0): + ... + +print(inspect.signature(f)) +# (a: int, b: list[int] = , c: int = , d: int = 0) +``` + +transpiles to: + +```python +_MISSING: Any = _by_default("[]", __name__, "_MISSING") +_MISSING2: Any = _by_default("LIMIT", __name__, "_MISSING2") + +def f(a: int, b: list[int] = _MISSING, c: int = _MISSING2, d: int = 0): + if b is _MISSING: + b = [] + if c is _MISSING2: + c = LIMIT + ... +``` + +- a scalar default is python's own, so the signature holds its value: `d` above + is `0` +- a marker is not the value. `parameters["c"].default == 2` is `False`, and no + value could be honest there: `LIMIT` is whatever it is when the call runs, + `[]` is a new list each time, and a default reading an earlier parameter, + such as `b=a + 1`, has none without an `a` +- the text is the default after the rewrites basedpython makes ahead of this one, + on one line: `= Red` naming an enum member shows as ``, a + `??` stays `??`, the comments written inside a default are dropped, and a line + break between two of its tokens becomes a space, or nothing just inside a + bracket. a string literal keeps its own text, line breaks and all +- each default has a marker of its own. handed back as an argument, a marker is + the argument not given: calling `f` with the arguments + `BoundArguments.apply_defaults()` filled in evaluates each default as leaving + the parameter out would, compiled with [`by compile`](../cli-reference.md#by-compile) or + not. handed to any other parameter, it is a value like any other. a `.by` call + passing one gets the [runtime check](soundness.md) any other value would, and + a marker is not an `int` +- a copy of a marker is the marker, and a pickled one comes back as the marker it + was, read from the name the module binds it under +- a required parameter after a defaulted one, which python cannot declare where + it stands, shows as ``. handed back, it raises the `TypeError` + leaving it out does +- a parameter with no annotation and such a default records no type at runtime, + so a [structural protocol test](parametric-type-tests.md#a-default-evaluated-per-call-records-no-type) + against its method raises rather than answer + ## a class body is not a default the same reading does not extend to a class-body declaration, because a class diff --git a/docs/basedpython/features/parametric-type-tests.md b/docs/basedpython/features/parametric-type-tests.md index 350674c0ee..6f82cb8d1e 100644 --- a/docs/basedpython/features/parametric-type-tests.md +++ b/docs/basedpython/features/parametric-type-tests.md @@ -275,6 +275,29 @@ InitLevel() is HasA[int] # TypeError: its type is declared inside a method declare the member in the class body and the check answers. this is the same line the erased builtin target takes: refuse, never guess. +### a default evaluated per call records no type + +a method parameter with no annotation is checked by the type of its default. +a default that is not a scalar has no value until a call +[evaluates it](mutable-defaults.md#what-inspectsignature-shows), so nothing at +runtime records the type the checker reads off it, and the test raises: + +```by +class Foo: ... + +class Feeds[T](Protocol): + def m(self, x: T) -> int: ... + +class TakesFoo: + def m(self, x = Foo()) -> int: + return 1 + +def g(v: object): + v is Feeds[Foo] # TypeError for a `TakesFoo`: annotate `x` +``` + +annotate the parameter and the check answers + ### the cast forms usually never get here a [checked cast](checked-cast.md) shares this engine, but only for a value the diff --git a/docs/basedpython/features/trailing-lambdas.md b/docs/basedpython/features/trailing-lambdas.md index 5c5d68b2e9..e08a946bef 100644 --- a/docs/basedpython/features/trailing-lambdas.md +++ b/docs/basedpython/features/trailing-lambdas.md @@ -332,7 +332,7 @@ inspected leaves the block unconstrained so a trailing block can bind the last parameter while earlier parameters keep defaults, a `def` (not a `lambda`) may declare a required parameter after a defaulted one. python rejects that shape at runtime, so the required parameter -is lowered to a `_MISSING` sentinel default plus a guard that raises — +is lowered to a `_MISSING` marker default plus a guard that raises — mirroring python's own error: ```python @@ -343,7 +343,8 @@ def f(x=1, a=_MISSING): ``` the checker still treats the parameter as required. see -[mutable defaults](mutable-defaults.md) for the sentinel machinery this shares +[mutable defaults](mutable-defaults.md) for the marker this shares, and for what +`inspect.signature` shows for it ## inlay hints From 296d86afb516a31a5f3e389ef1e03b3b41acabad Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:35:00 +1000 Subject: [PATCH 34/47] make a compiled function hand a call to its interpreted definition when an argument is the very object that definition holds as the parameter's default, as it already did when the argument was left out. in `.by` that object is the marker `inspect.signature` reports for a default re-evaluated per call, or for a required parameter after a defaulted one, so a call bound with `BoundArguments.apply_defaults` and made through `by compile` output evaluates the default (`def f(xs: list[int] = [])`) or raises the missing-argument `TypeError`, instead of the native body taking the marker as the argument (`AttributeError: '_by_default' object has no attribute 'append'`, or a soundness `TypeError` for `x: int = LIMIT`) --- crates/by_build/tests/differential.rs | 89 +++++++++++++++ crates/by_codegen_c/src/lib.rs | 101 +++++++++++++++--- crates/by_ir/src/builder.rs | 10 ++ crates/by_ir/src/function.rs | 53 ++++++++- crates/by_ir/src/print.rs | 2 + crates/by_ir/src/verify.rs | 5 + crates/by_irbuild/src/lib.rs | 3 + crates/by_rt/include/by.h | 72 +++++++++++++ .../development/compilation/plan.md | 8 +- 9 files changed, 326 insertions(+), 17 deletions(-) diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index f8bb9a8e8e..4a8a00cd1c 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -29062,6 +29062,95 @@ class Joiner: ); } +/// `inspect.signature` reads a compiled function's defaults off its interpreted definition, +/// and in `.by` a default re-evaluated per call is a marker there. a tool that binds a call +/// from the signature and calls with `apply_defaults` hands the markers back, and the +/// compiled boundary hands such a call to the definition, whose guard reads each one as the +/// argument not given — the boundary was unboxing the marker as the argument instead +#[test] +fn a_default_marker_handed_back_agrees() { + const HANDED_BACK: &str = "(lambda f: (lambda b: (b.apply_defaults(), f(*b.args, **b.kwargs))[1])(__import__('inspect').signature(f).bind()))"; + let calls = [ + format!("[{HANDED_BACK}(m.fresh), {HANDED_BACK}(m.fresh)]"), + format!("[{HANDED_BACK}(m.limit), setattr(m, 'LIMIT', 5), {HANDED_BACK}(m.limit)]"), + format!("_warnings_from({HANDED_BACK}, m.relaxed)"), + format!("[{HANDED_BACK}(m.keyword), {HANDED_BACK}(m.keyword)]"), + "(lambda d: [m.Bag([1]).add(d), m.Bag([1]).add(d)])(__import__('inspect').signature(m.Bag.add).parameters['more'].default)".to_owned(), + ]; + agree( + "default_markers", + "\ +LIMIT = 2 + + +def fresh(xs: list[int] = []) -> int: + xs.append(1) + return len(xs) + + +def limit(x: int = LIMIT) -> int: + return x + + +def relaxed(x: int = 1, a: int) -> int: + return x + a + + +def keyword(*, xs: list[int] = []) -> int: + xs.append(1) + return len(xs) + + +class Bag: + def __init__(self, items: list[int]) -> None: + self.items = items + + def add(self, more: list[int] = [2]) -> int: + more.append(3) + return len(self.items) + len(more) +", + &calls.iter().map(String::as_str).collect::>(), + ); +} + +/// the marker a compiled method compares an argument against is read off its interpreted +/// definition at module init, through `__wrapped__` where a decorator wrapped it. a +/// `__wrapped__` that leads back round stops the walk, as `inspect.unwrap` stops, rather +/// than holding the import forever. the decorated method itself is left to the class the +/// interpreted definition decorated, which is the one reading its handle at init +#[test] +fn a_wrapped_chain_that_leads_back_round_agrees() { + agree_with_declines( + "wrapped_cycle", + "\ +class Looped: + def __init__(self, fn: (...) -> int): + self.fn = fn + self.__wrapped__ = self + + def __get__(self, instance: object, owner: object = None) -> object: + if instance === None: + return self + return lambda xs: self.fn(instance, xs) + + +def looped(fn: (...) -> int) -> Looped: + return Looped(fn) + + +class Bag: + def __init__(self): + self.n = 1 + + @looped + def fresh(self, xs: list[int] = []) -> int: + xs.append(1) + return len(xs) + self.n +", + &["[m.Bag().fresh([5]), m.Bag().fresh([5])]"], + ); +} + #[test] fn the_numeric_slots_agree() { // every binary numeric dunder fills a `nb_*` slot, and python never looks one up by diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index d8b1c2362b..50f69bfc23 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -237,6 +237,17 @@ pub fn emit_module(module: &ModuleIr) -> String { "static PyObject *{} = NULL;", function.interpreted_symbol(module.name.dotted()) ); + let markers = function + .twin_defaults() + .into_iter() + .filter(|_| function.defaults_are_markers); + for index in markers { + let _ = writeln!( + out, + "static PyObject *{} = NULL;", + function.twin_default_symbol(module.name.dotted(), index) + ); + } } } if any_defers { @@ -1726,7 +1737,7 @@ fn emit_written_init(module: &ModuleIr, init: &Function) -> String { ); // the same boundary the method's own wrapper has, and the same hand-over: a slot // is handed a tuple and a dict rather than a vector, which is all that differs - let tests = defer_tests(init, true); + let tests = defer_tests(module, init, true); if !tests.is_empty() { let _ = writeln!( out, @@ -10578,7 +10589,7 @@ fn emit_terminator( /// /// `receiver` says whether the first parameter is a `self` that arrives outside the /// binding, which is what shifts every slot index -fn defer_tests(function: &Function, receiver: bool) -> Vec { +fn defer_tests(module: &ModuleIr, function: &Function, receiver: bool) -> Vec { let slot = |index: &usize| format!("by_bound[{}]", index - usize::from(receiver)); function .deferring @@ -10587,16 +10598,21 @@ fn defer_tests(function: &Function, receiver: bool) -> Vec { let slot = slot(index); format!("({slot} != NULL && !By_IsExactFloat({slot}))") }) - // an omitted parameter whose default is not an immediate: the interpreted - // definition holds the one object every such call has to share — unless the - // receiver holds it, in which case this boundary can fill it itself - .chain( - function - .computed_defaults - .iter() - .filter(|_| function.defaults_held_by == by_ir::function::DefaultsHeldBy::Twin) - .map(|index| format!("{} == NULL", slot(index))), - ) + // a parameter whose default only the interpreted definition holds, omitted: a + // computed default is the one object every such call has to share in python. in + // basedpython it is a marker the definition re-evaluates the default for, or + // raises for a required parameter after a defaulted one, so a call handed that + // very marker is the definition's too. a receiver that holds the default is not + // one of these, and this boundary fills it itself + .chain(function.twin_defaults().iter().map(|index| { + let slot = slot(index); + if function.defaults_are_markers { + let held = function.twin_default_symbol(module.name.dotted(), *index); + format!("({slot} == NULL || {slot} == {held})") + } else { + format!("{slot} == NULL") + } + })) .collect() } @@ -11062,7 +11078,7 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri ) }; - let tests = defer_tests(function, is_method); + let tests = defer_tests(module, function, is_method); if !tests.is_empty() { let _ = writeln!(out, " if ({}) goto by_wrap_defer;", tests.join(" || ")); } @@ -12613,6 +12629,31 @@ fn emit_module_init(module: &ModuleIr) -> String { ); } } + let markers = function + .twin_defaults() + .into_iter() + .filter(|_| function.defaults_are_markers); + for index in markers { + // a keyword-only parameter's default is held by name — the name the + // definition declares it under, which is the parameter's here — and a + // positional one's by its place among the definition's positional parameters + let position = if function.is_keyword_only(index) { + -1 + } else { + i64::try_from(index).unwrap_or(-1) + }; + let name = function + .params() + .get(index) + .and_then(|decl| decl.name.as_deref()) + .unwrap_or_default(); + let _ = writeln!( + interpreted_init, + " {} = By_TwinDefault({handle}, {position}, {});", + function.twin_default_symbol(module.name.dotted(), index), + c_string(name) + ); + } } // the body is run either way; what a class-level constant needs is what each `class` @@ -15898,11 +15939,18 @@ mod tests { /// a class whose `__init__` takes `value`, with a default only the interpreted /// definition holds fn holder() -> ModuleIr { + holder_holding(false) + } + + /// [`holder`], from a `.by` definition when `markers`, which holds a marker for the + /// computed default rather than its value + fn holder_holding(markers: bool) -> ModuleIr { let mut builder = FunctionBuilder::new("__init__", RType::NONE); let receiver = builder.param("self", RType::OBJECT); let value = builder.param("value", RType::OBJECT); builder.defaults(vec![None, None]); builder.computed_defaults(vec![1]); + builder.defaults_are_markers(markers); builder.push(Op::SetField { receiver: Value::Register(receiver), class: "Holder".to_string(), @@ -15971,7 +16019,8 @@ mod tests { "{slot}" ); // and the call that omits it goes to the definition holding the one object - // every such call has to share + // every such call has to share. one handed that object is the native's: python's + // default is a value, and handing the call over would only make it slower assert!( slot.contains( "if (by_bound[0] == NULL) return By_InitInterpreted(byi_app_Holder___init__, \ @@ -15979,6 +16028,7 @@ mod tests { ), "{slot}" ); + assert!(!c.contains("By_TwinDefault("), "{c}"); // which means the slot needs the handle the method wrapper already takes assert!( c.contains("static PyObject *byi_app_Holder___init__ = NULL;"), @@ -15986,6 +16036,29 @@ mod tests { ); } + /// a `.by` definition holds a marker for the default instead, which only its own guard + /// reads as the argument not given — so a call handed that very marker goes to it too + #[test] + fn a_constructor_handed_its_marker_hands_the_call_over() { + let c = emit_module(&holder_holding(true)); + let slot = body_of(&c, "static int By_app_Holder_Type_init("); + assert!( + slot.contains( + "if ((by_bound[0] == NULL || by_bound[0] == byi_app_Holder___init___d1)) \ + return By_InitInterpreted(byi_app_Holder___init__, \"Holder.__init__\", \ + selfobj, args, kwds);" + ), + "{slot}" + ); + // the marker is read off the definition once the module has run + assert!( + c.contains( + "byi_app_Holder___init___d1 = By_TwinDefault(byi_app_Holder___init__, 1, \"value\");" + ), + "{c}" + ); + } + #[test] fn a_nested_functions_computed_default_is_filled_from_its_environment() { // a nested function has no interpreted definition to hand the call to, so its diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index a1f3a0ad71..c011feb456 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -40,6 +40,7 @@ pub struct FunctionBuilder { deferring: Vec, computed_defaults: Vec, defaults_held_by: crate::function::DefaultsHeldBy, + defaults_are_markers: bool, posonly: usize, kwonly: usize, /// per-block `.by` spans, parallel to `blocks` @@ -85,6 +86,7 @@ impl FunctionBuilder { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, posonly: 0, kwonly: 0, block_ranges: vec![None], @@ -254,6 +256,13 @@ impl FunctionBuilder { self.computed_defaults = computed; } + /// that the interpreted definition holds basedpython's markers for those defaults + /// + /// see [`Function::defaults_are_markers`] + pub fn defaults_are_markers(&mut self, markers: bool) { + self.defaults_are_markers = markers; + } + /// that those defaults are held by the receiver rather than by an interpreted twin /// /// see [`crate::function::DefaultsHeldBy::Receiver`] @@ -441,6 +450,7 @@ impl FunctionBuilder { deferring: self.deferring, computed_defaults: self.computed_defaults, defaults_held_by: self.defaults_held_by, + defaults_are_markers: self.defaults_are_markers, binding: crate::function::Binding::Instance, coroutine_body: None, doc: self.doc, diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index 797d41d6b7..c101530cea 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -362,6 +362,12 @@ pub struct Function { pub computed_defaults: Vec, /// who keeps the values for [`Self::computed_defaults`] pub defaults_held_by: DefaultsHeldBy, + /// whether the defaults [`Self::twin_defaults`] names are basedpython's markers: a `.by` + /// definition holds, for a default it re-evaluates per call and for a required + /// parameter after a defaulted one, an object that stands for no argument at all. + /// python's own computed default is a value, and a call handed it is the native's + /// to answer + pub defaults_are_markers: bool, /// decorators to apply, outermost first, after the native function is /// installed in the module namespace pub decorators: Vec, @@ -501,8 +507,50 @@ impl Function { /// whether any call can be handed to the interpreted definition pub fn defers(&self) -> bool { - !self.deferring.is_empty() - || (!self.computed_defaults.is_empty() && self.defaults_held_by == DefaultsHeldBy::Twin) + !self.deferring.is_empty() || !self.twin_defaults().is_empty() + } + + /// the parameters whose default only the interpreted definition holds, by index + /// + /// a computed default, which python evaluated once where the definition stood, and a + /// required parameter after a defaulted one, which basedpython allows and python does + /// not: the definition declares a default for it that stands for no argument at all. + /// a call that omits one of these is the definition's to answer — and so is one that + /// hands it the very object the definition holds, where that is one of basedpython's + /// [markers](Self::defaults_are_markers), as `BoundArguments.apply_defaults` hands it + /// with a signature read off the definition + pub fn twin_defaults(&self) -> Vec { + if self.defaults_held_by != DefaultsHeldBy::Twin { + return Vec::new(); + } + let packed = usize::from(self.vararg) + usize::from(self.kwarg); + let named = self.param_count.saturating_sub(packed); + let positional = named.saturating_sub(self.kwonly); + let defaulted = |index: usize| self.defaults.get(index).is_some_and(Option::is_some); + let mut after_default = false; + let mut indices = Vec::new(); + for index in 0..named { + let computed = self.computed_defaults.contains(&index); + if computed || (index < positional && after_default && !defaulted(index)) { + indices.push(index); + } + after_default |= index < positional && (computed || defaulted(index)); + } + indices + } + + /// whether the parameter at `index` is keyword-only + pub fn is_keyword_only(&self, index: usize) -> bool { + let packed = usize::from(self.vararg) + usize::from(self.kwarg); + let named = self.param_count.saturating_sub(packed); + index < named && index >= named.saturating_sub(self.kwonly) + } + + /// the C identifier for the marker the interpreted definition holds for the parameter + /// at `index`, one of [`Self::twin_defaults`] of a function whose + /// [defaults are markers](Self::defaults_are_markers) + pub fn twin_default_symbol(&self, module: &str, index: usize) -> String { + format!("{}_d{index}", self.interpreted_symbol(module)) } /// the environment field holding the value for the parameter at `index`, for a @@ -1895,6 +1943,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: Binding::Instance, coroutine_body: None, doc: None, diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index f9be553047..16ac2d762e 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -1078,6 +1078,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -1155,6 +1156,7 @@ b2: deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 7ff963a97e..2badd88221 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -2311,6 +2311,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -2523,6 +2524,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -2577,6 +2579,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -2632,6 +2635,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, @@ -2739,6 +2743,7 @@ mod tests { deferring: Vec::new(), computed_defaults: Vec::new(), defaults_held_by: crate::function::DefaultsHeldBy::Twin, + defaults_are_markers: false, binding: crate::function::Binding::Instance, coroutine_body: None, doc: None, diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 9cf5006408..9abf134efe 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -1632,6 +1632,7 @@ fn lower_generator_constructor( db, model, layouts, + language, .. } = unit; let mut signed = signature( @@ -1669,6 +1670,7 @@ fn lower_generator_constructor( builder.binding_kinds(posonly, kwonly); builder.deferring(deferring); builder.computed_defaults(computed_defaults); + builder.defaults_are_markers(!language.defaults_evaluated_at_the_def()); let mut registers: HashMap = HashMap::new(); for (name, rtype) in ¶ms { registers.insert(name.clone(), builder.param(name.clone(), rtype.clone())); @@ -7940,6 +7942,7 @@ fn lower_function_with_receiver( builder.defaults_held_by_the_receiver(); } builder.computed_defaults(computed_defaults); + builder.defaults_are_markers(!unit.language.defaults_evaluated_at_the_def()); let mut locals: HashMap = HashMap::new(); for (name, rtype) in ¶ms { diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index a35eaf9cc6..09cf540420 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -1491,6 +1491,78 @@ static inline PyObject *By_CallInterpreted(PyObject *fn, const char *name, return PyObject_Vectorcall(fn, args, (size_t)nargs, kwnames); } +/* the object the interpreted definition `twin` holds as a parameter's default: the + * positional parameter at `position`, counted from its first and receiver included, or + * the keyword-only parameter `name` where `position` is negative. a boundary that is + * handed this very object hands the call to the definition, as it does one that omits + * the argument. + * + * a definition a decorator wrapped is reached through `__wrapped__`, which is where + * `inspect.signature` reads the same defaults from. the walk stops where `inspect.unwrap` + * refuses to go on: at an object it has already passed, so a `__wrapped__` that leads + * back round cannot hold module init forever, and past the recursion limit. the objects + * passed are held until the walk ends, so none of their addresses can be reused by the + * next. NULL, and no error, where nothing is held: such a boundary then defers only a + * call that omits the argument */ +static PyObject *By_TwinDefault(PyObject *twin, Py_ssize_t position, const char *name) { + PyObject *function, *passed, *found = NULL; + if (twin == NULL) return NULL; + passed = PyList_New(0); + if (passed == NULL) { + PyErr_Clear(); + return NULL; + } + function = Py_NewRef(twin); + while (function != NULL && !PyFunction_Check(function) && !PyMethod_Check(function)) { + PyObject *wrapped; + Py_ssize_t at, count = PyList_GET_SIZE(passed); + for (at = 0; at < count; at++) { + if (PyList_GET_ITEM(passed, at) == function) break; + } + if (at < count || count >= Py_GetRecursionLimit() || PyList_Append(passed, function) < 0) { + Py_DECREF(function); + function = NULL; + break; + } + wrapped = PyObject_GetAttrString(function, "__wrapped__"); + Py_DECREF(function); + function = wrapped; + } + Py_DECREF(passed); + if (function != NULL && PyMethod_Check(function)) { + PyObject *inner = Py_NewRef(PyMethod_GET_FUNCTION(function)); + Py_DECREF(function); + function = inner; + if (!PyFunction_Check(function)) { + Py_DECREF(function); + function = NULL; + } + } + if (function == NULL) { + PyErr_Clear(); + return NULL; + } + if (position >= 0) { + PyObject *defaults = PyFunction_GetDefaults(function); + PyCodeObject *code = (PyCodeObject *)PyFunction_GetCode(function); + if (defaults != NULL && PyTuple_Check(defaults)) { + Py_ssize_t held = PyTuple_GET_SIZE(defaults); + Py_ssize_t first = code->co_argcount - held; + if (position >= first && position - first < held) { + found = Py_NewRef(PyTuple_GET_ITEM(defaults, position - first)); + } + } + } else { + PyObject *kwdefaults = PyFunction_GetKwDefaults(function); + if (kwdefaults != NULL && PyDict_Check(kwdefaults)) { + found = PyDict_GetItemString(kwdefaults, name); + Py_XINCREF(found); + } + } + Py_DECREF(function); + return found; +} + /* `float`'s narrowing as a test, for a caller that branches on it and reads the double * itself: the double a float holds may be any value at all, so a failure that is only an * error value would have to ask the thread whether an exception is set on every read */ diff --git a/docs/basedpython/development/compilation/plan.md b/docs/basedpython/development/compilation/plan.md index 2173b72bc3..6c509e37ab 100644 --- a/docs/basedpython/development/compilation/plan.md +++ b/docs/basedpython/development/compilation/plan.md @@ -299,7 +299,13 @@ M0, M1 and M2 are done. what exists: class-level constant, both come from the *interpreted definition* that already evaluated them at definition time rather than being computed a second time. that is what makes a mutable default shared by every call that omits it, and it - keeps the object identical between the two halves + keeps the object identical between the two halves. a call that hands the + boundary the very object the interpreted definition holds as a default is handed + to that definition too: in `.by` that object is the + [marker](../../features/mutable-defaults.md#what-inspectsignature-shows) a call + gets from `inspect.signature`, which only the definition's guard reads as the + argument not given — evaluating the default per call, or raising for a required + parameter after a defaulted one - **plain classes** — a hand-written `__init__` is lowered as a method and `tp_init` binds against *its* signature; the layout is the attributes that constructor From c0f1e665ff6baf0e69507c63c3f300da444fa9fa Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:29:25 +1000 Subject: [PATCH 35/47] add `SemanticModel::deferrable_from_imports`: the module-level `from` imports whose binding is the only definition its name has in the module (no other assignment, `def`, `class`, loop/`with`/`except`/walrus target, second import, declaration, `del`, or `global` write from a nested scope), with every name node that reads it. a read resolves as python resolves it, so a class body that binds the name reads the module's binding until it has made its own, never a function's around it, and a class-body read that may find either disqualifies the import. a module with a star import or its own `__getattr__`/`__dir__` defers nothing --- crates/ty_python_semantic/src/lib.rs | 1 + .../ty_python_semantic/src/semantic_model.rs | 7 + crates/ty_python_semantic/src/types.rs | 1 + .../src/types/deferred_imports.rs | 451 ++++++++++++++++++ 4 files changed, 460 insertions(+) create mode 100644 crates/ty_python_semantic/src/types/deferred_imports.rs diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index a0721a8a19..c03cf51200 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -51,6 +51,7 @@ pub use types::conformance::{ConformanceRegistration, WitnessDispatch, WitnessKi pub use types::conversions::{ ConversionImport, ConversionInfo, ConversionRuntime, DISCARD_ADAPTER, }; +pub use types::deferred_imports::DeferrableImport; pub use types::extensions::{ ExtensionAttributeInfo, ExtensionMemberKind, decides_receiver_binding, extension_backing_function, extension_backing_functions, extension_member_kind, diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index f9d430b8b1..4e2d45bd19 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -384,6 +384,13 @@ impl<'db> SemanticModel<'db> { names } + /// basedpython: the module-level `from` imports whose binding is the only thing that ever + /// binds its name in this file, each with every read of it — the imports the lazy-import + /// polyfill can defer by rewriting those reads. see `types::deferred_imports` + pub fn deferrable_from_imports(&self) -> Vec { + crate::types::deferred_imports::deferrable_from_imports(self.db, self.file) + } + /// basedpython: when an attribute access reads a *requirement* off an /// interface-typed receiver, how it dispatches through the witness table. /// diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 5b92b91b3f..d73e765b58 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -194,6 +194,7 @@ pub(crate) mod conversions; mod cyclic; pub(crate) mod dedicated; mod deferred; +pub(crate) mod deferred_imports; mod diagnostic; pub(crate) mod display; pub(crate) mod enums; diff --git a/crates/ty_python_semantic/src/types/deferred_imports.rs b/crates/ty_python_semantic/src/types/deferred_imports.rs new file mode 100644 index 0000000000..cb16b8347f --- /dev/null +++ b/crates/ty_python_semantic/src/types/deferred_imports.rs @@ -0,0 +1,451 @@ +//! basedpython: which module-level `from` imports can be deferred by rewriting their reads +//! +//! below python 3.15 there is no `lazy` keyword, so the transpiler defers a `from m import x` +//! by binding nothing at the import and writing every read of `x` as a read through an object +//! that resolves `m.x` on first use. that is only the program python would run when every +//! read the rewrite reaches is a read of that import, and every read of that import is +//! rewritten: [`deferrable_from_imports`] answers which imports that holds for, and with +//! which reads. an import it leaves out stays an ordinary import + +use ruff_db::parsed::parsed_module; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk_stmt}; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange}; +use rustc_hash::FxHashMap; + +use ty_python_core::ast_ids::HasScopedUseId; +use ty_python_core::definition::{Definition, DefinitionState}; +use ty_python_core::place::ScopedPlaceId; +use ty_python_core::scope::{FileScopeId, ScopeKind}; +use ty_python_core::{ProgramFile, semantic_index}; + +use crate::Db; + +/// a module-level `from` import name whose every read can be rewritten +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeferrableImport { + /// the `ast::Alias` the name is imported by + pub alias: TextRange, + /// every read of the binding the alias makes, as the range of its `ast::ExprName` + pub reads: Vec, +} + +/// where a name read or written in a module resolves +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Resolution { + /// the module's own namespace + Module, + /// a binding of some scope inside the module + Elsewhere, + /// either, depending on what has run: a class body that binds the name on some paths + /// reads its own binding where it has made one and the module's where it has not + Either, +} + +/// basedpython: the module-level `from` imports of `file` whose binding is the only thing that +/// ever binds its name, with every read of it +/// +/// an import qualifies when, across the whole module: +/// +/// - nothing else binds, declares or deletes the name at module scope, nor from a scope that +/// declares it `global`: the index records such a binding among the module scope's own, for +/// the reads after the nested scope, and a `del` under `global` is a name node the read walk +/// finds +/// - every read that can reach the module's binding reaches nothing else: a class body that +/// binds the name on some paths to a read and not on others disqualifies it +/// - the module has no `from … import *`, which can bind any name unseen, and binds neither +/// `__getattr__` nor `__dir__`, which a deferred import is served through +/// +/// a name inside a string is not a read here: in `.by` a string in a type expression is a +/// literal type, and a forward reference the transpiler writes as a string is its business +pub(crate) fn deferrable_from_imports<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, +) -> Vec { + let module = parsed_module(db, file.python_file(db)).load(db); + let index = semantic_index(db, file); + let global = FileScopeId::global(); + let global_table = index.place_table(global); + + let mut statements = ModuleStatements::default(); + statements.visit_body(module.suite()); + if statements.star_import { + return Vec::new(); + } + let binds = |name: &str| { + let Some(symbol) = global_table.symbol_id(name) else { + return false; + }; + let symbol = global_table.symbol(symbol); + symbol.is_bound() || symbol.is_declared() + }; + if binds("__getattr__") || binds("__dir__") { + return Vec::new(); + } + + // every definition the module scope makes, by the place it makes it of. a deletion is + // a `del`, whose target is a name node the read walk below finds, or the end of an + // `except … as` clause, which bound the name first + let use_def = index.use_def_map(global); + let mut definitions: FxHashMap>> = FxHashMap::default(); + for (_, definition, _) in use_def.definitions_with_usage() { + definitions + .entry(definition.place(db)) + .or_default() + .push(definition); + } + + let mut candidates: FxHashMap> = FxHashMap::default(); + for alias in statements.aliases { + let bound = alias.asname.as_ref().unwrap_or(&alias.name).id.clone(); + let sole = global_table.symbol_id(&bound).is_some_and(|symbol| { + let place = ScopedPlaceId::Symbol(symbol); + index + .try_definitions(AnyNodeRef::from(alias)) + .is_some_and(|own| { + own.len() == 1 + && definitions + .get(&place) + .is_some_and(|made| made.as_slice() == own) + }) + }); + // a second import of the name is a second binding, so neither import qualifies + candidates + .entry(bound) + .and_modify(|slot| *slot = None) + .or_insert(sole.then_some(alias.range())); + } + candidates.retain(|_, alias| alias.is_some()); + if candidates.is_empty() { + return Vec::new(); + } + + let mut names = NameReferences { + names: &candidates, + found: Vec::new(), + }; + names.visit_body(module.suite()); + let mut reads: FxHashMap> = FxHashMap::default(); + for reference in names.found { + let Some(Some(_)) = candidates.get(&reference.id) else { + continue; + }; + match (resolve(db, file, reference), reference.ctx) { + (Resolution::Elsewhere, _) => {} + (Resolution::Module, ast::ExprContext::Load) => { + reads + .entry(reference.id.clone()) + .or_default() + .push(reference.range()); + } + // a write the module's own table does not show — a walrus in a comprehension — + // or a read that may find another binding + _ => { + candidates.insert(reference.id.clone(), None); + } + } + } + + let mut deferrable: Vec = candidates + .into_iter() + .filter_map(|(name, alias)| { + Some(DeferrableImport { + alias: alias?, + reads: reads.remove(&name).unwrap_or_default(), + }) + }) + .collect(); + deferrable.sort_by_key(|import| import.alias.start()); + deferrable +} + +/// where `reference` resolves, as python resolves it +/// +/// a class body is where this differs from a lexical walk. a name the body binds anywhere is +/// read with `LOAD_NAME`: the class namespace, and then the module's — never an enclosing +/// function's — so a read the body has not bound on any path resolves to the module even +/// when a function around the class binds the name +fn resolve<'db>(db: &'db dyn Db, file: ProgramFile<'db>, reference: &ast::ExprName) -> Resolution { + let index = semantic_index(db, file); + let Some(own_scope) = index.try_expression_scope_id(&ast::ExprRef::from(reference)) else { + return Resolution::Either; + }; + for (scope_id, scope) in index.visible_ancestor_scopes(own_scope) { + if scope_id.is_global() { + return Resolution::Module; + } + let table = index.place_table(scope_id); + let Some(symbol) = table.symbol_by_name(&reference.id) else { + continue; + }; + if symbol.is_global() { + return Resolution::Module; + } + if symbol.is_nonlocal() { + return Resolution::Elsewhere; + } + if !(symbol.is_bound() || symbol.is_declared()) { + continue; + } + if scope.kind() != ScopeKind::Class { + return Resolution::Elsewhere; + } + // an annotation scope sees the class it is defined in through the class namespace, + // falling back to the names outside it when the class has not bound the name yet + if scope_id != own_scope || !reference.ctx.is_load() { + return if reference.ctx.is_load() { + Resolution::Either + } else { + Resolution::Elsewhere + }; + } + let scope = scope_id.to_scope_id(db, file); + let use_id = reference.scoped_use_id(db, file); + let (mut defined, mut undefined) = (false, false); + for binding in ty_python_core::use_def_map(db, scope).bindings_at_use(use_id) { + match binding.binding { + DefinitionState::Defined(_) => defined = true, + DefinitionState::Undefined | DefinitionState::Deleted => undefined = true, + } + } + return match (defined, undefined) { + (true, false) => Resolution::Elsewhere, + (false, _) => Resolution::Module, + (true, true) => Resolution::Either, + }; + } + Resolution::Module +} + +/// the module-level `from` imports, and whether there is a star import anywhere +#[derive(Default)] +struct ModuleStatements<'ast> { + aliases: Vec<&'ast ast::Alias>, + star_import: bool, +} + +impl<'ast> SourceOrderVisitor<'ast> for ModuleStatements<'ast> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + match stmt { + Stmt::ImportFrom(import) => { + if import.names.iter().any(|alias| &alias.name == "*") { + self.star_import = true; + } else { + self.aliases.extend(import.names.iter()); + } + } + // a function's or a class's imports bind in that scope + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} + _ => walk_stmt(self, stmt), + } + } + + fn visit_expr(&mut self, _expr: &'ast Expr) {} +} + +/// every name node spelling one of `names`, whatever it does and wherever it is +struct NameReferences<'a, 'ast> { + names: &'a FxHashMap>, + found: Vec<&'ast ast::ExprName>, +} + +impl<'ast> SourceOrderVisitor<'ast> for NameReferences<'_, 'ast> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && self.names.contains_key(&name.id) + { + self.found.push(name); + } + walk_expr(self, expr); + } +} + +#[cfg(test)] +mod tests { + use ruff_db::files::system_path_to_file; + use ty_python_core::ProgramFile; + + use crate::db::tests::TestDbBuilder; + + /// each deferrable import of `/src/main.py`, as `source` spells its alias, with the text of + /// every read of it + fn deferrable(source: &str) -> Vec<(String, Vec)> { + let db = TestDbBuilder::new() + .with_file("/src/main.py", source) + .build() + .expect("the db builds"); + let file = system_path_to_file(&db, "/src/main.py").expect("the file exists"); + let file = ProgramFile::new(&db, file, db.program_environment().program(&db)); + super::deferrable_from_imports(&db, file) + .into_iter() + .map(|import| { + ( + source[import.alias].to_owned(), + import + .reads + .iter() + .map(|read| source[*read].to_owned()) + .collect(), + ) + }) + .collect() + } + + /// whether `source`'s import of `dumps` is deferred + fn defers_dumps(source: &str) -> bool { + deferrable(source).iter().any(|(alias, _)| alias == "dumps") + } + + #[test] + fn a_name_bound_only_by_its_import_is_deferred_with_every_read() { + let found = deferrable( + "from json import dumps, loads as parse\n\ + def f():\n return dumps(parse('1'))\n\ + print(dumps)\n", + ); + assert_eq!( + found, + vec![ + ( + "dumps".to_owned(), + vec!["dumps".to_owned(), "dumps".to_owned()] + ), + ("loads as parse".to_owned(), vec!["parse".to_owned()]), + ] + ); + } + + /// every way python binds, declares or deletes a name, at module scope or from a scope + /// that declares it `global`, is a second binding + #[test] + fn any_other_binding_of_the_name_keeps_the_import() { + for rebinding in [ + "dumps = 5", + "dumps += 1", + "dumps: int = 1", + "dumps: int", + "def dumps(): pass", + "class dumps: pass", + "for dumps in []: pass", + "with open('x') as dumps: pass", + "try:\n pass\nexcept Exception as dumps:\n pass", + "if (dumps := 1): pass", + "print([(dumps := i) for i in []])", + "del dumps", + "from json import dumps", + "from pickle import dumps", + "import dumps", + "import pickle as dumps", + "match 1:\n case dumps:\n pass", + "match [1]:\n case [*dumps]:\n pass", + "def f():\n global dumps\n dumps = 1", + "def f():\n global dumps\n del dumps", + "def f():\n global dumps\n def dumps(): pass", + "def f():\n global dumps\n import json as dumps", + "def f():\n global dumps\n for dumps in []: pass", + "class C:\n global dumps\n dumps = 1", + ] { + let source = format!("from json import dumps\n{rebinding}\nprint(dumps)\n"); + assert!(!defers_dumps(&source), "deferred:\n{source}"); + } + } + + /// a binding in a scope of its own is that scope's, and leaves the module's alone + #[test] + fn a_local_binding_of_the_name_is_not_the_module_s() { + let found = deferrable( + "from json import dumps\n\ + def f(dumps):\n return dumps\n\ + def g():\n dumps = 1\n return dumps\n\ + lambda dumps: dumps\n\ + [dumps for dumps in []]\n\ + class C:\n dumps = 1\n seen = dumps\n\ + def h():\n return dumps\n", + ); + assert_eq!(found, vec![("dumps".to_owned(), vec!["dumps".to_owned()])]); + } + + /// a read reaches the module's binding from any scope between it and the module that does + /// not bind the name + #[test] + fn a_read_from_any_scope_that_reaches_the_module_is_a_read() { + let found = deferrable( + "from json import dumps\n\ + def nested():\n def inner():\n return dumps\n return inner\n\ + class Body:\n value = dumps\n\ + def default(f=dumps):\n return f\n\ + def reads_global():\n global dumps\n return dumps\n\ + later = lambda: dumps\n\ + listed = [dumps for _ in []]\n\ + text = f'{dumps}'\n\ + @dumps\n\ + def decorated(): pass\n", + ); + assert_eq!(found.len(), 1); + assert_eq!(found[0].1.len(), 8, "{found:?}"); + } + + /// a class body reads a name it binds with `LOAD_NAME`: the class namespace, and then the + /// module's — never a function's around it + #[test] + fn a_class_body_reads_the_module_until_it_binds_the_name() { + let found = deferrable( + "from json import dumps\n\ + def outer():\n dumps = 1\n class Body:\n seen = dumps\n dumps = 2\n", + ); + assert_eq!(found, vec![("dumps".to_owned(), vec!["dumps".to_owned()])]); + } + + /// a read the body has bound on some paths finds either binding + #[test] + fn a_class_body_read_that_may_find_its_own_binding_keeps_the_import() { + assert!(!defers_dumps( + "from json import dumps\nimport random\nclass Body:\n if random.random():\n dumps = 1\n seen = dumps\n" + )); + } + + /// a star import can bind any name unseen, and a module's own `__getattr__` or `__dir__` + /// is what a deferred name would be served through + #[test] + fn a_module_that_could_rebind_unseen_defers_nothing() { + for source in [ + "from os import *\nfrom json import dumps\nprint(dumps)\n", + "from json import dumps\ndef __getattr__(name): return dumps\n", + "from json import dumps\ndef __dir__(): return []\n", + "from json import dumps\n__getattr__ = print\n", + ] { + assert!(!defers_dumps(source), "deferred:\n{source}"); + } + } + + /// an import inside a function or a class binds there + #[test] + fn an_import_in_a_scope_of_its_own_is_not_a_module_import() { + assert!(deferrable("def f():\n from json import dumps\n return dumps\n").is_empty()); + } + + /// a name an import binds only on some paths is still bound by nothing else, and reads of + /// it are reads of the import. what a read finds where the import did not run is the + /// runtime's business: python's lookup falls back to the builtins + #[test] + fn an_import_on_some_paths_is_still_the_only_binding() { + let found = deferrable( + "import sys\nif sys.version_info < (3, 0):\n from itertools import izip as zip\nprint(zip)\n", + ); + assert_eq!( + found, + vec![("izip as zip".to_owned(), vec!["zip".to_owned()])] + ); + } + + /// a package importing its own submodule, by relative or absolute name, is an ordinary + /// import of a name nothing else binds + #[test] + fn a_package_importing_its_own_submodule_is_deferrable() { + assert_eq!( + deferrable("from . import helper\ndef use():\n return helper.NAME\n"), + vec![("helper".to_owned(), vec!["helper".to_owned()])] + ); + } +} From 2ab311fd57add99a578c83d8bf0b15270253c27c Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:30:17 +1000 Subject: [PATCH 36/47] below python 3.15, a `from` import binds nothing where it is written: it registers its names with one `_LazyFrom` per module whose `__dict__` is the module's own namespace (`_lazy_from(_by_lazy, "json", {"dumps": "dumps"})`), and every read of a name that `deferrable_from_imports` reports is written as `_by_lazy.dumps`. the first read resolves the name off its module, with the same submodule fallback and `ImportError` wording as before, and stores the object in the namespace, so the name is the object itself: identity, `type()`, `json.dumps`, `pytest.approx` and `=== None` all answer as for an ordinary import, which the `_LazyAttr` proxy could not. a patch of the name in the importing module (`mock.patch`, `monkeypatch.setattr`) reaches its reads, a read before the statement has run raises python's `NameError`, and the module's `__getattr__`/`__dir__` serve the names to other modules. a name with a read that cannot be rewritten, or a name read in an `f"{x=}"` or a t-string, or beginning with two underscores, keeps an ordinary import, as do exception classes, special forms and names an annotation or a decorator reads. `_soundness_once_bound` counts a registered deferred name as bound. the interpreted fallback of a `by compile` module keeps its `from` imports ordinary (`Config::defer_from_imports`), since compiled functions read the module's globals by name. `_LazyAttr`, `_lazy_attr` and `_by_forward_operators` are removed a relative `from . import helper` reads `helper` off the package and imports it as a submodule only when the package has none, as python does, rather than importing the submodule `pkg.helper` unconditionally where the statement stood a name read inside a forward reference the transpiler writes as a string, such as a type variable's bound written ahead of the class it names (`TypeVar("T", bound="dict[Thing, C.Inner]")`), keeps an ordinary import, since whatever evaluates that string looks the name up in the module's namespace --- crates/by_build/src/lib.rs | 10 +- crates/by_build/tests/differential.rs | 78 +- crates/by_rt/include/by.h | 3 +- crates/by_transforms/src/config.rs | 9 + crates/by_transforms/src/lib.rs | 194 +++-- crates/by_transforms/src/runtime.rs | 73 +- .../by_transforms/src/runtime/_by_runtime.py | 239 ++++-- .../src/transforms/ast_driver.rs | 45 +- .../by_transforms/src/transforms/generics.rs | 42 +- .../src/transforms/inferred_annotation.rs | 5 +- .../src/transforms/lazy_import.rs | 557 ++++++++++--- .../tests/inferred_annotation_runtime.rs | 12 +- .../tests/lazy_import_runtime.rs | 788 ++++++++++++++++-- crates/ty/tests/by_e2e.rs | 13 +- .../ty_python_semantic/src/semantic_model.rs | 17 +- .../development/compilation/technology.md | 4 +- .../development/how-transpilation-works.md | 11 +- .../features/differences-from-python.md | 4 +- docs/basedpython/features/lazy-imports.md | 219 +++-- 19 files changed, 1793 insertions(+), 530 deletions(-) diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index 935b742fdc..af4ec152bd 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -390,9 +390,10 @@ fn finish( anyhow::anyhow!("the `{pass}` pass produced ill-formed IR:\n {detail}") })?; - // the default config on purpose, except for the target version: the - // lazy-import pass is what binds `JustFloat = float` locally instead of - // emitting a `from ty_extensions import ...` that has no module behind it. + // the default config on purpose, except for the target version and the + // deferral of `from` imports: the lazy-import pass is what binds + // `JustFloat = float` locally instead of emitting a + // `from ty_extensions import ...` that has no module behind it // // the version has to be the *interpreter's*, because this python runs inside // the extension at import time — emitting syntax the interpreter cannot parse @@ -402,6 +403,9 @@ fn finish( source.to_string() } else { let mut config = options.fallback.clone().unwrap_or_default(); + // the compiled functions read the module's globals by name, and a deferred + // `from` import is in them only once something has read it + config.defer_from_imports = false; if let Some((major, minor)) = toolchain.and_then(|toolchain| toolchain.version) && let Ok(parsed) = format!("{major}.{minor}").parse() { diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 4a8a00cd1c..17c82bc717 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -11561,6 +11561,24 @@ fn a_float_module_imports_without_any_extra_runtime_module() { assert_eq!(out, "12.0"); } +#[test] +fn a_compiled_function_reads_a_name_its_module_imports_with_from() { + // below 3.15 the lazy-import polyfill defers a `from` import by binding nothing where + // it is written, and a compiled function reads the module's globals by name. so the + // fallback a compiled module embeds keeps its `from` imports ordinary ones + agree( + "fromglobal", + "\ +from json import dumps +from os.path import basename as base + +def f(a: int) -> str: + return dumps([a]) + base('/x/y') +", + &["m.f(1)"], + ); +} + #[test] fn a_compiled_function_is_a_c_function_object() { // one of the few places the two builds are *meant* to differ: a natively @@ -22840,10 +22858,64 @@ def widened(value: object) -> str: // the class reports the module it was written in, not `builtins` "(m.Node.__module__, m.Keyed.__module__)", "(m.Node.__name__, m.Node.__qualname__)", - // an imported base reaches this module as a lazy proxy, so `__mro_entries__` - // is what turns it into the class — and python records what was written - "repr(m.Node.__orig_bases__)", + // an imported base is the class itself, with no `__mro_entries__` to + // record what was written in its place + "'__orig_bases__' in vars(m.Node)", + ], + ); +} + +#[test] +fn a_class_on_a_base_that_stands_for_others_agrees() { + // a base that is not a class may stand for a tuple of them through `__mro_entries__`, + // which python resolves before it works out a metaclass, recording what was written in + // `__orig_bases__`. the checker is shown the class the stand-in stands for, so the class + // compiles, and the metaclass here is `ABCMeta`, so the type is built by calling it + let Some(compiled) = agree_in( + "mroentries", + "\ +from abc import ABC +from typing import TYPE_CHECKING + + +class StandIn: + def __mro_entries__(self, bases: tuple[object, ...]) -> tuple[type, ...]: + return (ABC,) + + +if TYPE_CHECKING: + Base = ABC +else: + Base = StandIn() + + +class Node(Base): + def kind(self) -> str: + return \"node\" +", + &[ + "m.Node().kind()", + "[base.__name__ for base in m.Node.__mro__]", + "type(m.Node.__orig_bases__[0]).__name__", + "(type(m.Node).__name__, issubclass(m.Node, __import__('abc').ABC))", ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + // and the class is the one module init built, not the interpreted definition left + // standing because the build gave up on it + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_mroentries as m\nprint(type(m.Node.__dict__['kind']).__name__)\n", + ), + "method_descriptor" ); } diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 09cf540420..1613f2c36f 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -3925,8 +3925,7 @@ static inline PyObject *By_SpecSubclass(PyObject *module_dict, const char *name, * * python resolves `__mro_entries__` before it works out a metaclass or calls one: a * base that is not a class may stand for a tuple of them. `typing.Generic[T]` is the - * familiar one, and a name bound to a proxy — which is what a lazily imported base is - * until it is first read — is another */ + * familiar one, which a generic class's lowering writes among its bases */ static inline PyObject *By_ResolveBases(PyObject *bases) { Py_ssize_t index, prior; PyObject *tuple; diff --git a/crates/by_transforms/src/config.rs b/crates/by_transforms/src/config.rs index 562612cfdd..ce9d5142a8 100644 --- a/crates/by_transforms/src/config.rs +++ b/crates/by_transforms/src/config.rs @@ -23,6 +23,14 @@ pub struct Config { /// Tests that compare exact transpile output should set this to `false` /// to keep their expected strings free of the lazy preamble pub lazy_imports: bool, + /// when true (the default), the polyfill defers a `from` import below 3.15 by + /// binding nothing where it is written and reading the name through the + /// module's `_LazyFrom` on first use. off, a `from` import stays an ordinary + /// import, and only module imports are deferred. the interpreted fallback a + /// natively compiled module embeds turns it off: the compiled functions read + /// the module's globals by name, where a deferred name is not until something + /// has read it + pub defer_from_imports: bool, /// when true, every transpiled file is prefixed with /// `from __future__ import annotations`, deferring all annotation /// evaluation. off by default: forward references are handled surgically @@ -110,6 +118,7 @@ impl Default for Config { is_python: false, is_stub: false, lazy_imports: true, + defer_from_imports: true, inject_future_annotations: false, prune_unused_imports_after_reverse: true, soundness: SoundnessPositions::defaults(), diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 32b2e01cea..5791d8264d 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -44,6 +44,21 @@ pub(crate) fn make_in_memory_db(source: &str) -> (TestDb, File) { make_in_memory_db_at(source, PythonVersion::latest_ty()) } +/// a single-file db over `source` read as the python it is: transpiled output, which +/// python runs, and which is to be understood the way python understands it +fn make_in_memory_python_db(source: &str) -> (TestDb, File) { + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program_with_python_version(PythonVersion::latest_ty()) + .expect("program init failed"); + db.write_file("/output.py", source) + .expect("write file failed"); + let file = system_path_to_file(&db, "/output.py").expect("file not in db"); + (db, file) +} + /// [`make_in_memory_db`], analysing the source at `python_version` fn make_in_memory_db_at(source: &str, python_version: PythonVersion) -> (TestDb, File) { let mut db = TestDb::new(ProjectMetadata::new( @@ -243,7 +258,12 @@ fn transpile_with_report( let unchanged = !reified_changed && !qualified_changed && matches!(enum_lowered.output, std::borrow::Cow::Borrowed(_)); - let (source, ast_errors, _phase0_map) = transforms::ast_driver::run_against_source( + let transforms::ast_driver::Phase0 { + source, + errors: ast_errors, + forward_reference_reads, + .. + } = transforms::ast_driver::run_against_source( source, written_names, config, @@ -282,12 +302,16 @@ fn transpile_with_report( // --- Phase 2: import-redirect, surface-syntax cleanup, lazy-import marking --- let (final_output, requirements) = run_import_redirect_phase(output, config); let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; + // a forward reference a lowering wrote as a string reads its names when python evaluates + // it, against the module's namespace, so those are bound where they are imported + let mut eager_names = model.eagerly_imported_names(); + eager_names.extend(forward_reference_reads); let final_output = run_lazy_import_phase( final_output, config, &model.eagerly_imported_modules(), - &model.eagerly_imported_names(), - ); + &eager_names, + )?; let final_output = run_version_polyfill_phase(final_output, config); // --- Phase 3: syntax verification --- @@ -491,9 +515,16 @@ pub fn transpile_typed_with_report( // single-file db cannot resolve the modules that declare the conformances let eager_model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); let eager_imports = eager_model.eagerly_imported_modules(); - let eager_names = eager_model.eagerly_imported_names(); - let (spliced, ast_errors, phase0_map) = - transforms::ast_driver::run_against_source(working_source, written_names, config, project); + let mut eager_names = eager_model.eagerly_imported_names(); + let transforms::ast_driver::Phase0 { + source: spliced, + errors: ast_errors, + line_map: phase0_map, + forward_reference_reads, + } = transforms::ast_driver::run_against_source(working_source, written_names, config, project); + // a forward reference a lowering wrote as a string reads its names when python evaluates + // it, against the module's namespace, so those are bound where they are imported + eager_names.extend(forward_reference_reads); if let Some(first) = ast_errors.first() { return Err(first.clone().into()); } @@ -535,7 +566,7 @@ pub fn transpile_typed_with_report( let (final_output, requirements) = run_import_redirect_phase(output, config); let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; - let final_output = run_lazy_import_phase(final_output, config, &eager_imports, &eager_names); + let final_output = run_lazy_import_phase(final_output, config, &eager_imports, &eager_names)?; let final_output = run_version_polyfill_phase(final_output, config); // phases 1-2c only prepend preambles at the top and edit within lines, so @@ -738,19 +769,21 @@ fn run_import_redirect_phase(source: String, config: &Config) -> (String, Runtim /// prepends `lazy ` (PEP 810, Python 3.15+) to every `import` and /// `from import` statement. Skips `from __future__` and star imports. /// -/// Gated on `min_version >= 3.15`: PEP 810 syntax doesn't parse on older -/// Python, so we leave imports eager when the target version can't handle -/// the keyword. A redundant `lazy` keyword written in source is stripped -/// in that case so the output stays valid +/// Below 3.15 the keyword does not parse, so the imports are lowered onto the +/// runtime's polyfill instead: a module import binds a lazily loaded module, and +/// a `from` import binds nothing at all, every read of the name being written as +/// a read through the module's `_LazyFrom`, which resolves it on first use. A +/// redundant `lazy` keyword written in source is stripped so the output stays +/// valid /// /// `eager` names the modules that must not be deferred whatever the target: a /// module declaring a conformance exists at runtime only because its /// registration ran, so deferring it defers the conformance out of existence /// -/// `eager_names` names the *bindings* that must be bound to the real object: a -/// lazy proxy cannot stand where cpython checks for a real class, which is what -/// `except` does, nor where a special form is told apart by identity, nor in an -/// annotation something reads back +/// `eager_names` names the *bindings* that must be bound to the real object where +/// the import is written: what `except` catches, a special form, and a name an +/// annotation or a decorator reads, whose readers look them up in the module's +/// namespace or tell them apart by identity as the module is executing /// /// A stub defers nothing: it is never executed, and deferring its imports would /// hand a checker a call result where the stub declares a module or a class @@ -759,12 +792,12 @@ fn run_lazy_import_phase( config: &Config, eager: &[String], eager_names: &[String], -) -> String { +) -> Result { if !config.lazy_imports { - return source; + return Ok(source); } - let (db, file) = make_in_memory_db(&source); + let (db, file) = make_in_memory_python_db(&source); let source_ref = ruff_db::source::source_text(&db, file); let src = source_ref.as_str(); let module = ruff_db::parsed::parsed_module( @@ -772,17 +805,15 @@ fn run_lazy_import_phase( ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); + let model = ty_python_semantic::SemanticModel::new( + &db, + ty_python_semantic::Db::program_file(&db, file), + ); // the lowering writes imports of its own — an enum's `ClassVar` among them — which // the source the transpile began from never held, so the output is asked too. the // annotations are read off the output for the same reason: a lowering writes those let mut eager_names = eager_names.to_vec(); - eager_names.extend( - ty_python_semantic::SemanticModel::new( - &db, - ty_python_semantic::Db::program_file(&db, file), - ) - .eagerly_imported_names(), - ); + eager_names.extend(model.eagerly_imported_names()); eager_names.extend(transforms::lazy_import::annotation_names( module.suite(), src, @@ -796,30 +827,56 @@ fn run_lazy_import_phase( } else { transforms::lazy_import::Deferral::Polyfill }; - // the runtime is what the polyfill itself runs on, and a helper reached - // through a proxy would be a proxy call on every use + let deferrable = if deferral == transforms::lazy_import::Deferral::Polyfill + && config.defer_from_imports + { + transforms::lazy_import::deferrable_reads(module.suite(), model.deferrable_from_imports()) + } else { + std::collections::HashMap::default() + }; + let code = CodeNames::of(module.suite()); + let written = transforms::repeated_underscore::WrittenNames::new(src).with_code(&code); + let holder = written.fresh("_by_lazy"); + // the runtime is what the polyfill itself runs on let mut eager = eager.to_vec(); eager.extend(config.runtime_module.clone()); - let mut lazy = transforms::lazy_import::LazyImport::new(src, deferral, &eager, &eager_names); + let mut lazy = transforms::lazy_import::LazyImport::new( + src, + deferral, + &eager, + &eager_names, + deferrable.keys().copied().collect(), + &holder, + ); for stmt in module.suite() { lazy.visit_stmt(stmt); } - let needs_module = lazy.needs_module_helper; - let needs_attr = lazy.needs_attr_helper; - let needs_ty_ext = lazy.needs_ty_ext_marker; - let needs_character_class = lazy.needs_character_class; + let mut edits = std::mem::take(&mut lazy.edits); + for (alias, bound) in &lazy.deferred { + for read in deferrable.get(alias).into_iter().flatten() { + edits.push(Fix::safe_edit(Edit::range_replacement( + format!("{holder}.{bound}"), + *read, + ))); + } + } + let deferred_names: Vec = lazy + .deferred + .iter() + .map(|(_, bound)| bound.clone()) + .collect(); let helpers = transforms::lazy_import::polyfill_helpers( - needs_module, - needs_attr, - needs_ty_ext, - needs_character_class, + lazy.needs_module_helper, + !deferred_names.is_empty(), + lazy.needs_ty_ext_marker, + lazy.needs_character_class, ); - if lazy.edits.is_empty() && helpers.is_empty() { - return source; + if edits.is_empty() && helpers.is_empty() { + return Ok(source); } - let (mut body, _) = apply_transforms_once(src, lazy.edits); + let (mut body, _) = apply_transforms_once(src, edits); // phase 0 already gave the module the helpers *it* needed. where that was an import of the // runtime written beside it, the helpers this phase needs belong on that same line rather // than on a second one naming the same module @@ -828,17 +885,58 @@ fn run_lazy_import_phase( { let line = runtime::import_line(module, helpers.iter().copied()); let left_over = transforms::ast_driver::merge_into_own_imports(&mut body, vec![line]); - return match left_over.first() { - Some(line) => splice_preamble(&body, &format!("{line}\n")), - None => body, - }; - } - let preamble = runtime_preamble(config, &helpers, &body); - if preamble.is_empty() { - body + if let Some(line) = left_over.first() { + body = splice_preamble(&body, &format!("{line}\n")); + } } else { - splice_preamble(&body, &preamble) + let preamble = runtime_preamble(config, &helpers, &body); + if !preamble.is_empty() { + body = splice_preamble(&body, &preamble); + } + } + if deferred_names.is_empty() { + return Ok(body); + } + let globals = written.builtin("globals"); + let lines = transforms::lazy_import::holder_lines(&holder, &deferred_names, &globals); + let mut block = String::new(); + for import in written.builtin_imports(&lines, config.min_version.minor) { + block.push_str(&import); + block.push('\n'); } + block.push_str(&lines); + insert_after_binding(&body, runtime::LAZY_FROM.name(), &block) +} + +/// `text` inserted on the line after the top-level statement of `module` that binds `name`, +/// a runtime helper the preamble provides. a module that does not bind it was not given it, +/// and the lines that need it have nowhere to go +fn insert_after_binding(module: &str, name: &str, text: &str) -> Result { + let parsed = + ruff_python_parser::parse_unchecked_source(module, ruff_python_ast::PySourceType::Python); + let Some(statement) = parsed + .suite() + .iter() + .find(|stmt| runtime::bindings(stmt).contains_key(name)) + else { + return Err(format!( + "the lazy-import polyfill needs `{name}`, and the module was not given it" + )); + }; + let end = usize::from(statement.end()); + let at = module[end..] + .find('\n') + .map_or(module.len(), |newline| end + newline + 1); + let separator = if at == module.len() && !module.ends_with('\n') { + "\n" + } else { + "" + }; + Ok(format!( + "{}{separator}{text}{}", + &module[..at], + &module[at..] + )) } /// The lines that give `module`, python, the runtime helpers it calls: an import of diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs index 7e0c258a8a..12540b519b 100644 --- a/crates/by_transforms/src/runtime.rs +++ b/crates/by_transforms/src/runtime.rs @@ -20,12 +20,12 @@ //! a helper resolves to the top-level statement that binds its name, as our own //! parser reads the file. a comment written between two definitions falls outside //! both and never reaches the output; one inside a body is part of it. a -//! top-level statement that binds nothing, such as -//! `_by_forward_operators(_LazyAttr)`, travels with the definition above it +//! top-level statement that binds nothing, such as a call that finishes setting up +//! a class, travels with the definition above it //! //! # dependencies //! -//! helpers call each other: `_lazy_attr` resolves through `_lazy_module`, +//! helpers call each other: `_LazyFrom` resolves through `_lazy_module`, //! `_soundness_parametric` through `_parametric_is`. what a helper needs is read //! out of the calls in its body, so asking for one yields a set that runs @@ -106,7 +106,8 @@ helpers! { DEFAULT = "_by_default", RAISES = "_by_raises", LAZY_MODULE = "_lazy_module", - LAZY_ATTR = "_lazy_attr", + LAZY_FROM_HOLDER = "_LazyFrom", + LAZY_FROM = "_lazy_from", TY_EXT_MARKER = "_TyExtMarker", CHARACTER = "Character", MATCH_MISS = "_by_match_miss", @@ -1196,48 +1197,6 @@ mod tests { assert_eq!(DISCARD.name(), ty_python_semantic::DISCARD_ADAPTER); } - /// the proxy's operator forwarding is read by nothing; it has to travel with - /// the class it patches, or the proxy is left with no operators at all - #[test] - fn set_up_code_travels_with_its_definition() { - let pasted = inline( - [LAZY_ATTR], - &[], - crate::transforms::repeated_underscore::WrittenNames::new(""), - ) - .concat(); - assert!(pasted.contains("class _LazyAttr:"), "{pasted}"); - assert!( - pasted.contains("_by_forward_operators(_LazyAttr)"), - "{pasted}" - ); - assert!(pasted.contains("def _lazy_module("), "{pasted}"); - } - - /// the proxy's three overrides of `object`'s own members are reported by - /// `missing-override-decorator`, and the emitted python is read by this project's own - /// checker. `@override` is `typing.override`, which arrived in 3.12, and the polyfill - /// runs on 3.9 — so each carries the suppression the decorator would have been - #[test] - fn the_proxies_overrides_do_not_report_in_the_emitted_python() { - let pasted = inline( - [LAZY_ATTR], - &[], - crate::transforms::repeated_underscore::WrittenNames::new(""), - ) - .concat(); - for member in ["__class__", "__setattr__", "__delattr__"] { - let line = pasted - .lines() - .find(|line| line.trim_start().starts_with(&format!("def {member}("))) - .unwrap_or_else(|| panic!("`{member}` is no longer defined:\n{pasted}")); - assert!( - line.contains("# ty: ignore[missing-override-decorator]"), - "`{member}` lost its suppression: {line}" - ); - } - } - /// the reads a pasted definition renames are the names python itself looks up in the /// module's globals: its compiler's symbol table answers the same question, so the two /// are held to agree over the whole runtime. with no python to ask, there is nothing @@ -1289,7 +1248,7 @@ print(" ".join(sorted(found))) /// in a function of its own leaves them reading it as written #[test] fn a_pasted_definition_reads_a_builtin_the_module_binds_under_its_own_name() { - let module = "def getattr(o, n): return 42\ndef f(format): return format\n"; + let module = "def getattr(o, n): return 42\ndef f(sorted): return sorted\n"; let parsed = ruff_python_parser::parse_unchecked_source( module, ruff_python_ast::PySourceType::Python, @@ -1297,15 +1256,12 @@ print(" ".join(sorted(found))) let code = crate::transforms::repeated_underscore::CodeNames::of(parsed.suite()); let written = crate::transforms::repeated_underscore::WrittenNames::new(module).with_code(&code); - let pasted = inline([LAZY_ATTR], parsed.suite(), written).concat(); + let pasted = inline([LAZY_FROM_HOLDER], parsed.suite(), written).concat(); assert!( - pasted.contains("v = getattr2(m, self._by_attr)") && !pasted.contains("getattr(m,"), - "{pasted}" - ); - assert!( - pasted.contains("lambda s, f: format(s._by_resolve(), f)"), + pasted.contains("return getattr2(m, attr)") && !pasted.contains("getattr(m,"), "{pasted}" ); + assert!(pasted.contains("return sorted(names)"), "{pasted}"); } /// a function or class that declares a builtin's name `global` and binds it rebinds the @@ -1320,7 +1276,7 @@ print(" ".join(sorted(found))) " for getattr in [len]:\n", " pass\n", "def local():\n", - " format = 1\n", + " sorted = 1\n", ); let parsed = ruff_python_parser::parse_unchecked_source( module, @@ -1329,15 +1285,12 @@ print(" ".join(sorted(found))) let code = crate::transforms::repeated_underscore::CodeNames::of(parsed.suite()); let written = crate::transforms::repeated_underscore::WrittenNames::new(module).with_code(&code); - let pasted = inline([LAZY_ATTR], parsed.suite(), written).concat(); + let pasted = inline([LAZY_FROM_HOLDER], parsed.suite(), written).concat(); assert!( - pasted.contains("v = getattr2(m, self._by_attr)") && !pasted.contains("getattr(m,"), + pasted.contains("return getattr2(m, attr)") && !pasted.contains("getattr(m,"), "{pasted}" ); // a local binding leaves the module's alone - assert!( - pasted.contains("lambda s, f: format(s._by_resolve(), f)"), - "{pasted}" - ); + assert!(pasted.contains("return sorted(names)"), "{pasted}"); } } diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py index f73412db3e..a4ed952803 100644 --- a/crates/by_transforms/src/runtime/_by_runtime.py +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -40,8 +40,14 @@ async def _soundness_aiter(_it, _t): # `_target` carries def _soundness_once_bound(_names, _v, _check, _target, *_rest): _globals = _target.__globals__ + # a deferred `from` import is bound once its statement has run, which its + # module's `_LazyFrom` knows, and is in the namespace only once something has + # read it. the `_LazyFrom` is the module's `__getattr__`'s receiver, asked by + # what it holds rather than by its class: this check is often pasted in by + # another pass than the import's, and each paste has a class of its own + _deferred = getattr(getattr(_globals.get("__getattr__"), "__self__", None), "__by_from__", {}) for _name in _names: - if _name not in _globals: + if _name not in _globals and _deferred.get(_name) is None: return _v return _check(_v, _target(), *_rest) @@ -1261,18 +1267,25 @@ def _type_argument(owner, name): # --- lazy imports ---------------------------------------------------------- # below python 3.15 there is no `lazy` keyword, so an import is rewritten to a -# call: a module import becomes `_lazy_module`, and a `from` import a proxy that -# resolves the attribute the first time anything touches it +# call. a module import becomes `_lazy_module`, which binds the real module and +# runs its body on first attribute access # -# `__class__` makes `isinstance(proxy, C)` work, and `__instancecheck__` makes -# `isinstance(x, proxy)` work for a lazily-imported class (`isinstance` looks -# `__instancecheck__` up on `type(classinfo)`, which is `_LazyAttr`). -# `type(proxy)` and `proxy is x` cannot be fixed by any proxy — that is exactly -# why PEP 810 is a language feature — and are documented limits of this -# polyfill +# a `from` import binds nothing where it is written. the module holds one +# `_LazyFrom` whose `__dict__` is the module's own namespace, the statement +# registers its names with it (`_lazy_from`), and every read of an imported +# name is written as a read off it. a name already in the namespace is a plain +# dict hit; a missing one is resolved from its module, stored in the namespace +# and returned, so from its first use on the name is the real object, exactly +# as PEP 810 makes it. sharing the namespace is also what keeps patching +# working: `mock.patch("app.svc.fetch")` writes `app.svc`'s namespace, which is +# what every read in `app.svc` looks in first +# +# the module's `__getattr__` and `__dir__` serve the names nothing has read yet +# to other modules: `from app.svc import fetch`, `app.svc.fetch`, `dir(app.svc)` -import importlib as _by_il, importlib.util as _by_iu, sys as _by_sys +import builtins as _by_builtins, importlib as _by_il, importlib.util as _by_iu, sys as _by_sys +import _thread as _by_thread # a relative import names its module against the package doing the importing, # which only that module knows, so it hands `__package__` in def _lazy_module(name, package=None): @@ -1285,97 +1298,141 @@ def _lazy_module(name, package=None): return _by_il.import_module(name) spec = _by_iu.find_spec(name) if spec is None or spec.loader is None: - raise ImportError(f"No module named {name!r}", name=name) + raise ModuleNotFoundError(f"No module named {name!r}", name=name) spec.loader = _by_iu.LazyLoader(spec.loader) mod = _by_iu.module_from_spec(spec) spec.loader.exec_module(mod) return _by_sys.modules.setdefault(name, mod) -def _by_forward_operators(proxy): - import operator as op - def one(f): return lambda s: f(s._by_resolve()) - def two(f): return lambda s, o: f(s._by_resolve(), o) - def rtwo(f): return lambda s, o: f(o, s._by_resolve()) - for n, f in (("add", op.add), ("sub", op.sub), ("mul", op.mul), ("matmul", op.matmul), - ("truediv", op.truediv), ("floordiv", op.floordiv), ("mod", op.mod), - ("divmod", divmod), ("pow", op.pow), ("lshift", op.lshift), - ("rshift", op.rshift), ("and", op.and_), ("xor", op.xor), ("or", op.or_)): - setattr(proxy, "__" + n + "__", two(f)) - setattr(proxy, "__r" + n + "__", rtwo(f)) - for n in ("lt", "le", "eq", "ne", "gt", "ge"): - setattr(proxy, "__" + n + "__", two(getattr(op, n))) - for n, f in (("neg", op.neg), ("pos", op.pos), ("abs", abs), ("invert", op.inv), - ("len", len), ("iter", iter), ("next", next), ("bool", bool), - ("str", str), ("repr", repr), ("bytes", bytes), ("int", int), - ("float", float), ("complex", complex), ("index", op.index), - ("hash", hash), ("reversed", reversed)): - setattr(proxy, "__" + n + "__", one(f)) - setattr(proxy, "__getitem__", two(op.getitem)) - setattr(proxy, "__contains__", two(op.contains)) - setattr(proxy, "__delitem__", two(op.delitem)) - setattr(proxy, "__setitem__", lambda s, k, v: op.setitem(s._by_resolve(), k, v)) - setattr(proxy, "__format__", lambda s, f: format(s._by_resolve(), f)) - setattr(proxy, "__round__", lambda s, *a: round(s._by_resolve(), *a)) - setattr(proxy, "__enter__", lambda s: s._by_resolve().__enter__()) - setattr(proxy, "__exit__", lambda s, *a: s._by_resolve().__exit__(*a)) - -class _LazyAttr: - __slots__ = ("_by_mod", "_by_attr", "_by_val", "_by_has") - def __init__(self, mod, attr): - object.__setattr__(self, "_by_mod", mod) - object.__setattr__(self, "_by_attr", attr) - object.__setattr__(self, "_by_val", None) - object.__setattr__(self, "_by_has", False) - def _by_resolve(self): - if not self._by_has: - m = _lazy_module(self._by_mod) - try: - v = getattr(m, self._by_attr) - except AttributeError: - # a submodule rather than an attribute: `urllib/__init__.py` never - # imports `parse`, and cpython binds it only because `__import__` is - # handed a fromlist. reading the attribute alone never triggers that - try: - v = _by_il.import_module(self._by_mod + "." + self._by_attr) - except ImportError: - # worded as the import machinery words it, down to the module's - # file: a `from x import y` that fails is something programs catch - # and report, so the report must not say where the import was - # written. `name_from` is left off — cpython's own constructor - # only took it from 3.12, and this polyfill runs on 3.9 - p = getattr(m, "__file__", None) - raise ImportError("cannot import name " + repr(self._by_attr) + - " from " + repr(self._by_mod) + - ("" if p is None else " (" + p + ")"), - name=self._by_mod, path=p) from None - object.__setattr__(self, "_by_val", v) - object.__setattr__(self, "_by_has", True) - return self._by_val - # the three below really do override `object`'s, and `@override` is what says - # so — but it is `typing.override`, which arrived in 3.12, and this polyfill - # runs on 3.9. the suppression is what the decorator would have been - @property - def __class__(self): return self._by_resolve().__class__ # ty: ignore[missing-override-decorator] - def __getattr__(self, k): return getattr(self._by_resolve(), k) - def __setattr__(self, k, v): setattr(self._by_resolve(), k, v) # ty: ignore[missing-override-decorator] - def __delattr__(self, k): delattr(self._by_resolve(), k) # ty: ignore[missing-override-decorator] - def __call__(self, *a, **k): return self._by_resolve()(*a, **k) - def __class_getitem__(cls, k): return cls - def __instancecheck__(self, o): return isinstance(o, self._by_resolve()) - def __subclasscheck__(self, o): return issubclass(o, self._by_resolve()) - def __mro_entries__(self, bases): - r = self._by_resolve() - m = getattr(r, "__mro_entries__", None) - if m is None: return (r,) - return m(tuple(r if b is self else b for b in bases)) +# the names a module's deferred `from` imports bind. `names` is every one of them, +# so a read of one whose statement has not run yet is what python's global lookup +# gives — the builtin of that name, or a `NameError` — while any other attribute is +# an `AttributeError`, as tools that walk a module's namespace probing what they +# find expect. no deferred name begins with two underscores, so none can be +# mistaken for an attribute of the holder itself +class _LazyFrom: + __slots__ = ("__dict__", "__by_from__", "__by_resolving__") + + def __init__(self, namespace, names): + self.__dict__ = namespace + self.__by_from__ = dict.fromkeys(names) + self.__by_resolving__ = set() + + def __getattr__(self, name): + source = self.__by_from__.get(name) + if source is None: + if name in self.__by_from__: + # `if PY2: from itertools import izip as zip` leaves `zip` the + # builtin where the import did not run. the builtins are the ones + # the module runs with, a module or its dict + b = self.__dict__.get("__builtins__", _by_builtins) + b = b if isinstance(b, dict) else b.__dict__ + if name in b: + return b[name] + error = NameError(f"name {name!r} is not defined") + # `NameError.name` arrived in 3.10, and this polyfill runs on 3.9 + error.name = name + raise error + raise AttributeError(name) + # while a name resolves, the module's own `__getattr__` does not answer for + # it: `from . import helper` in a package's `__init__` reads `helper` off the + # package, which is this module, and has to fall through to importing the + # submodule rather than resolve itself again + key = (name, _by_thread.get_ident()) + self.__by_resolving__.add(key) + try: + value = _lazy_from_value(*source) + finally: + self.__by_resolving__.discard(key) + self.__dict__[name] = value + return value + + # the module's `__getattr__`: a deferred name another module reads before + # this one has + def __by_getattr__(self, name): + if ( + self.__by_from__.get(name) is None + or (name, _by_thread.get_ident()) in self.__by_resolving__ + ): + raise AttributeError( + f"module {self.__dict__.get('__name__')!r} has no attribute {name!r}" + ) + return getattr(self, name) + + # the module's `__dir__` + def __by_dir__(self): + names = set(self.__dict__) + names.update(name for name, source in self.__by_from__.items() if source is not None) + return sorted(names) + + +# what `from mod import attr` binds, read as python's import machinery reads it +def _lazy_from_value(mod, attr): + # the module is needed now, so it is imported as an ordinary import would, and + # whatever its body raises is the import's error, an `AttributeError` included + m = _by_sys.modules.get(mod) + if m is None: + m = _by_il.import_module(mod) + else: + # one another import made lazily runs its body on its first attribute read + m.__dict__ + try: + return getattr(m, attr) + except AttributeError: + pass + name = getattr(m, "__name__", mod) + # a package's submodule nothing has imported yet: `urllib/__init__.py` never + # imports `parse`, and cpython binds it because `__import__` is handed a + # fromlist, which imports it. a submodule that does not exist is the name + # missing; a module its body cannot import is the error that body raised + if hasattr(m, "__path__"): + submodule = name + "." + attr + try: + _by_il.import_module(submodule) + except ModuleNotFoundError as error: + if error.name != submodule or ( + submodule in _by_sys.modules and _by_sys.modules[submodule] is None + ): + raise + try: + return getattr(m, attr) + except AttributeError: + pass + value = _by_sys.modules.get(submodule) + if value is not None: + return value + # worded as the import machinery words it, down to the module's file: a + # `from x import y` that fails is something programs catch and report, so the + # report must not say where the import was written + path = getattr(m, "__file__", None) + where = path if isinstance(path, str) else "unknown location" + if getattr(getattr(m, "__spec__", None), "_initializing", False): + message = ( + f"cannot import name {attr!r} from partially initialized module {name!r} " + f"(most likely due to a circular import) ({where})" + ) + else: + message = f"cannot import name {attr!r} from {name!r} ({where})" + error = ImportError(message, name=name, path=path) + # `name_from` arrived in 3.12, and this polyfill runs on 3.9 + if hasattr(error, "name_from"): + error.name_from = attr + raise error -_by_forward_operators(_LazyAttr) -def _lazy_attr(mod, attr, package=None): +# a `from` import: registers each name it binds, as `{bound: imported}`, with the +# module's `_LazyFrom`. a relative import names its module against the package +# doing the importing, which only that module knows, so it hands `__package__` in +# +# a value the name already holds is from before the statement ran — a reload runs +# the module again over the namespace it had — and the statement replaces it +def _lazy_from(lazy, mod, names, package=None): if package is not None: mod = _by_iu.resolve_name(mod, package) - return _LazyAttr(mod, attr) + for bound, attr in names.items(): + lazy.__dict__.pop(bound, None) + lazy.__by_from__[bound] = (mod, attr) # a type-only marker for `ty_extensions` names, which have no runtime import diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index fe8a2af6fa..4e2fb50373 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -96,6 +96,10 @@ pub(crate) struct PassContext { /// Full source lines to prepend to the file (e.g. `from typing import cast`). /// Deduped before emission. pub(crate) required_imports: Vec, + /// the names read by a forward reference a pass writes as a string outside an + /// annotation, which python evaluates against the module's namespace when anything + /// evaluates it — so the lazy-import polyfill binds them where they are imported + pub(crate) forward_reference_reads: Vec, /// Runtime helpers the emitted code calls, by the name it calls them under. /// The driver turns these into an import of the module a build writes them /// to, or — when there is no such module — into the definitions themselves. @@ -1169,6 +1173,15 @@ fn merge_from_imports(lines: Vec) -> (Vec, Vec) { (from_lines, other) } +/// what phase 0 makes of a module: the rewritten source, the errors, the line map back to the +/// source, and the names read by the forward references its passes wrote as strings +pub(crate) struct Phase0<'a> { + pub(crate) source: Cow<'a, str>, + pub(crate) errors: Vec, + pub(crate) line_map: Vec>, + pub(crate) forward_reference_reads: Vec, +} + /// Run every registered AST pass against `source` and splice the rewritten /// statements back into the source text. Returns a borrowed `Cow` when /// nothing changed. @@ -1183,7 +1196,7 @@ pub(crate) fn run_against_source<'a>( written: repeated_underscore::WrittenNames, config: &Config, project: SemanticSource<'_>, -) -> (Cow<'a, str>, Vec, Vec>) { +) -> Phase0<'a> { // blank the keyword-prefix type markers — use-site variance and the // `literal`/`final` type modifiers — out up front; downstream passes // (callable, intersection) copy operand source verbatim and would leak the @@ -1218,13 +1231,23 @@ pub(crate) fn run_against_source<'a>( if !parsed_handle.errors().is_empty() { let cow = blanked.stripped(source); let table = crate::source_map::line_table(cow.as_ref(), &[]); - return (cow, Vec::new(), table); + return Phase0 { + source: cow, + errors: Vec::new(), + line_map: table, + forward_reference_reads: Vec::new(), + }; } let parsed = parse_unchecked_source(source_ref, PySourceType::BasedPython); if !parsed.errors().is_empty() { let cow = blanked.stripped(source); let table = crate::source_map::line_table(cow.as_ref(), &[]); - return (cow, Vec::new(), table); + return Phase0 { + source: cow, + errors: Vec::new(), + line_map: table, + forward_reference_reads: Vec::new(), + }; } // the parentheses grouping an expression are visible in the tokens and // nowhere in the AST, so anything that needs them has to measure now — the @@ -1965,7 +1988,12 @@ pub(crate) fn run_against_source<'a>( { let cow = blanked.stripped(source); let table = crate::source_map::line_table(cow.as_ref(), &[]); - return (cow, ctx.errors, table); + return Phase0 { + source: cow, + errors: ctx.errors, + line_map: table, + forward_reference_reads: ctx.forward_reference_reads, + }; } // splice changed statements back into the source string. process highest @@ -2237,7 +2265,12 @@ pub(crate) fn run_against_source<'a>( preamble_end, &ctx.extension_backing_functions, ); - (Cow::Owned(out), ctx.errors, table) + Phase0 { + source: Cow::Owned(out), + errors: ctx.errors, + line_map: table, + forward_reference_reads: ctx.forward_reference_reads, + } } #[cfg(test)] @@ -2434,7 +2467,7 @@ mod driver_tests { #[test] fn double_coalesce_spliced() { let src = "x = None\na = x ?? x ?? \"fallback\"\n"; - let (out, _, _) = run_against_source( + let Phase0 { source: out, .. } = run_against_source( src, repeated_underscore::WrittenNames::new(src), &Config::test_default(), diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 594375bc84..af4e4fcecf 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -70,6 +70,10 @@ pub(crate) struct GenericPolyfill<'src> { /// ranges of `pending_edits` this pass re-rendered; the driver drops them so /// the stale un-renamed text cannot win the overlap race superseded: Vec, + /// the names a bound or default reads that is written as a string — a forward + /// reference, which python evaluates against the module's namespace when anything + /// evaluates it at all + forward_reference_reads: Vec, /// the `TypeVar` definitions a polyfilled `class` / `def` needs, keyed on the /// start of the line its definition begins on /// @@ -236,6 +240,7 @@ impl<'src> GenericPolyfill<'src> { symbolic_substitutions, pending_edits, superseded: Vec::new(), + forward_reference_reads: Vec::new(), statement_prefixes: Vec::new(), } } @@ -474,6 +479,13 @@ impl<'src> GenericPolyfill<'src> { self.needed_imports.paramspec = true; } + /// record that `expr` is written as a forward reference, a string python evaluates + /// against the module's namespace when anything evaluates it + fn note_forward_reference(&mut self, expr: &Expr) { + self.forward_reference_reads + .extend(names_read(expr).into_iter().map(str::to_owned)); + } + /// lower `params`, each declared under the name at the same position in `names` fn process_type_params( &mut self, @@ -483,12 +495,16 @@ impl<'src> GenericPolyfill<'src> { ) -> ProcessedTypeParams { // a bound that reads a class's name through the class is written ahead of the class, // so it is a string, which `TypeVar` keeps as a forward reference + // — and whether it is one let through_classes = |text: String| { let read_through = apply_renames_to_rendered(&text, qualified); if read_through == text { - text + (text, false) } else { - super::source_util::python_string_literal(&read_through) + ( + super::source_util::python_string_literal(&read_through), + true, + ) } }; let mut generic_args: Vec = Vec::new(); @@ -587,18 +603,22 @@ impl<'src> GenericPolyfill<'src> { ) .unwrap_or_else(|| self.src(bound.range()).to_owned()) }; - extra_args.push(format!( - "bound={}", - through_classes(apply_renames_to_rendered(&bound_src, &visible)) - )); + let (written, quoted) = + through_classes(apply_renames_to_rendered(&bound_src, &visible)); + if quoted { + self.note_forward_reference(bound); + } + extra_args.push(format!("bound={written}")); } } if let Some(default) = &tv.default { - extra_args.push(format!( - "default={}", - through_classes(self.default_arg(default, &visible)) - )); + let (written, quoted) = + through_classes(self.default_arg(default, &visible)); + if quoted { + self.note_forward_reference(default); + } + extra_args.push(format!("default={written}")); } // basedpython variance keywords: forward `out`/`in`/`in out` @@ -2118,6 +2138,8 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { .map(|(index, _)| index) .collect(); ctx.withdrawn_text_edits.extend(withdrawn); + ctx.forward_reference_reads + .extend(std::mem::take(&mut inner.forward_reference_reads)); let emits_any = inner.needed_imports_any; for line in std::mem::take(&mut inner.needed_imports).into_lines(&inner.constructors, self.written) diff --git a/crates/by_transforms/src/transforms/inferred_annotation.rs b/crates/by_transforms/src/transforms/inferred_annotation.rs index 4d9e2e226e..4619aeb4bf 100644 --- a/crates/by_transforms/src/transforms/inferred_annotation.rs +++ b/crates/by_transforms/src/transforms/inferred_annotation.rs @@ -376,7 +376,10 @@ mod tests { ) .unwrap(); assert!(out.contains("decimal = _lazy_module(\"decimal\")"), "{out}"); - assert!(out.contains("d: \"decimal.Decimal\" = Dec(1)"), "{out}"); + assert!( + out.contains("d: \"decimal.Decimal\" = _by_lazy.Dec(1)"), + "{out}" + ); assert!(!out.contains("TYPE_CHECKING"), "{out}"); } diff --git a/crates/by_transforms/src/transforms/lazy_import.rs b/crates/by_transforms/src/transforms/lazy_import.rs index 7b81e1c7f9..3369ab1132 100644 --- a/crates/by_transforms/src/transforms/lazy_import.rs +++ b/crates/by_transforms/src/transforms/lazy_import.rs @@ -3,10 +3,12 @@ //! Two emission strategies, chosen by target Python version: //! - **`min_version >= 3.15`** — prepend the `lazy` keyword (PEP 810) //! - **`min_version < 3.15`** — rewrite the statement to call a runtime -//! polyfill (`_lazy_module` for module imports, `_lazy_attr` for `from` -//! imports). The helpers wrap `importlib.util.LazyLoader` and a small proxy -//! class, and live in [`crate::runtime`] with the rest of what the emitted -//! python calls +//! polyfill. a module import binds `_lazy_module(...)`, a module whose body +//! runs on first attribute access. a `from` import binds nothing: it +//! registers its names with the module's `_LazyFrom` (`_lazy_from(...)`), +//! and every read of one is written as a read through it, which resolves +//! the name on first use — see [`deferrable_reads`]. The helpers live in +//! [`crate::runtime`] with the rest of what the emitted python calls //! //! Both modes skip: //! - `from __future__ import ...` — compiler directive @@ -15,7 +17,8 @@ //! bound to `False` //! //! The polyfill additionally skips forms it can't safely rewrite: -//! - relative imports (`from .pkg import x`) +//! - a `from` import name whose reads cannot all be rewritten, or which is +//! read where only the object bound at the import will do //! - `import a.b` without an alias (binds the top package, which //! `LazyLoader` does not register) //! - bootstrap modules (`sys`, `importlib*`) — the helpers depend on them — and @@ -27,9 +30,13 @@ //! A stub defers nothing — see [`Deferral::Never`]. use ruff_diagnostics::{Edit, Fix}; +use ruff_python_ast::visitor::source_order::{ + SourceOrderVisitor, walk_interpolated_string_element, walk_t_string, +}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom}; use ruff_text_size::{Ranged, TextRange, TextSize}; +use std::collections::{HashMap, HashSet}; /// How a module-level import is deferred. #[derive(Clone, Copy, PartialEq, Eq)] @@ -58,22 +65,26 @@ pub(crate) struct LazyImport<'src> { /// registers it at import, so deferring the import defers the conformance /// out of existence eager: Vec, - /// bindings that must be bound to the real object rather than to a proxy. - /// cpython's `except` checks that what it catches is a class inheriting - /// `BaseException` and never consults `__instancecheck__`, so an exception - /// class reached through the proxy raises `TypeError` from the handler; a - /// special form, and a name an annotation reads, are told apart by identity - /// by whatever reads them — see [`annotation_names`] + /// bindings that must be made where the import is written: an exception + /// class, a special form, and a name an annotation or a decorator reads — see + /// [`ty_python_semantic::SemanticModel::eagerly_imported_names`], + /// [`annotation_names`] and [`decorator_names`] eager_names: Vec, deferral: Deferral, pub(crate) edits: Vec, /// True when at least one statement was rewritten to call /// `_lazy_module`; the preamble must define the module helper pub(crate) needs_module_helper: bool, - /// True when at least one statement was rewritten to call `_lazy_attr`; - /// the preamble must define the `_LazyAttr` proxy (and `_lazy_module`, - /// which `_lazy_attr` calls) - pub(crate) needs_attr_helper: bool, + /// the `from` import names [`deferrable_from_imports`] found every read of, and + /// whose names may be deferred, by the range of their `ast::Alias` + /// + /// [`deferrable_from_imports`]: ty_python_semantic::SemanticModel::deferrable_from_imports + deferrable: HashSet, + /// the name the module's `_LazyFrom` is bound under, which every read of a + /// deferred name is written through + holder: &'src str, + /// each name the walk deferred, with the range of the alias that imports it + pub(crate) deferred: Vec<(TextRange, String)>, /// True when at least one `from ty_extensions import X` was rewritten to /// a `_TyExtMarker` assignment; the preamble must define the marker pub(crate) needs_ty_ext_marker: bool, @@ -90,6 +101,8 @@ impl<'src> LazyImport<'src> { deferral: Deferral, eager: &[String], eager_names: &[String], + deferrable: HashSet, + holder: &'src str, ) -> Self { Self { source, @@ -99,7 +112,9 @@ impl<'src> LazyImport<'src> { deferral, edits: Vec::new(), needs_module_helper: false, - needs_attr_helper: false, + deferrable, + holder, + deferred: Vec::new(), needs_ty_ext_marker: false, needs_character_class: false, } @@ -157,8 +172,8 @@ impl<'src> LazyImport<'src> { /// a module the polyfill leaves eager. the helpers run on `sys` and `importlib`, and /// `builtins` is loaded before any module runs, so deferring it defers nothing — while - /// its proxy would stand where a lowering reads a builtin under a name of its own, and - /// fail an identity test such as `type(x) is str2` + /// a lowering's import of a builtin under a name of its own is one of the imports the + /// runtime preamble is placed after fn is_bootstrap(name: &str) -> bool { matches!(name, "sys" | "importlib" | "builtins") || name.starts_with("importlib.") } @@ -243,8 +258,9 @@ impl<'src> LazyImport<'src> { .is_some_and(|m| m.id.as_str() == "__future__"); let is_star = node.names.iter().any(|a| a.name.id.as_str() == "*"); // `TYPE_CHECKING` is a flag a checker reads statically, not an ordinary - // binding: deferring it rebinds the name to a proxy, and the name may - // only ever be bound to `False` + // binding: deferring it would make each read of it a read through the + // module's `_LazyFrom`, and a checker only ever accepts the name bound to + // `False` let binds_type_checking = node .names .iter() @@ -318,6 +334,10 @@ impl<'src> LazyImport<'src> { // `lines` the one import line they share is written into let mut eager_spellings: Vec = Vec::new(); let mut eager_line: Option = None; + // the names this statement registers with the module's `_LazyFrom`, as + // `(bound, imported)`, and the slot in `lines` the registration goes in + let mut registered: Vec<(&str, &str)> = Vec::new(); + let mut registration_line: Option = None; // whether the statement imported a bare `Character` that contributed no // binding line — the import must still be *removed* (the preamble class // defines `Character`), so force an empty replacement below @@ -352,11 +372,26 @@ impl<'src> LazyImport<'src> { } continue; } - if self.eager_names.iter().any(|eager| eager == bind) { - // the proxy cannot stand here, so re-emit the import as written. - // ahead of the relative branches: a relative import is the - // common shape inside a package, and its exceptions are caught - // just the same + let eager = self.eager_names.iter().any(|eager| eager == bind); + // `from . import x` is no different: `x` is read off the package, and + // imported as its submodule only when the package has no `x` + if !eager + // a name beginning with two underscores could be one the holder's own type + // answers, `__doc__` among them, and a read of it inside a class body is + // mangled the moment it is written as an attribute + && !bind.starts_with("__") + && self.deferrable.contains(&alias.range()) + { + if registration_line.is_none() { + registration_line = Some(lines.len()); + lines.push(String::new()); + } + registered.push((bind, name)); + self.deferred.push((alias.range(), bind.to_owned())); + } else { + // re-emit the import as written: the name is read where only the + // object itself will do, or somewhere a read of it cannot be + // rewritten let spelling = if bind == name { name.to_owned() } else { @@ -364,31 +399,12 @@ impl<'src> LazyImport<'src> { }; // names that shared one import statement go on writing one line // between them. the line is held at the position of the first of - // them, so it keeps its place among the lazy bindings around it + // them, so it keeps its place among the deferred names around it if eager_line.is_none() { eager_line = Some(lines.len()); lines.push(String::new()); } eager_spellings.push(spelling); - } else if is_relative && module_part.is_empty() { - // `from . import x` — `x` is a submodule of the current - // package. Resolve the relative target at runtime - self.needs_module_helper = true; - let rel = format!("{dots}{name}"); - lines.push(format!("{bind} = _lazy_module(\"{rel}\", __package__)")); - } else if is_relative { - // `from .pkg import x` — lazy attribute on the resolved - // parent, matching the `from pkg import x` shape - self.needs_attr_helper = true; - let rel = format!("{dots}{module_part}"); - lines.push(format!( - "{bind} = _lazy_attr(\"{rel}\", \"{name}\", __package__)" - )); - } else { - self.needs_attr_helper = true; - lines.push(format!( - "{bind} = _lazy_attr(\"{module_part}\", \"{name}\")" - )); } } if let Some(index) = eager_line { @@ -397,6 +413,18 @@ impl<'src> LazyImport<'src> { eager_spellings.join(", ") ); } + if let Some(index) = registration_line { + let names: Vec = registered + .iter() + .map(|(bind, name)| format!("\"{bind}\": \"{name}\"")) + .collect(); + let package = if is_relative { ", __package__" } else { "" }; + lines[index] = format!( + "_lazy_from({}, \"{dots}{module_part}\", {{{}}}{package})", + self.holder, + names.join(", ") + ); + } // `character_only` means a bare `Character` import produced no lines but // must still be dropped (the preamble class defines `Character`) so it // doesn't survive as a runtime `from ty_extensions import Character` @@ -409,10 +437,12 @@ impl<'src> LazyImport<'src> { } return; } - let indent = self.line_indent(node.range()); - let separator = format!("\n{indent}"); + // one line, padded to the statement's height: everything after the + // statement keeps the line it had, and the statement can share its line + // with others, as `if c: from m import x` does + let height = self.source[node.range()].matches('\n').count(); self.edits.push(Fix::safe_edit(Edit::range_replacement( - lines.join(&separator), + format!("{}{}", lines.join("; "), "\n".repeat(height)), node.range(), ))); } @@ -420,11 +450,12 @@ impl<'src> LazyImport<'src> { /// every name the annotations python keeps for introspection read, strings included /// -/// under the polyfill a `from` import is a proxy, and whatever reads an annotation back is -/// handed that proxy rather than the object: `dataclasses` tells a `KW_ONLY` pseudo-field -/// apart by asking whether the annotation *is* `dataclasses.KW_ONLY`, and -/// `typing.get_type_hints` answers with the proxy, which is not the class to anything -/// comparing by identity. a name read here is therefore bound eagerly +/// whatever reads an annotation back reads its names from the module's namespace — +/// `typing.get_type_hints` evaluates a string annotation against the module's globals, and +/// `dataclasses` tells a `KW_ONLY` pseudo-field apart by asking whether the annotation *is* +/// `dataclasses.KW_ONLY` as the class is made. under the polyfill a deferred `from` import +/// is in that namespace only once something has read it, so a name read here is bound +/// where the import is written /// /// the annotations are the ones python evaluates, or keeps as a string to evaluate on /// request: a parameter's and a return's, of a `def` at any depth, and a variable's in a @@ -521,14 +552,14 @@ pub(crate) fn annotation_names(suite: &[Stmt], source: &str) -> Vec { /// /// A decorator is the other thing read by identity rather than by value. `typing.overload` /// says the `def` under it is one arm of an overload set — but only while it *is* -/// `typing.overload`; reached through the polyfill's proxy it is an unknown callable, and a -/// checker reading the emitted python sees two ordinary functions with `...` bodies instead -/// of the overloads the source declared. The same goes for `final`, `abstractmethod`, +/// `typing.overload`; written as a read through the polyfill's `_LazyFrom` it is an unknown +/// callable, and a checker reading the emitted python sees two ordinary functions with `...` +/// bodies instead of the overloads the source declared. The same goes for `final`, `abstractmethod`, /// `dataclass` and everything else whose whole meaning is which object it is. /// /// Binding these eagerly costs no laziness at all: a decorator is applied while the module -/// is executing, so a proxy standing in for one resolves at import time anyway — the defer -/// buys nothing and loses the declaration. +/// is executing, so a deferred one resolves at import time anyway — the defer buys nothing +/// and loses the declaration pub(crate) fn decorator_names(suite: &[Stmt]) -> Vec { struct Reads<'n> { names: &'n mut Vec, @@ -572,6 +603,75 @@ pub(crate) fn decorator_names(suite: &[Stmt]) -> Vec { names.read } +/// the aliases whose names the polyfill may defer, each with every read of the name: the +/// imports [`deferrable_from_imports`] found every read of, less those with a read whose source +/// text python keeps — `f"{x=}"` prints the expression as written, and a t-string's +/// `Interpolation` carries it — since a read written through the module's `_LazyFrom` would +/// change that text +/// +/// [`deferrable_from_imports`]: ty_python_semantic::SemanticModel::deferrable_from_imports +pub(crate) fn deferrable_reads( + suite: &[Stmt], + imports: Vec, +) -> HashMap> { + #[derive(Default)] + struct KeptText { + in_t_string: bool, + ranges: Vec, + } + + impl<'a> SourceOrderVisitor<'a> for KeptText { + fn visit_t_string(&mut self, t_string: &'a ruff_python_ast::TString) { + let outer = std::mem::replace(&mut self.in_t_string, true); + walk_t_string(self, t_string); + self.in_t_string = outer; + } + + fn visit_interpolated_string_element( + &mut self, + element: &'a ruff_python_ast::InterpolatedStringElement, + ) { + if let ruff_python_ast::InterpolatedStringElement::Interpolation(interpolation) = + element + && (self.in_t_string || interpolation.debug_text.is_some()) + { + self.ranges.push(interpolation.range()); + } + // an f-string nested in a t-string's interpolation is part of its text all the same + walk_interpolated_string_element(self, element); + } + } + + let mut kept = KeptText::default(); + kept.visit_body(suite); + imports + .into_iter() + .filter(|import| { + !import + .reads + .iter() + .any(|read| kept.ranges.iter().any(|range| range.contains_range(*read))) + }) + .map(|import| (import.alias, import.reads)) + .collect() +} + +/// the statements that bind `holder`, the module's `_LazyFrom`, to every name in `names`, and +/// serve those names to other modules through the module's `__getattr__` and `__dir__` +/// +/// `globals` is the name the module reads the builtin under +pub(crate) fn holder_lines(holder: &str, names: &[String], globals: &str) -> String { + let names: Vec = names.iter().map(|name| format!("\"{name}\"")).collect(); + // a one-element tuple needs its comma + let trailing = if names.len() == 1 { "," } else { "" }; + format!( + "{holder} = _LazyFrom({globals}(), ({}{trailing}))\n\ + __getattr__ = {holder}.__by_getattr__\n\ + __dir__ = {holder}.__by_dir__\n", + names.join(", ") + ) +} + /// whether `test` is the `TYPE_CHECKING` flag, spelled bare or through the module it /// comes from fn is_type_checking_test(test: &ruff_python_ast::Expr) -> bool { @@ -609,8 +709,8 @@ impl<'ast> Visitor<'ast> for LazyImport<'_> { self.at_module_level = outer; } // an `if TYPE_CHECKING:` body never runs, so an import inside it has - // no execution to defer — and deferring it would rebind the name to - // a proxy, which is the one thing a type expression cannot read + // no execution to defer — and the checker reading the emitted python + // needs the name it binds for the type expressions that read it Stmt::If(node) if is_type_checking_test(&node.test) => { let outer = std::mem::replace(&mut self.at_module_level, false); walk_stmt(self, stmt); @@ -622,27 +722,28 @@ impl<'ast> Visitor<'ast> for LazyImport<'_> { } /// The runtime helpers a polyfill-mode lazified module calls, by the name it -/// calls them under. What each of those needs in turn — the proxy class, the -/// `sys` binding, the operator forwarding — is settled in [`crate::runtime`], -/// off the calls in their own bodies. +/// calls them under. What each of those needs in turn — `_lazy_module`, the +/// `sys` binding — is settled in [`crate::runtime`], off the calls in their own +/// bodies #[expect( clippy::fn_params_excessive_bools, reason = "independent which-helpers-are-needed flags, not a state machine" )] pub(crate) fn polyfill_helpers( needs_module: bool, - needs_attr: bool, + needs_from: bool, needs_ty_ext: bool, needs_character_class: bool, ) -> Vec { let mut helpers = Vec::new(); - // each only where the emitted code names it: a `from` import calls - // `_lazy_attr` alone, and what that calls in turn is the runtime's business + // each only where the emitted code names it, and what those call in turn + // is the runtime's business if needs_module { helpers.push(crate::runtime::LAZY_MODULE); } - if needs_attr { - helpers.push(crate::runtime::LAZY_ATTR); + if needs_from { + helpers.push(crate::runtime::LAZY_FROM_HOLDER); + helpers.push(crate::runtime::LAZY_FROM); } if needs_ty_ext { helpers.push(crate::runtime::TY_EXT_MARKER); @@ -703,8 +804,7 @@ mod tests { #[test] fn keyword_type_checking_unchanged() { - // deferring the flag would rebind the name to a proxy; a checker only - // ever accepts it bound to `False` + // a checker only ever accepts the flag bound to `False` check( "from typing import TYPE_CHECKING\n", "from typing import TYPE_CHECKING\n", @@ -760,8 +860,8 @@ mod tests { ) .unwrap(); assert!( - out.contains("def _lazy_module("), - "missing _lazy_module helper in:\n{out}" + out.contains("def _lazy_module(") || out.contains("class _LazyFrom:"), + "missing the polyfill's runtime in:\n{out}" ); assert!( out.ends_with(expected_body), @@ -787,32 +887,6 @@ mod tests { ); } - #[test] - fn attr_proxy_is_isinstance_transparent() { - // other lowerings (soundness checks, checked casts) pass imported - // names as runtime types; the proxy must delegate isinstance/issubclass - let out = transpile( - "from os.path import basename\n", - &Config { - lazy_imports: true, - ..Config::test_default() - }, - ) - .unwrap(); - assert!( - out.contains( - "def __instancecheck__(self, o): return isinstance(o, self._by_resolve())" - ), - "proxy must delegate __instancecheck__ in:\n{out}" - ); - assert!( - out.contains( - "def __subclasscheck__(self, o): return issubclass(o, self._by_resolve())" - ), - "proxy must delegate __subclasscheck__ in:\n{out}" - ); - } - #[test] fn polyfill_dotted_with_alias() { check_polyfill_body("import os.path as p\n", "p = _lazy_module(\"os.path\")\n"); @@ -836,7 +910,7 @@ mod tests { fn polyfill_from_import() { check_polyfill_body( "from os import path\n", - "path = _lazy_attr(\"os\", \"path\")\n", + "_lazy_from(_by_lazy, \"os\", {\"path\": \"path\"})\n", ); } @@ -844,10 +918,7 @@ mod tests { fn polyfill_from_import_multiple() { check_polyfill_body( "from os import path, getcwd\n", - indoc! {" - path = _lazy_attr(\"os\", \"path\") - getcwd = _lazy_attr(\"os\", \"getcwd\") - "}, + "_lazy_from(_by_lazy, \"os\", {\"path\": \"path\", \"getcwd\": \"getcwd\"})\n", ); } @@ -900,10 +971,10 @@ mod tests { } #[test] - fn polyfill_relative_submodule() { + fn polyfill_relative_package() { check_polyfill_body( "from . import x\n", - "x = _lazy_module(\".x\", __package__)\n", + "_lazy_from(_by_lazy, \".\", {\"x\": \"x\"}, __package__)\n", ); } @@ -911,7 +982,7 @@ mod tests { fn polyfill_relative_attr() { check_polyfill_body( "from .pkg import x\n", - "x = _lazy_attr(\".pkg\", \"x\", __package__)\n", + "_lazy_from(_by_lazy, \".pkg\", {\"x\": \"x\"}, __package__)\n", ); } @@ -919,7 +990,7 @@ mod tests { fn polyfill_relative_double_dot() { check_polyfill_body( "from .. import x\n", - "x = _lazy_module(\"..x\", __package__)\n", + "_lazy_from(_by_lazy, \"..\", {\"x\": \"x\"}, __package__)\n", ); } @@ -1019,8 +1090,7 @@ mod tests { #[test] fn polyfill_exception_class_stays_eager() { - // cpython's `except` refuses anything that is not a real class - // inheriting `BaseException`, so the proxy cannot stand for one + // an exception class is bound where it is imported let source = indoc! {" from json import JSONDecodeError @@ -1030,7 +1100,7 @@ mod tests { except JSONDecodeError: pass "}; - // nothing was deferred, so the proxy runtime is not emitted at all + // nothing was deferred, so the polyfill's runtime is not emitted at all assert_eq!(transpile_polyfill(source), source); } @@ -1039,10 +1109,9 @@ mod tests { // the statement is split: only the exception has to be bound for real let out = transpile_polyfill("from json import JSONDecodeError, dumps\n"); assert!( - out.ends_with(indoc! {" - from json import JSONDecodeError - dumps = _lazy_attr(\"json\", \"dumps\") - "}), + out.ends_with( + "from json import JSONDecodeError; _lazy_from(_by_lazy, \"json\", {\"dumps\": \"dumps\"})\n" + ), "got:\n{out}" ); } @@ -1059,17 +1128,17 @@ mod tests { #[test] fn polyfill_special_forms_stay_eager() { // a special form is recognised by what it *is*: a checker reading the output - // rejects `ClassVar[int]` once `ClassVar` names a variable, and `dataclasses` - // tells a bare `ClassVar` or `Final` annotation from a field by identity, which - // a proxy does not have. an ordinary function beside them is still deferred + // rejects `ClassVar[int]` once `ClassVar` is anything else, and `dataclasses` + // tells a bare `ClassVar` or `Final` annotation from a field by identity as the + // class is made. an ordinary function beside them is still deferred let out = transpile_polyfill(indoc! {" from typing import ClassVar, Final, Literal, Annotated as Note, cast "}); assert!( - out.ends_with(indoc! {" - from typing import ClassVar, Final, Literal, Annotated as Note - cast = _lazy_attr(\"typing\", \"cast\") - "}), + out.ends_with( + "from typing import ClassVar, Final, Literal, Annotated as Note; \ + _lazy_from(_by_lazy, \"typing\", {\"cast\": \"cast\"})\n" + ), "got:\n{out}" ); } @@ -1089,7 +1158,7 @@ mod tests { "the enum no longer lowers through ClassVar, got:\n{out}" ); assert!( - !out.contains("ClassVar = _lazy_attr"), + !out.contains("\"ClassVar\": "), "a special form was deferred, got:\n{out}" ); assert!( @@ -1141,17 +1210,17 @@ mod tests { "got:\n{out}" ); assert!( - !out.contains("overload = _lazy_attr"), + !out.contains("\"overload\": "), "the overload declaration was deferred, got:\n{out}" ); } /// a decorator says what the definition under it *is*, and it says so by identity. - /// deferred, `@dataclass` is a proxy a checker reading the emitted python cannot + /// deferred, `@dataclass` is a read a checker reading the emitted python cannot /// recognize: it read `Row` as an ordinary class and reported `Row("x")` as too many /// positional arguments, where the source declares a one-field dataclass. deferring one - /// buys nothing either — a decorator is applied while the module executes, so the proxy - /// standing in for it resolves at import time regardless + /// buys nothing either — a decorator is applied while the module executes, so a + /// deferred one resolves at import time regardless #[test] fn polyfill_a_decorator_stays_eager() { let out = transpile_polyfill(indoc! {" @@ -1161,9 +1230,10 @@ mod tests { def f() -> int: return reduce(lambda a, b: a + b, [1, 2], 0) "}); - assert!(out.contains("from functools import cache\n"), "got:\n{out}"); + assert!(out.contains("from functools import cache; "), "got:\n{out}"); assert!( - out.contains("reduce = _lazy_attr(\"functools\", \"reduce\")"), + out.contains("_lazy_from(_by_lazy, \"functools\", {\"reduce\": \"reduce\"})") + && out.contains("return _by_lazy.reduce("), "a name only called stopped being deferred, got:\n{out}" ); } @@ -1212,10 +1282,10 @@ mod tests { return dumps(value) "}, indoc! {" - dumps = _lazy_attr(\"json\", \"dumps\") + _lazy_from(_by_lazy, \"json\", {\"dumps\": \"dumps\"}) def show(value: int) -> str: - return dumps(value) + return _by_lazy.dumps(value) "}, ); } @@ -1230,7 +1300,7 @@ mod tests { value: Fraction = 1 "}); assert!( - out.contains("Fraction = _lazy_attr(\"fractions\", \"Fraction\")"), + out.contains("_lazy_from(_by_lazy, \"fractions\", {\"Fraction\": \"Fraction\"})"), "got:\n{out}" ); } @@ -1238,7 +1308,7 @@ mod tests { #[test] fn polyfill_an_annotation_under_future_annotations_stays_eager() { // every annotation is a string at runtime, and `typing.get_type_hints` evaluates - // it against the module's globals, where a proxy would be what it finds + // it against the module's globals, where a deferred name is not until it is read let out = transpile_polyfill(indoc! {" from __future__ import annotations from fractions import Fraction @@ -1252,6 +1322,229 @@ mod tests { ); } + fn assert_eager(source: &str, out: &str) { + assert!( + out.contains("from json import dumps") && !out.contains("_by_lazy"), + "the import was deferred:\n{source}\n---got---\n{out}" + ); + } + + /// a read reaches the module's binding from any scope that does not bind the name + /// itself, and each one is written through the module's `_LazyFrom` + #[test] + fn polyfill_every_read_of_a_deferred_name_is_written_through_the_holder() { + let out = transpile_polyfill(indoc! {r#" + from json import dumps + + def nested() -> object: + def inner() -> object: + return dumps + return inner() + + class Body: + value = dumps + + def default(f: object = dumps) -> object: + return f + + def reads_global() -> object: + global dumps + return dumps + + later = lambda: dumps + listed = [dumps for _ in range(1)] + text = f"{dumps!r:>4}" + "#}); + for read in [ + "return _by_lazy.dumps\n", + "value = _by_lazy.dumps\n", + // a default is evaluated per call in `.by`, so it is read in the function + "f = _by_lazy.dumps\n", + "lambda: _by_lazy.dumps\n", + "[_by_lazy.dumps for _ in range(1)]", + "f\"{_by_lazy.dumps!r:>4}\"", + ] { + assert!(out.contains(read), "missing `{read}` in:\n{out}"); + } + assert_eq!(out.matches("_by_lazy.dumps").count(), 7, "{out}"); + } + + /// anything else that binds the name, anywhere in the module, leaves a read that may find + /// either binding, so the import is left as it was written + #[test] + fn polyfill_a_name_bound_again_stays_eager() { + for rebinding in [ + "dumps = 5", + "dumps += 1", + "dumps: int = 1", + "def dumps() -> None:\n pass", + "class dumps:\n pass", + "for dumps in range(1):\n pass", + "with open(__file__) as dumps:\n pass", + "try:\n pass\nexcept Exception as dumps:\n pass", + "if (dumps := 1):\n pass", + "print([(dumps := i) for i in range(1)])", + "del dumps", + "from json import dumps", + "def f() -> None:\n global dumps\n dumps = 1", + "def f() -> None:\n global dumps\n del dumps", + // a binding that is not a name node: only the scope's own table shows it + "def f() -> None:\n global dumps\n def dumps() -> None:\n pass", + "def f() -> None:\n global dumps\n import json as dumps", + "class C:\n global dumps\n dumps = 1", + "match [1]:\n case [dumps]:\n pass", + ] { + let source = format!("from json import dumps\n{rebinding}\nprint(dumps)\n"); + assert_eager(&source, &transpile_polyfill(&source)); + } + } + + /// a class body reads a name it binds itself from its own namespace and then the + /// module's — never from a function around it — so a read the body has not bound yet + /// reads the import, even where an enclosing function binds the name + #[test] + fn polyfill_a_class_body_read_before_its_own_binding_reads_the_module() { + let out = transpile_polyfill(indoc! {" + from json import dumps + + def outer() -> object: + dumps = 1 + class Body: + seen = dumps + dumps = 2 + return Body.seen + "}); + assert!(out.contains("seen = _by_lazy.dumps"), "{out}"); + } + + /// a read the body has bound on some paths and not on others finds either + #[test] + fn polyfill_a_class_body_that_may_have_bound_the_name_keeps_it_eager() { + let source = indoc! {" + from json import dumps + + def flip() -> bool: + return True + + class Body: + if flip(): + dumps = 1 + seen = dumps + "}; + assert_eager(source, &transpile_polyfill(source)); + } + + /// `f"{x=}"` prints the expression as written, and a t-string's `Interpolation` keeps + /// it, so a read there cannot be written as anything else + #[test] + fn polyfill_a_name_whose_text_python_keeps_stays_eager() { + let source = "from json import dumps\nprint(f\"{dumps=}\")\n"; + assert_eager(source, &transpile_polyfill(source)); + let source = "from json import dumps\nprint(t\"{dumps}\")\n"; + let out = transpile( + source, + &Config { + lazy_imports: true, + min_version: PythonVersion::from((3, 14)), + ..Config::test_default() + }, + ) + .unwrap(); + assert_eager(source, &out); + } + + /// a name beginning with two underscores could be one the holder's own type answers + #[test] + fn polyfill_a_name_beginning_with_two_underscores_stays_eager() { + let out = transpile_polyfill("from json import __doc__ as __text\nprint(__text)\n"); + assert!( + out.contains("from json import __doc__ as __text\n") && !out.contains("_by_lazy"), + "{out}" + ); + } + + /// a star import can bind any name unseen, and a module's own `__getattr__` or + /// `__dir__` is where a deferred import would be served from + #[test] + fn polyfill_a_module_that_could_rebind_a_name_unseen_defers_nothing() { + for source in [ + "from os import *\nfrom json import dumps\nprint(dumps)\n", + "from json import dumps\ndef __getattr__(name: str) -> object:\n return dumps\n", + "from json import dumps\ndef __dir__() -> list[str]:\n return [str(dumps)]\n", + ] { + assert_eager(source, &transpile_polyfill(source)); + } + } + + /// a deferred name is served to other modules, and declared to the holder so that a read + /// before its import has run is the `NameError` python raises + #[test] + fn polyfill_the_holder_is_made_ahead_of_the_module_body() { + let out = transpile_polyfill("\"\"\"doc\"\"\"\nfrom json import dumps, loads as parse\n"); + let holder = out.find("_by_lazy = _LazyFrom(globals(), (\"dumps\", \"parse\"))\n"); + let statement = out + .find("_lazy_from(_by_lazy, \"json\", {\"dumps\": \"dumps\", \"parse\": \"loads\"})\n"); + assert!( + holder.zip(statement).is_some_and(|(h, s)| h < s), + "got:\n{out}" + ); + assert!( + out.contains("__getattr__ = _by_lazy.__by_getattr__\n"), + "{out}" + ); + assert!(out.contains("__dir__ = _by_lazy.__by_dir__\n"), "{out}"); + assert!(out.starts_with("\"\"\"doc\"\"\"\n"), "{out}"); + } + + /// the replacement keeps the statement's height, so every line after it keeps its number + #[test] + fn polyfill_a_multi_line_import_keeps_its_height() { + let out = + transpile_polyfill("from json import (\n dumps,\n loads,\n)\nprint(dumps)\n"); + assert!( + out.ends_with( + "_lazy_from(_by_lazy, \"json\", {\"dumps\": \"dumps\", \"loads\": \"loads\"})\n\n\n\n\ + print(_by_lazy.dumps)\n" + ), + "got:\n{out}" + ); + } + + /// the module binds `globals` to something of its own, so the holder reads the builtin + /// under a name of its own + #[test] + fn polyfill_the_holder_reads_the_builtin_globals() { + let out = transpile_polyfill( + "from json import dumps\n\ndef globals() -> int:\n return 1\n\nprint(dumps)\n", + ); + assert!( + out.contains("from builtins import globals as globals2\n") + && out.contains("_by_lazy = _LazyFrom(globals2(), (\"dumps\",))"), + "got:\n{out}" + ); + } + + /// a type parameter's bound that reads a name of the class it stands in is written ahead + /// of the class, as a string `TypeVar` keeps as a forward reference. python evaluates that + /// string against the module's namespace when anything evaluates it, so every name it + /// reads is bound where it is imported + #[test] + fn polyfill_a_name_a_written_forward_reference_reads_stays_eager() { + let out = transpile_polyfill(indoc! {" + from json import JSONDecoder + + class C: + class Inner: ... + def m[T: dict[JSONDecoder, Inner]](self, x: T) -> T: ... + "}); + assert!( + out.contains("bound=\"dict[JSONDecoder, C.Inner]\"") + && out.contains("from json import JSONDecoder\n") + && !out.contains("\"JSONDecoder\": "), + "{out}" + ); + } + #[test] fn annotation_names_reads_the_annotations_python_keeps() { use ruff_python_parser::parse_module; diff --git a/crates/by_transforms/tests/inferred_annotation_runtime.rs b/crates/by_transforms/tests/inferred_annotation_runtime.rs index b6cf2e695f..b0014ebbe5 100644 --- a/crates/by_transforms/tests/inferred_annotation_runtime.rs +++ b/crates/by_transforms/tests/inferred_annotation_runtime.rs @@ -57,8 +57,10 @@ fn run_at( ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); + // a deferred `Dec` is read through the module's `_LazyFrom` + let dec = if lazy_imports { "_by_lazy.Dec" } else { "Dec" }; assert!( - transpiled.contains(&format!("d: {annotation} = Dec(1)")), + transpiled.contains(&format!("d: {annotation} = {dec}(1)")), "the annotation names the module:\n{transpiled}" ); let output = Command::new(&interpreter.command) @@ -187,7 +189,13 @@ fn an_inferred_annotation_adds_no_import_cycle() { fs::write(dir.join(relative), transpiled).expect("write module"); } let m = fs::read_to_string(dir.join("m.py")).expect("read m"); - assert!(m.contains("x: \"x_mod.Thing\" = make()"), "{m}"); + // a deferred `make` is read through the module's `_LazyFrom` + let make = if lazy_imports { + "_by_lazy.make" + } else { + "make" + }; + assert!(m.contains(&format!("x: \"x_mod.Thing\" = {make}()")), "{m}"); let output = Command::new(&interpreter.command) .arg("main.py") .current_dir(&dir) diff --git a/crates/by_transforms/tests/lazy_import_runtime.rs b/crates/by_transforms/tests/lazy_import_runtime.rs index 3d817b983b..a8a22de7da 100644 --- a/crates/by_transforms/tests/lazy_import_runtime.rs +++ b/crates/by_transforms/tests/lazy_import_runtime.rs @@ -1,70 +1,500 @@ -//! Runtime tests for the lazy-import polyfill (`--min-version` < 3.15, which -//! has no PEP 810 `lazy` keyword and so binds `from x import y` to a -//! `_LazyAttr` proxy). +//! Runtime tests for the lazy-import polyfill (`--min-version` < 3.15, which has no PEP 810 +//! `lazy` keyword). A `from x import y` there binds nothing where it is written: every read of +//! `y` is written as a read through the module's `_LazyFrom`, which resolves `x.y` on first use +//! and stores it in the module's namespace //! -//! These have to be *runtime* tests spanning two modules: the proxy only -//! misbehaves once a real interpreter applies an operator to it across a real -//! import, so neither the transform unit tests (which assert lowered text) nor -//! the mdtests (which assert types) can see it. The proxy previously forwarded -//! only `__getattr__` / `__call__` / `__getitem__`, which meant every -//! *unforwarded* dunder silently fell back to `object`'s version — `a == b` -//! compared proxy identity and answered `False` for equal values, and `a + 1` -//! raised `TypeError`. Python looks special methods up on the type and never -//! routes them through `__getattr__`, so that class of bug is invisible until -//! executed. +//! These have to be *runtime* tests spanning several modules: what a read evaluates to, what +//! another module importing the name sees, and what a patch of the name reaches only show once +//! a real interpreter runs the emitted python across a real import, so neither the transform +//! unit tests (which assert lowered text) nor the mdtests (which assert types) can see them //! -//! [`import_stays_lazy_until_first_use`] guards the other side of the trade: -//! the proxy exists to defer the imported module's execution, so a "fix" that -//! resolved eagerly would pass every forwarding assertion while quietly -//! destroying the feature. +//! [`import_stays_lazy_until_first_use`] guards the other side of the trade: the rewrite exists +//! to defer the imported module's execution, so a "fix" that imported eagerly would pass every +//! other assertion while quietly destroying the feature use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use by_transforms::{Config, transpile}; +use by_transforms::{Config, transpile, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; mod interpreters; use interpreters::Interpreter; -/// module whose values the importer exercises. every binding is a plain value -/// (not a callable), which is exactly the case the proxy used to break +/// a soundness check in a function on a class the module imports further down is made once +/// the import has run. the import is deferred, so the name is not in the module's namespace +/// then, and the check asks the module's `_LazyFrom` whether its statement has run +const ONCE_BOUND: &[(&str, &str)] = &[ + ( + "/by_once_mod.by", + r#" +class Thing: + pass + +def things() -> list[Thing]: + wrong: dynamic = [1] + return wrong +"#, + ), + ( + "/main.by", + r#" +def first[T](xs: list[T]) -> T: + return xs[0] + +def use() -> object: + return first(things()) + +from by_once_mod import Thing, things + +try: + use() +except TypeError as error: + assert "type soundness violation" in str(error), str(error) +else: + raise AssertionError("the check is made once the import has run") +print("ok") +"#, + ), +]; + +/// `from . import x` names `x` in the package itself: a function its `__init__` defines, +/// or a submodule nothing has imported yet. python reads the attribute first and imports +/// the submodule only when there is none +const RELATIVE_PACKAGE: &str = r#" +def helper() -> str: + return "helper" +"#; + +const RELATIVE_SUBMODULE: &str = r#" +NAME = "other" +"#; + +const RELATIVE_USER: &str = r#" +from . import helper +from . import other + +def run() -> str: + return helper() + " " + other.NAME +"#; + +const RELATIVE_MAIN: &str = r#" +from by_relative_pkg.user import run + +assert run() == "helper other", run() +print("ok") +"#; + +/// a package whose `__init__` imports its own submodule, relatively and by its absolute name +/// +/// the submodule is read off the package, which is the module doing the reading, and has +/// to be imported as python's fromlist imports it +const INIT_SUBMODULE: &[(&str, &str)] = &[ + ( + "by_init_rel/__init__", + "from . import helper\n\ndef use() -> str:\n return helper.NAME\n", + ), + ("by_init_rel/helper", "NAME = \"relative\"\n"), + ( + "by_init_abs/__init__", + "from by_init_abs import helper\n\ndef use() -> str:\n return helper.NAME\n", + ), + ("by_init_abs/helper", "NAME = \"absolute\"\n"), + ( + "main", + r#" +import by_init_rel +import by_init_abs +from by_init_rel import helper as outside + +print(outside.NAME) +print(by_init_rel.use(), by_init_abs.use()) +"#, + ), +]; + +/// two modules re-exporting a name each takes from the other, which neither binds +const REEXPORT_CYCLE: &[(&str, &str)] = &[ + ("by_rx_a", "from by_rx_b import x\n"), + ("by_rx_b", "from by_rx_a import x\n"), + ( + "main", + r#" +try: + import by_rx_a + by_rx_a.x +except ImportError as error: + print(type(error).__name__) +"#, + ), +]; + +/// a name an import binds only on some paths reads the builtin of that name where the +/// import did not run, as python's global lookup does, and is a `NameError` where there is +/// no builtin +const BUILTIN_FALLBACK: &[(&str, &str)] = &[( + "main", + r#" +import sys + +if sys.version_info < (3, 0): + from itertools import izip as zip + +if "--pretty" in sys.argv: + from pprint import pprint as print + +if len(sys.argv) > 5: + from json import dumps + +print(list(zip([1, 2], [3, 4]))) +try: + dumps +except NameError as error: + print(str(error)) +"#, +)]; + +/// a reload runs a module again over the namespace it had, so its imports have to bind what +/// their modules hold now, not what the first run read +const RELOAD: &[(&str, &str)] = &[ + ("by_reload_src", "VALUE = 1\n"), + ("by_reload_user", "from by_reload_src import VALUE\n"), + ( + "main", + r#" +import importlib +import sys + +sys.dont_write_bytecode = True +import by_reload_src +import by_reload_user + +first = by_reload_user.VALUE +with open(by_reload_src.__file__, "w") as source: + source.write("VALUE = 2\n") +importlib.invalidate_caches() +importlib.reload(by_reload_src) +importlib.reload(by_reload_user) +print(first, by_reload_user.VALUE) +"#, + ), +]; + +/// a failing `from` import reports the error python reports: a submodule's own missing +/// dependency, an `AttributeError` the module's body raises, the name that is not there, and +/// the name a module still being imported has not bound yet +/// each case is a module of its own that imports and reads the name at its top level, where +/// python raises at the import and the polyfill at the read — both while the case module is +/// being imported, inside the `try` that catches it +const IMPORT_ERRORS: &[(&str, &str)] = &[ + ("by_err_pkg/__init__", ""), + ( + "by_err_pkg/broken", + "import by_not_installed_dependency\n\nX = 1\n", + ), + ( + "by_err_body", + "raise AttributeError(\"raised by the body\")\n", + ), + ( + "by_err_case_dependency", + "from by_err_pkg import broken\n\nbroken.X\n", + ), + ( + "by_err_case_body", + "from by_err_body import anything\n\nanything\n", + ), + ( + "by_err_case_name", + "from json import by_no_such_name\n\nby_no_such_name\n", + ), + ( + "by_err_case_submodule", + "from urllib import by_no_such_submodule\n\nby_no_such_submodule\n", + ), + // a module whose name is read while it is still being imported + ( + "by_err_case_circular", + "from by_err_circular_other import f\n\nf()\nx = 1\n", + ), + ( + "by_err_circular_other", + "from by_err_case_circular import x\n\ndef f() -> None:\n pass\n\nx\n", + ), + ( + "main", + r#" +import importlib + +for case in ("dependency", "body", "name", "submodule", "circular"): + try: + importlib.import_module("by_err_case_" + case) + except ImportError as error: + print(case, type(error).__name__, str(error), error.name, error.path) + print(getattr(error, "name_from", None)) + except AttributeError as error: + # the attribute read the polyfill resolves the name in gives the error the name it + # was read under, which an import raising it does not + print(case, type(error).__name__, str(error)) +"#, + ), +]; + +/// module whose values the importer exercises const VALUES_MODULE: &str = r#" num: int = 5 text: str = "hi" items: list[int] = [1, 2, 3] +ratio: float = 200.0 +nothing: int? = None class Point: def __init__(self, x: int): self.x = x + +ORIGIN = Point(0) + +def origin() -> Point: + return ORIGIN "#; -/// every operator here reaches the proxy as a dunder looked up on its *type*, -/// so each assertion fails unless the proxy forwards that specific method +/// an imported name is the object it names, so everything that asks *which* object it is — +/// identity, `type()`, `json`, a library deciding how to treat a value by what its type +/// defines — gets the answer the eager import gives const VALUES_MAIN: &str = r#" -from by_values_mod import num, text, items, Point - -assert num == 5, "eq" -assert not (num != 5), "ne" -assert num < 9 and num > 1 and num <= 5 and num >= 5, "ordering" -assert num + 1 == 6 and 1 + num == 6, "add / radd" -assert num * 2 == 10 and 10 - num == 5, "mul / rsub" -assert num % 3 == 2 and 2 ** num == 32, "mod / rpow" -assert -num == -5 and abs(-num) == 5, "unary" -assert text + "!" == "hi!" and "oh " + text == "oh hi", "concat / rconcat" -assert len(items) == 3, "len" -assert [i * 2 for i in items] == [2, 4, 6], "iter" -assert 2 in items, "contains" -assert items[0] == 1 and text[1] == "i", "getitem" -assert bool(num) and hash(num) == hash(5), "bool / hash" -assert str(text) == "hi" and repr(text) == "'hi'", "str / repr" -assert f"{num:03d}" == "005", "format" -assert int(num) == 5 and float(num) == 5.0, "int / float" -assert sorted([3, num, 1]) == [1, 3, 5], "sorting uses forwarded comparisons" -assert list(reversed(items)) == [3, 2, 1], "reversed" -assert isinstance(num, int), "isinstance via __class__" -assert isinstance(Point(1), Point), "isinstance against a lazily-imported class" -assert Point(7).x == 7, "call + attribute" +import json +import by_values_mod as values +from by_values_mod import num, text, items, ratio, nothing, Point, ORIGIN, origin + +def approx_like(expected: object) -> str: + # `pytest.approx` reads anything whose type has `__iter__` as a collection + return "collection" if hasattr(type(expected), "__iter__") else "scalar" + +assert origin() === ORIGIN, "identity across the module boundary" +assert ORIGIN === values.ORIGIN, "the name is the module's object" +assert type(num) === int and type(text) === str and type(ORIGIN) === Point, "type()" +assert json.dumps({"n": num, "t": text, "i": items}) == '{"n": 5, "t": "hi", "i": [1, 2, 3]}', "json" +assert approx_like(ratio) == "scalar" and approx_like(items) == "collection", "approx" +assert nothing === None, "identity with None" +assert num + 1 == 6 and 1 + num == 6 and text + "!" == "hi!" and len(items) == 3, "operators" +assert isinstance(Point(1), Point) and Point(7).x == 7, "a class constructs and tests" +print("ok") +"#; + +/// a module whose one value is told apart by identity +const TOKEN_MODULE: &str = r#" +class Token: + pass + +TOKEN = Token() +"#; + +/// every scope a read can reach the module's binding from +const SCOPES_MAIN: &str = r#" +import by_token_mod as m +from by_token_mod import TOKEN + +def nested() -> object: + def inner() -> object: + return TOKEN + return inner() + +class Body: + seen = TOKEN + +def default(value: object = TOKEN) -> object: + return value + +def reads_global() -> object: + global TOKEN + return TOKEN + +def outer() -> object: + TOKEN = "local" + class Inner: + seen = TOKEN + TOKEN = "class" + return Inner.seen + +later = lambda: TOKEN +listed = [TOKEN for _ in range(1)] +text = f"{TOKEN!r}" + +assert nested() === m.TOKEN, "a nested function" +assert Body.seen === m.TOKEN, "a class body" +assert default() === m.TOKEN, "a default value" +assert reads_global() === m.TOKEN, "a `global` read" +assert outer() === m.TOKEN, "a class body reads the module, not the function around it" +assert later() === m.TOKEN, "a lambda" +assert listed[0] === m.TOKEN, "a comprehension" +assert text == repr(m.TOKEN), "an f-string" +print("ok") +"#; + +/// a name bound again keeps its import as written, and every read finds what python's would +const REBOUND_MAIN: &str = r#" +import by_token_mod as m +from by_token_mod import TOKEN + +first = TOKEN + +def read() -> object: + return TOKEN + +def rebind() -> None: + global TOKEN + TOKEN = "rebound" + +assert first === m.TOKEN and read() === m.TOKEN, "before the rebinding, the import" +rebind() +assert read() == "rebound", "after it, the new binding" +print("ok") +"#; + +/// a read made before the import statement has run fails as python's does +const EARLY_MAIN: &str = r#" +def early() -> object: + return TOKEN + +try: + early() +except NameError as error: + assert str(error) == "name 'TOKEN' is not defined", str(error) +else: + raise AssertionError("a read before the import ran must raise") + +from by_token_mod import TOKEN +import by_token_mod as m + +assert early() === m.TOKEN, "and once it has run, reads the import" +print("ok") +"#; + +const REEXPORT_SOURCE: &str = r#" +class Marker: + pass + +VALUE = Marker() +"#; + +/// imports `VALUE` and never reads it, so only another module can +const REEXPORT_MIDDLE: &str = r#" +from by_reexport_src import VALUE +"#; + +const REEXPORT_LISTED: &str = r#" +from by_reexport_src import VALUE + +__all__ = ["VALUE"] +"#; + +const REEXPORT_STAR: &str = r#" +from by_reexport_listed import * + +def got() -> object: + return VALUE +"#; + +/// a module that imports a name passes it on, as python's would, through every way another +/// module asks for it. until something reads it, the name is not in the module's namespace +const REEXPORT_MAIN: &str = r#" +import by_reexport_src as src +import by_reexport_middle as middle +import by_reexport_star as star + +assert "VALUE" not in vars(middle), "an unread name is not in the namespace yet" +assert "VALUE" in dir(middle), "dir() lists it" +assert hasattr(middle, "VALUE") and middle.VALUE === src.VALUE, "an attribute read" +assert vars(middle)["VALUE"] === src.VALUE, "which leaves the object in the namespace" +assert not hasattr(middle, "NOT_THERE"), "a name the module never bound is still missing" +assert star.got() === src.VALUE, "a star import of a module with `__all__`" + +from by_reexport_middle import VALUE +assert VALUE === src.VALUE, "a `from` import of it" +print("ok") +"#; + +/// two modules importing each other's names: neither runs the other until a name is read +const CYCLE_A: &str = r#" +from by_cycle_b import pong + +def ping(n: int) -> int: + return 0 if n == 0 else pong(n - 1) +"#; + +const CYCLE_B: &str = r#" +from by_cycle_a import ping + +def pong(n: int) -> int: + return 1 if n == 0 else ping(n - 1) + +PARTNER = ping +"#; + +/// the cycle the other way round: one side's name is read in an annotation, which keeps that +/// import eager, and the other side's import of it is deferred +const CYCLE_C: &str = r#" +from by_cycle_d import Maker + +class Thing: + def maker(self) -> Maker: + return Maker() +"#; + +const CYCLE_D: &str = r#" +from by_cycle_c import Thing + +class Maker: + def thing(self) -> object: + return Thing() +"#; + +const CYCLE_MAIN: &str = r#" +import by_cycle_a as a +import by_cycle_b as b +from by_cycle_a import ping +from by_cycle_c import Thing + +assert ping(3) == 1, "a call crosses the cycle" +assert b.PARTNER === a.ping, "and the name is the other module's object" +assert isinstance(Thing().maker().thing(), Thing), "an eager side and a deferred side" +print("ok") +"#; + +const PATCH_DEPENDENCY: &str = r#" +def fetch() -> str: + return "real" +"#; + +const PATCH_SERVICE: &str = r#" +from by_patch_dep import fetch + +def run() -> str: + return fetch() +"#; + +/// a test patches a name where it is used, in the module that imported it. the patch writes +/// that module's namespace, which is where each of its reads looks first +const PATCH_MAIN: &str = r#" +from unittest import mock +import by_patch_svc as svc + +with mock.patch("by_patch_svc.fetch", lambda: "patched"): + assert svc.run() == "patched", "a patch made before the name was read" +assert svc.run() == "real", "and undone" + +with mock.patch.object(svc, "fetch", lambda: "object"): + assert svc.run() == "object", "a patch made after" +assert svc.run() == "real", "and undone" + +original = svc.fetch +setattr(svc, "fetch", lambda: "set") +assert svc.run() == "set", "what `monkeypatch.setattr` does" +setattr(svc, "fetch", original) +assert svc.run() == "real", "and undoes" print("ok") "#; @@ -105,12 +535,8 @@ made_there: Character = "a" assert made_here == made_there, "str equality holds regardless of class identity" assert isinstance(made_there, Character), "same-module Character" assert isinstance(made_here, Character), "Character identity must survive an import" -# `===` is identity here — in basedpython `is` means isinstance. and identity is -# read off `__class__` rather than `type()`: `made_here` is a lazily-imported -# proxy whose `type()` is `_LazyAttr` by design (a documented limit of the -# < 3.15 polyfill), while `__class__` is both forwarded by the proxy and what -# `isinstance` itself consults -assert made_here.__class__ === made_there.__class__, "one Character class per process" +# `===` is identity here — in basedpython `is` means isinstance +assert type(made_here) === type(made_there), "one Character class per process" assert made_here.__class__.__name__ == "Character", "still named Character" print("ok") "#; @@ -123,7 +549,7 @@ const SUBMODULE_MAIN: &str = r#" from urllib import parse from urllib import by_not_there as missing -assert parse.quote("a b") == "a%20b", "a submodule resolves through the proxy" +assert parse.quote("a b") == "a%20b", "a submodule resolves on first use" assert parse.urlparse("http://h/p").path == "/p", "and keeps working after the first use" try: @@ -200,11 +626,10 @@ print("ok") "#; /// the readers of an annotation, each asserting the answer python gives for the names it -/// finds. a proxy is not the object it stands for, and no forwarded operator makes it one: -/// `dataclasses` tells a `KW_ONLY` annotation from a field by identity, and a type hint is -/// compared by identity by whatever consumes it. so a name an annotation reads is imported -/// eagerly. `InitVar[int]` and `singledispatch`'s registration by annotation are readers -/// the proxy happened to satisfy, and are kept here as readers all the same +/// finds. an annotation is read back from the module's namespace — `typing.get_type_hints` +/// evaluates a string annotation against it — where a deferred name is not until something +/// reads it, and `dataclasses` tells a `KW_ONLY` annotation from a field by identity as the +/// class is made. so a name an annotation reads is imported eagerly const ANNOTATIONS_MAIN: &str = r#" from dataclasses import dataclass, fields, KW_ONLY, InitVar from fractions import Fraction @@ -254,9 +679,9 @@ assert typing.get_type_hints(half)["value"] === fractions.Fraction, "the string print("ok") "#; -/// a module that binds names of builtins the proxy reads when it resolves, formats and -/// iterates. the proxy is pasted into the module and reads the builtins through its -/// globals, so it reads them under names of its own +/// a module that binds names of builtins the polyfill reads when it resolves a name, and +/// the one the module's `_LazyFrom` is made with. the runtime is pasted into the module and +/// reads the builtins through its globals, so it reads them under names of its own const BUILTIN_NAMES_MAIN: &str = r#" from decimal import Decimal from math import pi @@ -271,10 +696,13 @@ def format(value: object, spec: str) -> str: def iter(o: object) -> int: return 0 -assert Decimal(1) + 1 == Decimal(2), "resolving the proxy calls `getattr`" -assert f"{pi:.2f}" == "3.14", "formatting the proxy calls `format`" -assert [c for c in digits][:2] == ["0", "1"], "iterating the proxy calls `iter`" -assert getattr(1, "x") == 42 and format(1, "") == "shadowed" and iter(1) == 0 +def globals() -> int: + return 7 + +assert Decimal(1) + 1 == Decimal(2), "resolving a name calls `getattr`" +assert f"{pi:.2f}" == "3.14", "formatting the value calls `format`" +assert [c for c in digits][:2] == ["0", "1"], "iterating the value calls `iter`" +assert getattr(1, "x") == 42 and format(1, "") == "shadowed" and iter(1) == 0 and globals() == 7 print("ok") "#; @@ -294,11 +722,55 @@ fn build_case(case: &str, sources: &[(&str, &str)]) -> PathBuf { for (stem, source) in sources { let transpiled = transpile(source, &Config::default()).expect("transpile should succeed"); - fs::write(dir.join(format!("{stem}.py")), transpiled).expect("write module"); + let path = dir.join(format!("{stem}.py")); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("create package dir"); + } + fs::write(path, transpiled).expect("write module"); } dir } +/// run `sources` both transpiled and as the python they are written in, asserting the two +/// print the same, and something. the sources are plain python, so python running them is +/// the reference +fn agrees_with_python(python: &Interpreter, case: &str, sources: &[(&str, &str)]) { + let transpiled = build_case(case, sources); + let reference = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{case}_python")); + let _ = fs::remove_dir_all(&reference); + for (stem, source) in sources { + let path = reference.join(format!("{stem}.py")); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("create package dir"); + } + fs::write(path, source).expect("write module"); + } + let run = |dir: &Path| { + let output = Command::new(&python.command) + .arg("main.py") + .current_dir(dir) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "{} failed:\n--- stdout ---\n{}\n--- stderr ---\n{}", + dir.display(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + // the two trees are in directories of their own, which a message naming a file says + String::from_utf8_lossy(&output.stdout).replace(&dir.display().to_string(), "") + }; + let expected = run(&reference); + assert!(!expected.trim().is_empty(), "the reference printed nothing"); + assert_eq!( + run(&transpiled), + expected, + "the transpiled program disagrees with python" + ); + interpreters::ran(python, Config::default().min_version); +} + /// run `main.py` in `dir`, asserting it exits cleanly and prints `ok` fn run_main(python: &Interpreter, dir: &Path) { let output = Command::new(&python.command) @@ -318,7 +790,7 @@ fn run_main(python: &Interpreter, dir: &Path) { } #[test] -fn proxy_forwards_value_operations() { +fn an_imported_value_is_the_object_itself() { let Some(python) = python() else { return; }; @@ -375,7 +847,7 @@ fn an_unlazifiable_name_in_a_multi_name_import_stays_bound() { } #[test] -fn a_submodule_resolves_through_the_proxy() { +fn a_submodule_resolves_on_first_use() { let Some(python) = python() else { return; }; @@ -405,10 +877,190 @@ fn a_string_annotation_resolves_to_the_imported_object() { } #[test] -fn the_proxy_reads_builtins_the_module_binds_for_itself() { +fn the_polyfill_reads_builtins_the_module_binds_for_itself() { let Some(python) = python() else { return; }; let dir = build_case("lazy_builtin_names", &[("main", BUILTIN_NAMES_MAIN)]); run_main(&python, &dir); } + +#[test] +fn a_read_in_every_scope_is_the_object_itself() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_scopes", + &[("by_token_mod", TOKEN_MODULE), ("main", SCOPES_MAIN)], + ); + run_main(&python, &dir); +} + +#[test] +fn a_name_bound_again_reads_as_python_does() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_rebound", + &[("by_token_mod", TOKEN_MODULE), ("main", REBOUND_MAIN)], + ); + run_main(&python, &dir); +} + +#[test] +fn a_read_before_the_import_runs_raises_python_s_name_error() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_early", + &[("by_token_mod", TOKEN_MODULE), ("main", EARLY_MAIN)], + ); + run_main(&python, &dir); +} + +#[test] +fn a_deferred_name_is_passed_on_to_other_modules() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_reexport", + &[ + ("by_reexport_src", REEXPORT_SOURCE), + ("by_reexport_middle", REEXPORT_MIDDLE), + ("by_reexport_listed", REEXPORT_LISTED), + ("by_reexport_star", REEXPORT_STAR), + ("main", REEXPORT_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn modules_importing_each_other_resolve_on_use() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_cycle", + &[ + ("by_cycle_a", CYCLE_A), + ("by_cycle_b", CYCLE_B), + ("by_cycle_c", CYCLE_C), + ("by_cycle_d", CYCLE_D), + ("main", CYCLE_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_patch_of_the_importing_module_reaches_its_reads() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_patch", + &[ + ("by_patch_dep", PATCH_DEPENDENCY), + ("by_patch_svc", PATCH_SERVICE), + ("main", PATCH_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_check_on_a_deferred_class_is_made_once_its_import_has_run() { + let Some(python) = python() else { + return; + }; + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in ONCE_BOUND { + db.write_file(path, source).expect("write file failed"); + } + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("lazy_once_bound"); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + for (path, _) in ONCE_BOUND { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = transpile_typed(&db, file, &Config::default(), None) + .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); + if path.ends_with("main.by") { + // the check this is about, on a name the polyfill defers + assert!( + transpiled.contains("_soundness_once_bound((\"Thing\",)") + && transpiled.contains("lambda: _by_lazy.Thing"), + "{transpiled}" + ); + } + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + run_main(&python, &dir); +} + +#[test] +fn a_relative_import_of_the_package_reads_its_attribute_or_its_submodule() { + let Some(python) = python() else { + return; + }; + let dir = build_case( + "lazy_relative", + &[ + ("by_relative_pkg/__init__", RELATIVE_PACKAGE), + ("by_relative_pkg/other", RELATIVE_SUBMODULE), + ("by_relative_pkg/user", RELATIVE_USER), + ("main", RELATIVE_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_package_importing_its_own_submodule_agrees_with_python() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_init_submodule", INIT_SUBMODULE); +} + +#[test] +fn a_re_export_cycle_is_python_s_import_error() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_reexport_cycle", REEXPORT_CYCLE); +} + +#[test] +fn a_name_whose_import_did_not_run_reads_the_builtin() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_builtin_fallback", BUILTIN_FALLBACK); +} + +#[test] +fn a_reloaded_module_binds_what_its_modules_hold_now() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_reload", RELOAD); +} + +#[test] +fn a_failing_import_reports_python_s_error() { + let Some(python) = python() else { + return; + }; + agrees_with_python(&python, "lazy_import_errors", IMPORT_ERRORS); +} diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 833d244324..84b9fb0876 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -4887,9 +4887,10 @@ def main(): reason = "a skipped test prints why it was skipped" )] fn a_lazy_from_import_resolves_a_submodule_and_refuses_a_missing_name_as_python_does() { - // `_LazyAttr` defers the attribute read, and reading an attribute is not how a - // submodule gets bound: `urllib/__init__.py` never imports `parse`, and cpython - // binds it only because `__import__` is handed a fromlist. so a transpiled + // a deferred `from` import reads an attribute off the module on first use, and + // reading an attribute is not how a submodule gets bound: `urllib/__init__.py` + // never imports `parse`, and cpython binds it only because `__import__` is + // handed a fromlist. so a transpiled // `from urllib import parse` used to raise `AttributeError` where the same source // run by cpython is fine — a wrong answer in shipped output rather than a decline // @@ -4940,7 +4941,7 @@ fn a_lazy_from_import_resolves_a_submodule_and_refuses_a_missing_name_as_python_ let program = String::from_utf8_lossy(&transpiled.stdout).into_owned(); // the laziness is the thing under test, so its absence must not pass silently assert!( - program.contains("_lazy_attr(\"urllib\", \"parse\")"), + program.contains("_lazy_from(_by_lazy, \"urllib\", {\"parse\": \"parse\"})"), "{program}" ); fs::write(dir.path().join("prog.py"), &program).unwrap(); @@ -6672,9 +6673,9 @@ fn a_built_package_runs_off_one_copy_of_the_runtime() { !one.contains("def _checked_cast("), "and does not define it as well:\n{one}" ); - // a helper reached through the lazy-import proxy is a proxy call on every use + // a deferred helper would be an attribute read on every use assert!( - !one.contains("_lazy_attr(\"app._by_runtime\""), + !one.contains("\"app._by_runtime\""), "the runtime import is eager:\n{one}" ); // the lowering asks for helpers in two places — `cast!` during the AST rewrite, the diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 4e2d45bd19..62bd74d9ab 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -337,21 +337,18 @@ impl<'db> SemanticModel<'db> { crate::types::conformance::eagerly_imported_modules(self.db, self.file.file(db)) } - /// basedpython: the names this file imports that must be bound to the real - /// object rather than to a lazy proxy — the exception classes and the special - /// forms. + /// basedpython: the names this file imports that the lazy-import polyfill binds where the + /// import is written rather than deferring — the exception classes and the special forms /// - /// `except` is one place cpython refuses a stand-in: it checks that what it - /// catches is a class inheriting `BaseException`, and never consults - /// `__instancecheck__`. A proxy there raises `TypeError` from the handler — - /// the line least likely to be covered by a happy-path test — so an - /// exception class is imported eagerly and the proxy never stands where it - /// cannot work. + /// A deferred name is resolved where it is first read, and for an exception class that is + /// a handler: an import that fails there would surface while another exception is being + /// handled, and replace it. So an exception class is imported where the import is written, + /// where a failure surfaces as it does without the polyfill /// /// A special form is the other. What reads one tells it apart by identity: /// `dataclasses` decides that a bare `ClassVar` annotation is not a field by /// asking whether it *is* `typing.ClassVar`, and a checker reading the emitted - /// python rejects `ClassVar[int]` once `ClassVar` names a variable. + /// python rejects `ClassVar[int]` once `ClassVar` is anything else pub fn eagerly_imported_names(&self) -> Vec { let db = self.db; let env = self.program_environment(); diff --git a/docs/basedpython/development/compilation/technology.md b/docs/basedpython/development/compilation/technology.md index 9333b8663c..2cc55c134e 100644 --- a/docs/basedpython/development/compilation/technology.md +++ b/docs/basedpython/development/compilation/technology.md @@ -338,7 +338,9 @@ needs whatever that python needs at import time. that turns out to be nothing extra, but only because the transpiler is configured the same way `by build` configures it: the [lazy-import pass](../../features/lazy-imports.md) binds `JustFloat = float` locally rather than emitting a `from ty_extensions import …` -that has no module behind it +that has no module behind it. the one setting that differs is the deferral of `from` +imports, which is off: below 3.15 a deferred `from` import binds nothing where it +is written, and the compiled functions read the module's globals by name this is a sharp edge worth naming. transpiling for the fallback with a *different* config than the interpreted build uses produces an extension that diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index b23af75c61..c41a42dae0 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -117,8 +117,15 @@ body, so a helper brings the rest of what it needs along all three paste the definitions in both renderings are slices of the same text. the lazy-import pass leaves the -runtime import eager, since a helper reached through its proxy would be a proxy -call on every use +runtime import eager: the polyfill runs on it, and a deferred helper would be an +attribute read on every use + +below 3.15 the lazy-import pass asks ty which `from` imports it can defer +(`SemanticModel::deferrable_from_imports`, run over the output itself, so it sees +every name an earlier lowering wrote). an import qualifies when it is the only +binding its name has anywhere in the module, and the answer carries every read +that reaches it; the pass writes each of those reads through the module's +`_LazyFrom`, whose `__dict__` is the module's namespace `_by_runtime.py` is excluded from this repository's own ruff configuration. reformatting it rewrites transpiled output, and the isort rule would put a diff --git a/docs/basedpython/features/differences-from-python.md b/docs/basedpython/features/differences-from-python.md index 8cc2af8d6f..b9753517b5 100644 --- a/docs/basedpython/features/differences-from-python.md +++ b/docs/basedpython/features/differences-from-python.md @@ -73,7 +73,9 @@ print(os) # this is what loads it an import with a side effect — registering a plugin, patching something at module scope — no longer happens just because the importing module was loaded. `from __future__ import ...`, `from x import *`, and an unaliased `import a.b` -stay eager. see [lazy imports](lazy-imports.md) +stay eager. below python 3.15 a name imported with `from` is read off its module +on first use, and is in the importing module's namespace only from then on. see +[lazy imports](lazy-imports.md) ### an annotation read at runtime is its python spelling diff --git a/docs/basedpython/features/lazy-imports.md b/docs/basedpython/features/lazy-imports.md index facea98208..3d670486e9 100644 --- a/docs/basedpython/features/lazy-imports.md +++ b/docs/basedpython/features/lazy-imports.md @@ -57,59 +57,145 @@ set the target with `--min-version 3.15` on `by transpile`/`by build`/`by run` ## the polyfill (python < 3.15) without the `lazy` keyword there is no language-level way to defer a binding, -so the polyfill emits two different shapes: - -- `import os` → `os = _lazy_module("os")`, which wraps the loader in - `importlib.util.LazyLoader` and hands back a **real module object**. nothing - is proxied, so this form behaves exactly like an eager import apart from when - the body runs -- `from os import path` → `path = _lazy_attr("os", "path")`, a `_LazyAttr` - **proxy**. a proxy is unavoidable here: `_lazy_module` already defers the - module body, but reading `path` off the module would force it to run - immediately, which is the very thing being deferred - -the proxy is transparent — it forwards the operators (`==`, `+`, `len`, `in`, -`str`, `hash`, comparisons, …) to the value behind it, so it behaves like the -imported object. this matters more than it sounds: python looks special methods -up on the *type* and never routes them through `__getattr__`, so any dunder the -proxy fails to forward silently falls back to `object`'s version rather than -raising — an unforwarded `__eq__` would make `a == b` compare proxy identity -and quietly answer `False` for equal values - -`isinstance` works in both directions: `isinstance(x, C)` for a proxied `x` -(via `__class__`), and `isinstance(x, C)` where `C` itself was lazily imported -(via `__instancecheck__`) - -an **exception class is never proxied**. `except` is the one place cpython -refuses a stand-in: it checks that what it catches is a real class inheriting -`BaseException`, and never consults `__instancecheck__`. so -`from errors import ParseError` stays an ordinary import, and the rest of the -statement's names still don't: +so the polyfill lowers the two forms differently -```py -from errors import ParseError -decode = _lazy_attr("errors", "decode") +`import os` binds a real module object whose body runs on first attribute access: + +```python +os = _lazy_module("os") +``` + +nothing stands in for the module, so this form behaves exactly like an eager +import apart from when the body runs + +`from json import dumps` binds nothing where it is written. every read of `dumps` +is written as a read through one object the module keeps for its deferred names, +which resolves the name on first use: + +```by +from json import dumps + +def show(value: int) -> str: + return dumps(value) ``` -nor is a **special form** — `ClassVar`, `Final`, `Literal`, `Annotated`, `Protocol` and -the rest of what a type checker gives a meaning of its own. what reads one tells it -apart by identity: `dataclasses` decides a bare `ClassVar` annotation is not a field -by asking whether it *is* `typing.ClassVar`, and a checker reading the emitted python -rejects `ClassVar[int]` once `ClassVar` names a variable. that holds for the imports the -transpiler writes itself, such as the `ClassVar` an `enum class` lowers through: +transpiles to: + +```python +_by_lazy = _LazyFrom(globals(), ("dumps",)) +__getattr__ = _by_lazy.__by_getattr__ +__dir__ = _by_lazy.__by_dir__ +_lazy_from(_by_lazy, "json", {"dumps": "dumps"}) + +def show(value: int) -> str: + return _by_lazy.dumps(value) +``` + +(the runtime the polyfill calls is left out here) + +the first read of `dumps` imports `json`, reads `json.dumps` and stores it in the +module's namespace under `dumps`, and every read after that finds it there. so +the name evaluates to the object itself — `type(dumps)`, `dumps === json.dumps`, +handing it to `json.dumps` or `pytest.approx`, all answer as they would for an +ordinary import + +### what other modules see + +the module serves the name to other modules as an ordinary import would: +`from app import dumps`, `app.dumps`, `hasattr(app, "dumps")` and `dir(app)` all +find it, through the module's `__getattr__` and `__dir__` + +a patch of the name where it is used — `mock.patch("app.dumps")`, +`monkeypatch.setattr(app, "dumps", fake)` — writes the module's namespace, which +is where every read in the module looks first, so the module's own code sees the +patch, and sees the original again once it is undone + +### the name is resolved once + +a name is read off its module at its first use and kept, as PEP 810 does it. an +ordinary `from` import reads it when the import runs instead, so a module that +rebinds `json.dumps` between the import and the first use is seen by the polyfill +and not by an ordinary import. a rebinding after the first use is seen by neither + +### until the first use, the name is not in the namespace + +nothing is bound where the import is written, so until something reads the +name, it is not in the module's namespace: + +| expression | before the first use | +| ------------------------------------------ | ----------------------- | +| `globals()["dumps"]`, `vars(app)["dumps"]` | `KeyError` | +| `from app import *`, with no `__all__` | does not import `dumps` | +| `eval("dumps")` in the module | `NameError` | +| `typing.ForwardRef("dumps")`, evaluated | `NameError` | + +anything that walks a module's `__dict__` rather than asking for a name misses it +too: pytest does not collect a test class a test module imports and never uses + +reading the name once puts it there, and a module with an `__all__` that lists it +passes it on to `from app import *` whether it was read or not + +a read made before the import statement has run — or where it never ran, as in +`if PY2: from itertools import izip as zip` — finds what python's lookup finds: the +builtin of that name if there is one, and a `NameError` if not + +a reload runs the module again, and each of its `from` imports reads its name off the +module again on its next use, as an ordinary import reads it again when it runs + +### what stays an ordinary import + +the polyfill defers a name only where every read of it can be rewritten, and +where nothing reads it by identity before it is used. a name it does not defer +keeps an ordinary import, and the other names of the statement are still +deferred: ```py -from typing import ClassVar -cast = _lazy_attr("typing", "cast") +from json import JSONDecodeError; _lazy_from(_by_lazy, "json", {"dumps": "dumps"}) ``` -nor is a name an **annotation** reads. whatever reads an annotation back is handed what -the name is bound to, and a proxy is not the object it stands for: `dataclasses` decides -that `_: KW_ONLY` is not a field by asking whether the annotation *is* -`dataclasses.KW_ONLY`, and `typing.get_type_hints` answers with the proxy rather than the -class. so a `from` import whose name any annotation reads stays an ordinary import — a -parameter's or a return's annotation, a variable's in a module or a class body, or one -that is a string, as every annotation is under `from __future__ import annotations`: +- **an exception class.** a deferred name is resolved where it is first read, and + for an exception class that is a handler: an import failing there would + surface while another exception is being handled, and replace it +- **a special form** — `ClassVar`, `Final`, `Literal`, `Annotated`, `Protocol` and + the rest of what a type checker gives a meaning of its own. what reads one + tells it apart by identity: `dataclasses` decides a bare `ClassVar` annotation + is not a field by asking whether it *is* `typing.ClassVar`, and a checker + reading the emitted python rejects `ClassVar[int]` once `ClassVar` is anything + else. that holds for the imports the transpiler writes itself, such as the + `ClassVar` an `enum class` lowers through +- **a name an annotation reads.** whatever reads an annotation back reads its names + from the module's namespace: `typing.get_type_hints` evaluates a string + annotation against it, and `dataclasses` decides that `_: KW_ONLY` is not a + field by asking whether the annotation *is* `dataclasses.KW_ONLY`. that counts a + parameter's or a return's annotation, a variable's in a module or a class + body, and one that is a string, as every annotation is under + `from __future__ import annotations`. an annotation on a variable inside a + function body is never evaluated, so it does not count +- **a name a forward reference the transpiler writes reads** — a type parameter's + bound written ahead of the class it reads a name of, as + `TypeVar("T", bound="dict[Thing, C.Inner]")`: whatever evaluates that string + looks the name up in the module's namespace. a string handed to + `typing.ForwardRef` by hand or to `eval` is not seen, and is the last two rows of + the table above +- **a name a decorator reads.** a decorator says what the definition under it + *is*, and a checker reading the emitted python recognizes `@overload` or + `@dataclass` only by name. deferring one buys nothing either: a decorator is + applied while the module is executing +- **a name bound again anywhere in the module** — assigned, deleted, a `def` or a + `class` of that name, a `for`, `with`, `except` or walrus target, a second + import, or a `global` declaration that binds it from a function. a read could + find either binding, and only the one the import makes is deferred +- **a name a class body reads where the body may have bound it already**: the read + finds the class's binding on some paths and the import on others +- **a name read where python keeps the text of the read**: `f"{dumps=}"` prints + the expression as written, and a t-string's `Interpolation` carries it +- **a name beginning with two underscores**, which a class body mangles +- **every name of a module that has a `from … import *`**, which can bind any + name, **or its own `__getattr__` or `__dir__`**, which the deferred names are + served through +- **every name of a module compiled with [`by compile`](../cli-reference.md#by-compile)**, + whose compiled functions read the module's globals by name. its module imports + are still deferred ```by from fractions import Fraction @@ -121,45 +207,10 @@ def show(value: Fraction) -> str: ```py from fractions import Fraction -dumps = _lazy_attr("json", "dumps") +_lazy_from(_by_lazy, "json", {"dumps": "dumps"}) ``` this is an edge of the polyfill: **an import used only in annotations is not deferred on python before 3.15**, and the module it names runs where the import is written — so one -this python does not have fails there, as an ordinary import does. an -annotation on a variable inside a function body is never evaluated, so it does not count. -on 3.15 and later the `lazy` keyword binds the real object, and such an import is deferred -like any other - -the functional forms build annotations out of values instead: `NamedTuple("P", [("x", T)])`, -`TypedDict("TD", {"a": T})` and `dataclasses.make_dataclass(...)` are handed their types as -arguments, when the call runs. a name imported with `from` and read only there is read as a -value, so under the polyfill it is still a proxy, and whatever reads those annotations back is -handed the proxy: `typing.get_type_hints(P)["x"]` is the proxy rather than the class, and -`make_dataclass` makes a field of the `_` in this `Row`, and leaves `extra` positional: - -```by -from dataclasses import KW_ONLY, make_dataclass - -Row = make_dataclass("Row", [("name", str), ("_", KW_ONLY), ("extra", int)]) -``` - -reach such a name through its module — `import dataclasses` and `dataclasses.KW_ONLY` — which -binds the real module rather than a proxy, or target `--min-version 3.15` - -two things a proxy cannot emulate, and which are therefore limitations of the -polyfill only: - -| expression | result | -| ---------------------------- | ----------------------------------------------- | -| `type(x)` | `_LazyAttr`, not the value's real type | -| `x === y` (identity) | compares proxy identity, not value identity | -| `x === None` on imported `x` | always `False`, even when the value *is* `None` | - -the `=== None` case is the sharp one, because identity-against-a-singleton is a -common idiom: `from cfg import SENTINEL` then `SENTINEL === None` is `False` -even when `SENTINEL` is `None`, because `===` (python `is`) sees the proxy, not -the value it stands for. test imported values for emptiness by *value* — `x`'s -truthiness, `x == None`, `isinstance(x, T)` — all of which the proxy forwards. -or target `--min-version 3.15`, where the `lazy` keyword binds the real object -and none of these limitations exist +this python does not have fails there, as an ordinary import does. on 3.15 and later the +`lazy` keyword binds the real object, and such an import is deferred like any other From 660dcc8c20ba13dc2a7d20d0eca3231763b72893 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:24:00 +1000 Subject: [PATCH 37/47] a string in a `.by` type expression is emitted as the literal type the checker reads it as wherever it stands, a generic's type argument included: `list["a"]`, `dict[str, "k"]`, `Callable[["p"], "r"]`, an alias's value and a `type` statement's become `list[Literal["a"]]` and so on, where they were left bare and python read them as forward references to names that do not exist (`typing.get_type_hints` raised `NameError`). a union with no literal arm is now searched for one inside its arms (`list[1] | int`). a forward reference the transpiler quotes itself is still a string, with the literal types inside it lowered --- .../src/reverse_transforms/auto_quote.rs | 6 +- .../reverse_transforms/forward_references.rs | 125 +++++++++++++-- .../src/transforms/lazy_import.rs | 12 ++ .../src/transforms/literal_types.rs | 148 +++++++++++++++--- .../src/transforms/type_expr_walker.rs | 14 +- crates/by_transforms/src/type_info.rs | 15 ++ .../tests/literal_type_runtime.rs | 73 +++++++++ .../mdtest/basedpython_literal_annotations.md | 19 +++ .../ty_python_semantic/src/semantic_model.rs | 18 +++ docs/basedpython/features/lazy-imports.md | 7 +- 10 files changed, 391 insertions(+), 46 deletions(-) create mode 100644 crates/by_transforms/tests/literal_type_runtime.rs diff --git a/crates/by_transforms/src/reverse_transforms/auto_quote.rs b/crates/by_transforms/src/reverse_transforms/auto_quote.rs index a7056c46f8..b1ce518639 100644 --- a/crates/by_transforms/src/reverse_transforms/auto_quote.rs +++ b/crates/by_transforms/src/reverse_transforms/auto_quote.rs @@ -137,9 +137,11 @@ mod tests { ); } + /// a string in a base's type argument is a forward reference in python and a literal type + /// in basedpython, so it comes back as the reference it is, whatever it names #[test] - fn other_strings_unchanged() { - check("class A(list[\"B\"]): ...\n", "class A(list[\"B\"])\n"); + fn a_forward_reference_to_another_name_is_unquoted_too() { + check("class A(list[\"B\"]): ...\n", "class A(list[B])\n"); } #[test] diff --git a/crates/by_transforms/src/reverse_transforms/forward_references.rs b/crates/by_transforms/src/reverse_transforms/forward_references.rs index 0c10f7cdd6..0a55253cd9 100644 --- a/crates/by_transforms/src/reverse_transforms/forward_references.rs +++ b/crates/by_transforms/src/reverse_transforms/forward_references.rs @@ -13,33 +13,56 @@ //! inside the unquoted text is a forward reference too, and is unquoted the same //! way //! -//! the positions are the annotation positions: parameters, returns, variables -//! (local ones included — the reader of a `.by` file reads them all the same), -//! `TypeAlias` values and pep 695 bounds and defaults. inside one, the -//! arguments of `Literal[…]` and the metadata of `Annotated[…]` are values, and -//! stay strings. a string whose text is not a single expression, or is a bare -//! tuple — which means something else once it is not a string — is left as it -//! is +//! the positions are the ones where basedpython reads a string as a literal type, which +//! the forward transpile writes as `Literal[…]`: the very strings that lowering wraps +//! ([`literal_type_strings`]), so that a forward reference python wrote comes back from a +//! round trip as the reference it was. a string left a string here would come back a +//! `Literal`. inside the unquoted text a string is a forward reference in every type +//! position but the arguments of `Literal[…]` and the metadata of `Annotated[…]`. a +//! string whose text is not a single expression, or is a bare tuple — which means +//! something else once it is not a string — is left as it is use ruff_python_ast::Expr; -use ruff_python_parser::{parse_expression, parse_module}; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; +use ruff_python_parser::parse_expression; use ruff_text_size::{Ranged, TextRange}; -use crate::transforms::source_util::for_each_annotation_in_stmt; +use crate::transforms::literal_types::literal_type_strings; use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr}; -/// `source` with each string annotation replaced by the expression it spells +/// `source` with each string the forward transpile would write as a literal type replaced by +/// the expression it spells pub(crate) fn unquote_forward_references(source: &str) -> String { - let Ok(parsed) = parse_module(source) else { + let (db, file) = crate::make_in_memory_db(source); + let program_file = ty_python_semantic::Db::program_file(&db, file); + let module = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); + if !module.errors().is_empty() { return source.to_owned(); + } + let model = ty_python_semantic::SemanticModel::new(&db, program_file); + let ranges = literal_type_strings(source, module.suite(), &model); + let mut strings = Strings { + ranges: &ranges, + collector: Collector { edits: Vec::new() }, }; - let mut collector = Collector { edits: Vec::new() }; - for stmt in parsed.suite() { - for_each_annotation_in_stmt(stmt, |annotation| { - walk_one_type_expr(annotation, &mut collector); - }); + strings.visit_body(module.suite()); + splice(source, strings.collector.edits) +} + +/// the strings at `ranges`, each unquoted as a forward reference +struct Strings<'a> { + ranges: &'a [TextRange], + collector: Collector, +} + +impl<'ast> SourceOrderVisitor<'ast> for Strings<'_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if expr.is_string_literal_expr() && self.ranges.contains(&expr.range()) { + self.collector.visit(expr, TypePos::Root); + } else { + walk_expr(self, expr); + } } - splice(source, collector.edits) } struct Collector { @@ -229,6 +252,74 @@ mod tests { ); } + /// every string the forward transpile would write as a literal type is a forward reference + /// in python, wherever it stands: a type argument in a value expression, a base's, a typing + /// construct's and a type variable's bound among them + #[test] + fn a_string_the_forward_transpile_reads_as_a_literal_type_is_unquoted() { + check( + indoc! {" + from typing import Callable, Generic, List, Optional, TypeVar, Union + + T = TypeVar(\"T\") + + class Foo(Generic[T]): + pass + + class Later: + pass + + Alias = List[\"Later\"] + UAlias = Union[\"Later\", int] + CAlias = Callable[[\"Later\"], \"Later\"] + MaybeLater = Optional[\"Later\"] + B = TypeVar(\"B\", bound=\"Later\") + + class Bar(Foo[\"Later\"]): + pass + "}, + indoc! {" + from typing import Callable, Generic, List, Optional, TypeVar, Union + + T = TypeVar(\"T\") + + class Foo(Generic[T]): + pass + + class Later: + pass + + Alias = List[Later] + UAlias = Union[Later, int] + CAlias = Callable[[Later], Later] + MaybeLater = Optional[Later] + B = TypeVar(\"B\", bound=Later) + + class Bar(Foo[Later]): + pass + "}, + ); + } + + /// the value of a `TypeAlias` is a type expression + #[test] + fn a_type_alias_value_is_unquoted() { + check( + "from typing import TypeAlias\n\nclass Later: ...\n\nX: TypeAlias = \"Later\"\n", + "from typing import TypeAlias\n\nclass Later\n\ntype X = Later\n", + ); + } + + /// a string that is no type expression — a subscript on a class that is not generic, an + /// enum's member lookup — is a value, and stays one + #[test] + fn a_member_lookup_stays_a_string() { + let source = + "from enum import Enum\n\nclass Colour(Enum):\n Red = 1\n\nred = Colour[\"Red\"]\n"; + let out = reverse_transpile(source, &Config::test_default()).unwrap(); + assert!(out.contains("red = Colour[\"Red\"]"), "{out}"); + } + /// a bare tuple means something else once it is not a string #[test] fn a_bare_tuple_stays_a_string() { diff --git a/crates/by_transforms/src/transforms/lazy_import.rs b/crates/by_transforms/src/transforms/lazy_import.rs index 3369ab1132..7cff073d82 100644 --- a/crates/by_transforms/src/transforms/lazy_import.rs +++ b/crates/by_transforms/src/transforms/lazy_import.rs @@ -1545,6 +1545,18 @@ mod tests { ); } + /// a string `.by` reads as a literal type is written as one, and names nothing + #[test] + fn polyfill_a_string_type_argument_names_nothing() { + let out = transpile_polyfill( + "from json import JSONDecoder\nAlias = list[\"JSONDecoder\"]\nprint(JSONDecoder)\n", + ); + assert!( + out.contains("_lazy_from(_by_lazy, \"json\", {\"JSONDecoder\": \"JSONDecoder\"})"), + "{out}" + ); + } + #[test] fn annotation_names_reads_the_annotations_python_keeps() { use ruff_python_parser::parse_module; diff --git a/crates/by_transforms/src/transforms/literal_types.rs b/crates/by_transforms/src/transforms/literal_types.rs index 912015d39b..0f18b3d254 100644 --- a/crates/by_transforms/src/transforms/literal_types.rs +++ b/crates/by_transforms/src/transforms/literal_types.rs @@ -35,6 +35,8 @@ pub(crate) struct LiteralType<'src> { float_literals: FloatLiteralLowering, pub(crate) edits: Vec, pub(crate) needs_literal_import: bool, + /// every string the edits write as a literal type + literal_strings: Vec, } impl<'src> LiteralType<'src> { @@ -52,6 +54,7 @@ impl<'src> LiteralType<'src> { float_literals, edits: Vec::new(), needs_literal_import: false, + literal_strings: Vec::new(), } } @@ -86,6 +89,9 @@ impl<'src> LiteralType<'src> { return; } if is_literal_expr(expr, self.float_literals) { + if expr.is_string_literal_expr() { + self.literal_strings.push(expr.range()); + } self.needs_literal_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( format!("{}[{}]", self.literal, self.src(expr.range())), @@ -122,6 +128,13 @@ impl<'src> LiteralType<'src> { } return; } + // the parameter list of a `Callable[[A, B], R]` + if let Expr::List(list) = expr { + for element in &list.elts { + self.emit_type_edits(element, false); + } + return; + } if let Expr::Subscript(s) = expr { if self.is_literal_name(&s.value) { return; @@ -137,18 +150,20 @@ impl<'src> LiteralType<'src> { if !self.is_type_subscript(&s.value) { return; } + // a string here is a literal type, as it is anywhere else in a `.by` type + // expression — `list["a"]` is `list[Literal["a"]]` — while python would read + // it as a forward reference to a name `a` + // a slice the value is not specialized by is a value, an enum member's name + let specializes = self.types.subscript_specializes(&s.value); match s.slice.as_ref() { Expr::Tuple(t) if !t.parenthesized => { for e in &t.elts { - // bare strings inside a generic subscript are PEP 484 - // forward references; don't promote them to `Literal` - if matches!(e, Expr::StringLiteral(_)) { - continue; + if specializes || !matches!(e, Expr::StringLiteral(_)) { + self.emit_type_edits(e, false); } - self.emit_type_edits(e, false); } } - Expr::StringLiteral(_) => {} + Expr::StringLiteral(_) if !specializes => {} slice => self.emit_type_edits(slice, false), } } @@ -163,6 +178,10 @@ impl<'src> LiteralType<'src> { if !parts.iter().any(|p| { is_literal_expr(p, float_literals) || nominal_float_type(p, float_literals).is_some() }) { + // no arm is a literal, but an arm may hold one: `list["a"] | int` + for part in parts { + self.emit_type_edits(part, false); + } return; } @@ -202,6 +221,9 @@ impl<'src> LiteralType<'src> { group_start = Some(p.range().start()); } } + if p.is_string_literal_expr() { + self.literal_strings.push(p.range()); + } group_list.push(self.src(p.range()).to_owned()); group_end = p.range().end(); } else { @@ -217,6 +239,26 @@ impl<'src> LiteralType<'src> { } } +/// every string in `stmts` that the lowering writes as a literal type, because the checker reads +/// a string there as one — which python, reading the same text, takes for a forward reference +/// +/// the reverse direction unquotes exactly these, so that a forward reference python wrote comes +/// back from a round trip as the reference it was +pub(crate) fn literal_type_strings( + source: &str, + stmts: &[Stmt], + types: &dyn TypeInfo, +) -> Vec { + let mut inner = LiteralType::new( + source, + WrittenNames::new(source), + types, + FloatLiteralLowering::default(), + ); + walk_type_positions_skipping(stmts, Some(types), &[], &mut inner); + inner.literal_strings +} + pub(crate) struct LiteralTypePass<'src> { source: &'src str, written: WrittenNames<'src>, @@ -855,33 +897,97 @@ mod tests { unchanged("a: 1 | 2\n"); } + /// a string in a `.by` type expression is a literal type wherever it is, a subscript's + /// slice included: the checker reads `Foo["Later"]` as `Foo[Literal["Later"]]`, and python + /// would read the bare string as a forward reference to a name `Later` #[test] - fn forward_ref_in_generic_class_subscript_not_promoted() { - // `Foo["Later"]` is a PEP 484 forward reference, not a Literal; - // promoting it to `Foo[Literal["Later"]]` would change the program. + fn a_string_in_a_generic_subscript_is_a_literal_type() { check( indoc! {" - class Foo: pass + class Foo[T]: pass class Bar(Foo[\"Later\"]): pass "}, indoc! {" - class Foo: pass - class Bar(Foo[\"Later\"]): pass + from typing import Literal, TypeVar, Generic + _T = TypeVar(\"_T\") + class Foo(Generic[_T]): pass + class Bar(Foo[Literal[\"Later\"]]): pass "}, ); } #[test] - fn forward_ref_inside_dict_arg_not_promoted() { + fn a_string_argument_of_a_generic_annotation_is_a_literal_type() { check( - indoc! {" - class Foo: pass - a: dict[str, \"Later\"] - "}, - indoc! {" - class Foo: pass - a: dict[str, \"Later\"] - "}, + "a: dict[str, \"Later\"]\n", + "from typing import Literal\na: dict[str, Literal[\"Later\"]]\n", + ); + check( + "a: list[\"x\" | \"y\"]\n", + "from typing import Literal\na: list[Literal[\"x\", \"y\"]]\n", + ); + } + + #[test] + fn a_string_in_a_callable_s_parameter_list_is_a_literal_type() { + check( + "from typing import Callable\na: Callable[[\"x\", int], \"y\"]\n", + "from typing import Callable, Literal\na: Callable[[Literal[\"x\"], int], Literal[\"y\"]]\n", + ); + } + + /// a union with no literal arm can still hold one inside an arm + #[test] + fn a_literal_inside_an_arm_of_a_union_is_lowered() { + check( + "a: list[\"x\"] | int\n", + "from typing import Literal\na: list[Literal[\"x\"]] | int\n", + ); + check( + "a: list[1] | int\n", + "from typing import Literal\na: list[Literal[1]] | int\n", + ); + } + + /// subscripting an enum class looks a member up by name at run time: the string is the + /// name, not a type + #[test] + fn a_runtime_lookup_on_a_class_keeps_its_string() { + let source = indoc! {" + from enum import Enum + class Colour(Enum): + Red = 1 + red = Colour[\"Red\"] + "}; + check(source, source); + } + + /// nor is a subscript the checker resolves to no type at all + #[test] + fn an_unresolved_subscript_on_a_class_keeps_its_string() { + let out = transpile( + "enum class Shape:\n case Circle(radius: float)\n case Point\n\nkind = Shape[\"Circle\"]\n", + &Config::test_default(), + ) + .unwrap(); + assert!(out.contains("kind = Shape[\"Circle\"]"), "{out}"); + } + + /// the value of a `TypeAlias` is a type expression + #[test] + fn a_string_type_alias_value_is_a_literal_type() { + check( + "from typing import TypeAlias\nX: TypeAlias = \"Later\"\n", + "from typing import TypeAlias, Literal\nX: TypeAlias = Literal[\"Later\"]\n", + ); + } + + /// `Annotated`'s metadata and `Literal`'s own arguments are not type expressions + #[test] + fn a_string_that_is_no_type_expression_stays_a_string() { + check( + "from typing import Annotated\na: Annotated[\"x\", \"meta\"]\n", + "from typing import Annotated, Literal\na: Annotated[Literal[\"x\"], \"meta\"]\n", ); } } diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index fe904f472d..28cccec184 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -22,8 +22,8 @@ //! 1. `AnnAssign` annotation //! 2. function parameter annotations (regular, vararg, kwarg, lambda params) //! 3. function return annotation -//! 4. `type X = T` RHS / `X: TypeAlias = T` (the `TypeAlias` form is reached -//! via `AnnAssign`, no special case needed) +//! 4. `type X = T` RHS / `X: TypeAlias = T` (the value of the `TypeAlias` form, +//! as ty reads the annotation) //! 5. `TypeParam` bound (PEP 695 + the basedpython type mapping `T in (…)`) //! 6. `TypeParam` default //! 7. value-position type applications (`list[T]`, `dict[K, V]` outside an @@ -357,7 +357,15 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { declaration_annotation_type(&a.annotation).unwrap_or(&a.annotation); self.visit_root(annotation, RootKind::Annotation); if let Some(v) = &a.value { - self.visit_expr(v); + // the value of `X: TypeAlias = …` is a type expression of its own + if self + .types + .is_some_and(|types| types.is_type_alias_annotation(annotation)) + { + self.visit_root(v, RootKind::Evaluated); + } else { + self.visit_expr(v); + } } } Stmt::TypeAlias(a) => { diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 867fd1c09c..504a008140 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -263,6 +263,13 @@ pub(crate) trait TypeInfo { /// unresolved name should be treated as a runtime subscript, not a type) fn subscript_is_known_type_context(&self, value: &Expr) -> bool; + /// whether subscripting `value` specializes it, so the checker infers its slice as type + /// expressions — see [`SemanticModel::subscript_specializes`] + fn subscript_specializes(&self, value: &Expr) -> bool; + + /// whether `annotation` declares a PEP 613 type alias, whose value is a type expression + fn is_type_alias_annotation(&self, annotation: &Expr) -> bool; + /// the arguments of `call` that are type expressions, because the call is one of the /// typing constructs that spells a type through a call — `NewType("D", int)`, /// `TypeVar("T", bound=int)`, the functional `NamedTuple` and `TypedDict` @@ -902,6 +909,14 @@ impl TypeInfo for SemanticModel<'_> { } } + fn subscript_specializes(&self, value: &Expr) -> bool { + SemanticModel::subscript_specializes(self, value) + } + + fn is_type_alias_annotation(&self, annotation: &Expr) -> bool { + SemanticModel::is_type_alias_annotation(self, annotation) + } + fn call_type_expression_arguments<'ast>(&self, call: &'ast ExprCall) -> Vec<&'ast Expr> { call.func .inferred_type(self) diff --git a/crates/by_transforms/tests/literal_type_runtime.rs b/crates/by_transforms/tests/literal_type_runtime.rs new file mode 100644 index 0000000000..08f3671871 --- /dev/null +++ b/crates/by_transforms/tests/literal_type_runtime.rs @@ -0,0 +1,73 @@ +//! Runtime test for a string in a `.by` type expression, which is a literal type wherever a +//! type expression holds it: the checker reads `list["a"]` as a list of the string `"a"` +//! (`basedpython_literal_annotations.md`, "a string in a generic's type argument is a +//! Literal"). The emitted python has to say the same to whatever reads it back, or +//! `typing.get_type_hints` reads the bare string as a forward reference to a name `a` +//! +//! A forward reference the transpiler quotes itself is a string python has to read as a +//! name, so it stays one, with the literal types inside it lowered + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod interpreters; + +/// the shapes the mdtest reveals, read back through `typing` +const PROGRAM: &str = r#" +import typing +from typing import Callable, Literal, get_args, get_type_hints +from enum import Enum + +class Colour(Enum): + Red = 1 + +type Named = dict[str, "y"] +Alias = list["x"] + +def f(a: list["a"], b: dict[str, "k"], c: Callable[["p"], "r"], n: Named, al: Alias) -> None: + pass + +def later(x: list["q"] | Later) -> None: + pass + +class Later: + pass + +hints = get_type_hints(f) +assert get_args(hints["a"]) == (Literal["a"],), hints["a"] +assert get_args(hints["b"]) == (str, Literal["k"]), hints["b"] +assert get_args(hints["c"]) == ([Literal["p"]], Literal["r"]), hints["c"] +assert get_args(Named.__value__) == (str, Literal["y"]), Named.__value__ +assert get_args(Alias) == (Literal["x"],), Alias +quoted = get_type_hints(later)["x"] +assert get_args(quoted) == (list[Literal["q"]], Later), quoted +# a subscript that looks a member up by name holds no type at all +assert Colour["Red"] === Colour.Red +print("ok") +"#; + +/// from 3.12, where `type` is a statement of its own +#[test] +fn a_string_type_argument_reads_back_as_its_literal_type() { + let Some(interpreter) = interpreters::oldest(PythonVersion::PY312, "", "") else { + return; + }; + let config = Config { + min_version: PythonVersion::PY312, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "failed on {interpreter}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, PythonVersion::PY312); +} diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md b/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md index 14245b340b..f514b526a7 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md @@ -30,6 +30,25 @@ x: Literal["a", "b"] = "a" reveal_type(x) # revealed: "a" ``` +## a string in a generic's type argument is a Literal + +a string is a literal type wherever a type expression holds it, so `list["a"]` is a list of the +string `"a"` rather than a forward reference to a name `a` + +```by +from typing import Callable + +type Named = dict[str, "y"] +Alias = list["x"] + +def f(a: list["a"], b: dict[str, "k"], c: Callable[["p"], "r"], n: Named, al: Alias): + reveal_type(a[0]) # revealed: "a" + reveal_type(b["z"]) # revealed: "k" + reveal_type(c("p")) # revealed: "r" + reveal_type(n["z"]) # revealed: "y" + reveal_type(al[0]) # revealed: "x" +``` + ## a union of string literals is a union of literal types `"foo" | "bar"` is two literal types joined, not a `str.__or__` that would fail at runtime: the diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 62bd74d9ab..7a7b6e19ff 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -963,6 +963,24 @@ impl<'db> SemanticModel<'db> { .is_some_and(|ty| crate::types::subscript::is_runtime_subscript(self.db, env, ty)) } + /// basedpython: whether subscripting `value` specializes it, so that the checker infers + /// the slice as type expressions: a generic class, `tuple`, `type`, a generic alias or a + /// special form. a class that is not generic is subscripted through its metaclass's + /// `__getitem__` or its `__class_getitem__` — `Colour["Red"]` looks an enum member up — + /// and a slice read there is a value. `false` for a value the checker could not resolve + pub fn subscript_specializes(&self, value: &Expr) -> bool { + let db = self.db; + match value.inferred_type(self) { + Some(Type::ClassLiteral(class)) => { + class.generic_context(db).is_some() + || class.is_tuple(db) + || class.is_known(db, crate::types::KnownClass::Type) + } + Some(Type::GenericAlias(_) | Type::SpecialForm(_) | Type::KnownInstance(_)) => true, + _ => false, + } + } + /// basedpython: whether an attribute access resolves through an *implicit /// receiver* — `x.fn` where `fn` names a receiver callable (`int.() -> str`) /// in scope rather than a member of `x`. The transpiler rewrites those to diff --git a/docs/basedpython/features/lazy-imports.md b/docs/basedpython/features/lazy-imports.md index 3d670486e9..7360cc81a9 100644 --- a/docs/basedpython/features/lazy-imports.md +++ b/docs/basedpython/features/lazy-imports.md @@ -174,9 +174,10 @@ from json import JSONDecodeError; _lazy_from(_by_lazy, "json", {"dumps": "dumps" - **a name a forward reference the transpiler writes reads** — a type parameter's bound written ahead of the class it reads a name of, as `TypeVar("T", bound="dict[Thing, C.Inner]")`: whatever evaluates that string - looks the name up in the module's namespace. a string handed to - `typing.ForwardRef` by hand or to `eval` is not seen, and is the last two rows of - the table above + looks the name up in the module's namespace. a string in a `.by` type expression + is a [literal type](literal-types.md) and is written as `Literal["Thing"]`, so it + names nothing; a string handed to `typing.ForwardRef` by hand or to `eval` is not + seen, and is the last two rows of the table above - **a name a decorator reads.** a decorator says what the definition under it *is*, and a checker reading the emitted python recognizes `@overload` or `@dataclass` only by name. deferring one buys nothing either: a decorator is From c4beb25ca3a639cfb44bff1ce4e9a6888c3ee8e7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:46:32 +1000 Subject: [PATCH 38/47] let a type argument written as a value name a class defined further down in `.by`, and quote it in the output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit in a `.by` file the checker now reads the type arguments of a specialization written as a value (`list[F]()`, `Alias = dict[str, F]`, the `Foo[F]` of a base) and the type-expression arguments of a typing construct assigned to a name (`T = TypeVar("T", bound=F)`, `NewType`, the functional `NamedTuple` and `TypedDict`) as deferred, as it reads an annotation, and records which ones it read that way (`TypeExpressionFlags::READ_DEFERRED`, `SemanticModel::is_read_deferred`). the transpiler quotes a forward reference in exactly those (`a = list[F](); class F` → `a = list["F"]()`), and keeps the names a quote reads out of the lazy `from` import polyfill, since a quote is evaluated against the module's namespace. this replaces the rule that quoted only a class's own name in its bases and class body the positions python reads as an object rather than a type are read where they are written, and a name not yet bound there is `unresolved-reference`: a base itself, which was read as deferred before, so `class A(A)` and `class Bar(F)` above `class F` are now `unresolved-reference` where `class A(A)` was silent (a cycle across modules, or in a stub, is still `cyclic-class-definition`); a reified class's type arguments; the bases of `type()` and `types.new_class`; the type an `is`, `cast!`, `cast?` or `cast` reads; and a value inside a type, the arguments of `Literal[...]` and the metadata of `Annotated[...]`, even inside a deferred specialization --- crates/by_transforms/src/lib.rs | 28 +- .../src/transforms/annotation.rs | 149 ++++-- .../src/transforms/ast_driver.rs | 46 +- .../src/transforms/auto_quote.rs | 478 ++++++++++++++---- .../by_transforms/src/transforms/callable.rs | 10 + .../by_transforms/src/transforms/generics.rs | 42 +- .../src/transforms/type_expr_walker.rs | 54 +- crates/by_transforms/src/type_info.rs | 17 + .../tests/forward_reference_runtime.rs | 177 ++++++- .../mdtest/basedpython_class_self_ref.md | 41 +- .../mdtest/basedpython_conformance.md | 19 + .../basedpython_value_forward_references.md | 270 ++++++++++ crates/ty_python_semantic/src/assumed.rs | 22 +- .../ty_python_semantic/src/semantic_model.rs | 68 ++- crates/ty_python_semantic/src/types.rs | 7 +- crates/ty_python_semantic/src/types/infer.rs | 8 + .../src/types/infer/builder.rs | 59 +++ .../src/types/infer/builder/class.rs | 57 ++- .../src/types/infer/builder/subscript.rs | 79 ++- .../types/infer/builder/type_expression.rs | 13 +- .../ty_python_semantic/src/types/subscript.rs | 37 ++ .../features/differences-from-python.md | 7 +- .../features/forward-references.md | 79 ++- docs/basedpython/features/literal-types.md | 14 +- 24 files changed, 1484 insertions(+), 297 deletions(-) create mode 100644 crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 5791d8264d..3c9008ac33 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -132,6 +132,7 @@ fn run_context_sensitive_phase<'a>( return std::borrow::Cow::Borrowed(source); } let model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); + model.infer_in_check_order(); transforms::context_sensitive::qualify(source, parsed.suite(), &model) } @@ -159,6 +160,7 @@ fn run_erased_union_phase<'a>( return std::borrow::Cow::Borrowed(source); } let model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); + model.infer_in_check_order(); transforms::erased_union::reify(source, parsed.suite(), &model, config.min_version) } @@ -258,12 +260,7 @@ fn transpile_with_report( let unchanged = !reified_changed && !qualified_changed && matches!(enum_lowered.output, std::borrow::Cow::Borrowed(_)); - let transforms::ast_driver::Phase0 { - source, - errors: ast_errors, - forward_reference_reads, - .. - } = transforms::ast_driver::run_against_source( + let (source, ast_errors, _phase0_map) = transforms::ast_driver::run_against_source( source, written_names, config, @@ -302,15 +299,11 @@ fn transpile_with_report( // --- Phase 2: import-redirect, surface-syntax cleanup, lazy-import marking --- let (final_output, requirements) = run_import_redirect_phase(output, config); let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; - // a forward reference a lowering wrote as a string reads its names when python evaluates - // it, against the module's namespace, so those are bound where they are imported - let mut eager_names = model.eagerly_imported_names(); - eager_names.extend(forward_reference_reads); let final_output = run_lazy_import_phase( final_output, config, &model.eagerly_imported_modules(), - &eager_names, + &model.eagerly_imported_names(), )?; let final_output = run_version_polyfill_phase(final_output, config); @@ -515,16 +508,9 @@ pub fn transpile_typed_with_report( // single-file db cannot resolve the modules that declare the conformances let eager_model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); let eager_imports = eager_model.eagerly_imported_modules(); - let mut eager_names = eager_model.eagerly_imported_names(); - let transforms::ast_driver::Phase0 { - source: spliced, - errors: ast_errors, - line_map: phase0_map, - forward_reference_reads, - } = transforms::ast_driver::run_against_source(working_source, written_names, config, project); - // a forward reference a lowering wrote as a string reads its names when python evaluates - // it, against the module's namespace, so those are bound where they are imported - eager_names.extend(forward_reference_reads); + let eager_names = eager_model.eagerly_imported_names(); + let (spliced, ast_errors, phase0_map) = + transforms::ast_driver::run_against_source(working_source, written_names, config, project); if let Some(first) = ast_errors.first() { return Err(first.clone().into()); } diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index fe19648202..77d1c63fce 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -1,16 +1,16 @@ use std::cell::{Cell, RefCell}; use ruff_diagnostics::{Edit, Fix}; -use ruff_python_ast::{Expr, PythonVersion, Stmt}; +use ruff_python_ast::{Expr, ExprSubscript, ExprTuple, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::Config; -use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; +use crate::transforms::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::transforms::callable::CallableSyntax; use crate::transforms::optional_type; use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{ - Recurse, RootKind, TypeExprVisitor, TypePos, walk_type_positions, + Recurse, RootKind, TypeExprVisitor, TypePos, callable_parameter_type, walk_type_positions, }; use crate::type_info::TypeInfo; @@ -49,6 +49,7 @@ pub(crate) struct TupleLiteralType<'src> { /// has to carry the fold rather than re-emit the operation symbolic_substitutions: &'src [(TextRange, String)], edits: Vec, + templates: Vec<(TextRange, Vec)>, } impl<'src> TupleLiteralType<'src> { @@ -72,6 +73,7 @@ impl<'src> TupleLiteralType<'src> { leaves: RefCell::new(leaves), symbolic_substitutions, edits: Vec::new(), + templates: Vec::new(), } } @@ -196,19 +198,7 @@ impl<'src> TupleLiteralType<'src> { // parameter list, not a tuple type. Lower to `Callable[[int, // str], R]` (list form) so the runtime / type-checker reads // it as parameters - let is_callable_subscript = matches!(s.value.as_ref(), - Expr::Name(n) if n.id.as_str() == "Callable" - ) || matches!(s.value.as_ref(), - Expr::Attribute(a) if a.attr.id.as_str() == "Callable" - ); - if is_callable_subscript - && let Expr::Tuple(slice_tuple) = s.slice.as_ref() - && !slice_tuple.parenthesized - && slice_tuple.elts.len() == 2 - && let (Some(params_expr), Some(returns_expr)) = - (slice_tuple.elts.first(), slice_tuple.elts.get(1)) - && let Expr::Tuple(params_tuple) = params_expr - && params_tuple.parenthesized + if let Some((params_tuple, returns_expr)) = callable_tuple_parameters(self.types, s) { let params_str = params_tuple .elts @@ -275,19 +265,12 @@ impl<'src> TupleLiteralType<'src> { /// - `*: T` / `*name: T` → dropped (Callable list has no variadic) /// - `**: T` / `**name: T` → dropped fn lower_callable_param(&self, elt: &Expr) -> String { - match elt { - Expr::Named(named) => { - if matches!(named.target.as_ref(), Expr::Starred(_)) { - return String::new(); - } - self.transform_annotation(&named.value) - .unwrap_or_else(|| self.fallback_src(&named.value)) - } - Expr::Starred(_) => String::new(), - _ => self - .transform_annotation(elt) - .unwrap_or_else(|| self.fallback_src(elt)), - } + callable_parameter_type(elt) + .map(|ty| { + self.transform_annotation(ty) + .unwrap_or_else(|| self.fallback_src(ty)) + }) + .unwrap_or_default() } /// Lowers one element of a parameter-shape tuple to a Python type @@ -382,6 +365,26 @@ impl TypeExprVisitor for TupleLiteralType<'_> { if matches!(expr, Expr::Starred(_)) { return Recurse::Descend; } + // basedpython: `Callable[(int, name: str), R]` writes the parameter list as a tuple. it + // becomes python's list, with each parameter's type passed through where it stands, so + // whatever lowers a parameter — a tuple type of its own, a forward reference quoted — + // composes inside the list + if let Expr::Subscript(s) = expr + && let Some((parameters, _)) = callable_tuple_parameters(self.types, s) + { + self.templates + .push((parameters.range(), callable_parameter_list(parameters))); + return Recurse::Descend; + } + // a generic's type arguments are lowered one by one, each where it stands, so a + // pass that writes one of them whole, a forward reference quoted where python reads + // it lazily, wraps that argument's lowering rather than landing inside a rewrite of + // the whole subscript. `X[()]` is python's spelling already + if let Expr::Subscript(s) = expr + && !matches!(s.slice.as_ref(), Expr::Tuple(t) if t.parenthesized && t.elts.is_empty()) + { + return Recurse::Descend; + } // `transform_annotation` is a deep recursive rewriter that produces // a single replacement string for the whole expression. emit the // edit at the expression's range and tell the walker to stop — @@ -439,6 +442,7 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { &symbolic_substitutions, ); walk_type_positions(stmts, Some(types), &mut inner); + ctx.template_edits.append(&mut inner.templates); for fix in inner.edits { for edit in fix.edits() { let range = edit.range(); @@ -466,6 +470,44 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { } } +/// `(int, name: str)`, a `Callable` parameter list written as a tuple, as python's list +/// `[int, str]`, each parameter's type passed through where it stands +fn callable_parameter_list(parameters: &ExprTuple) -> Vec { + let mut fragments = vec![Fragment::Lit("[".to_owned())]; + for (index, ty) in parameters + .elts + .iter() + .filter_map(callable_parameter_type) + .enumerate() + { + if index > 0 { + fragments.push(Fragment::Lit(", ".to_owned())); + } + fragments.push(Fragment::Src(ty.range())); + } + fragments.push(Fragment::Lit("]".to_owned())); + fragments +} + +/// the parameters and return of `Callable[(int, str), R]`, where basedpython writes the +/// parameter list as a tuple. which value is `Callable` is the checker's answer, however it is +/// spelled +fn callable_tuple_parameters<'a>( + types: &dyn TypeInfo, + subscript: &'a ExprSubscript, +) -> Option<(&'a ExprTuple, &'a Expr)> { + if types.is_callable_form(&subscript.value) + && let Expr::Tuple(slice_tuple) = subscript.slice.as_ref() + && !slice_tuple.parenthesized + && let [Expr::Tuple(params_tuple), returns_expr] = slice_tuple.elts.as_slice() + && params_tuple.parenthesized + { + Some((params_tuple, returns_expr)) + } else { + None + } +} + #[cfg(test)] mod tests { use crate::python_passthrough::unchanged; @@ -751,6 +793,55 @@ mod tests { ); } + /// `Callable` is recognised by what the name is bound to, so an alias and a + /// reassignment read their tuple as the parameter list too + #[test] + fn callable_under_another_name_reads_its_tuple_as_parameters() { + check( + indoc! {" + import typing + from typing import Callable as Fn + C = typing.Callable + a: Fn[(int, str), bool] + b: C[(int, str), bool] + "}, + indoc! {" + import typing + from typing import Callable as Fn + C = typing.Callable + a: Fn[[int, str], bool] + b: C[[int, str], bool] + "}, + ); + } + + #[test] + fn collections_abc_callable_reads_its_tuple_as_parameters() { + check( + "from collections.abc import Callable\na: Callable[(int, str), bool]\n", + "from collections.abc import Callable\na: Callable[[int, str], bool]\n", + ); + } + + /// a class of the program's own that happens to be named `Callable` is an + /// ordinary generic, whose tuple argument is a tuple type + #[test] + fn a_class_named_callable_keeps_its_tuple_argument() { + check( + indoc! {" + class Callable[A, B]: ... + a: Callable[(int, str), bool] + "}, + indoc! {" + from typing import TypeVar, Generic + _A = TypeVar(\"_A\") + _B = TypeVar(\"_B\") + class Callable(Generic[_A, _B]): ... + a: Callable[tuple[int, str], bool] + "}, + ); + } + #[test] fn callable_with_variadic_tuple_params() { // `*: T` in callable params is dropped — Callable's list form has diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 4e2fb50373..1d9e37b5e0 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -96,10 +96,6 @@ pub(crate) struct PassContext { /// Full source lines to prepend to the file (e.g. `from typing import cast`). /// Deduped before emission. pub(crate) required_imports: Vec, - /// the names read by a forward reference a pass writes as a string outside an - /// annotation, which python evaluates against the module's namespace when anything - /// evaluates it — so the lazy-import polyfill binds them where they are imported - pub(crate) forward_reference_reads: Vec, /// Runtime helpers the emitted code calls, by the name it calls them under. /// The driver turns these into an import of the module a build writes them /// to, or — when there is no such module — into the definitions themselves. @@ -1173,15 +1169,6 @@ fn merge_from_imports(lines: Vec) -> (Vec, Vec) { (from_lines, other) } -/// what phase 0 makes of a module: the rewritten source, the errors, the line map back to the -/// source, and the names read by the forward references its passes wrote as strings -pub(crate) struct Phase0<'a> { - pub(crate) source: Cow<'a, str>, - pub(crate) errors: Vec, - pub(crate) line_map: Vec>, - pub(crate) forward_reference_reads: Vec, -} - /// Run every registered AST pass against `source` and splice the rewritten /// statements back into the source text. Returns a borrowed `Cow` when /// nothing changed. @@ -1196,7 +1183,7 @@ pub(crate) fn run_against_source<'a>( written: repeated_underscore::WrittenNames, config: &Config, project: SemanticSource<'_>, -) -> Phase0<'a> { +) -> (Cow<'a, str>, Vec, Vec>) { // blank the keyword-prefix type markers — use-site variance and the // `literal`/`final` type modifiers — out up front; downstream passes // (callable, intersection) copy operand source verbatim and would leak the @@ -1231,24 +1218,15 @@ pub(crate) fn run_against_source<'a>( if !parsed_handle.errors().is_empty() { let cow = blanked.stripped(source); let table = crate::source_map::line_table(cow.as_ref(), &[]); - return Phase0 { - source: cow, - errors: Vec::new(), - line_map: table, - forward_reference_reads: Vec::new(), - }; + return (cow, Vec::new(), table); } let parsed = parse_unchecked_source(source_ref, PySourceType::BasedPython); if !parsed.errors().is_empty() { let cow = blanked.stripped(source); let table = crate::source_map::line_table(cow.as_ref(), &[]); - return Phase0 { - source: cow, - errors: Vec::new(), - line_map: table, - forward_reference_reads: Vec::new(), - }; + return (cow, Vec::new(), table); } + semantic_model.infer_in_check_order(); // the parentheses grouping an expression are visible in the tokens and // nowhere in the AST, so anything that needs them has to measure now — the // tokens go when the syntax tree is taken out of the parse @@ -1988,12 +1966,7 @@ pub(crate) fn run_against_source<'a>( { let cow = blanked.stripped(source); let table = crate::source_map::line_table(cow.as_ref(), &[]); - return Phase0 { - source: cow, - errors: ctx.errors, - line_map: table, - forward_reference_reads: ctx.forward_reference_reads, - }; + return (cow, ctx.errors, table); } // splice changed statements back into the source string. process highest @@ -2265,12 +2238,7 @@ pub(crate) fn run_against_source<'a>( preamble_end, &ctx.extension_backing_functions, ); - Phase0 { - source: Cow::Owned(out), - errors: ctx.errors, - line_map: table, - forward_reference_reads: ctx.forward_reference_reads, - } + (Cow::Owned(out), ctx.errors, table) } #[cfg(test)] @@ -2467,7 +2435,7 @@ mod driver_tests { #[test] fn double_coalesce_spliced() { let src = "x = None\na = x ?? x ?? \"fallback\"\n"; - let Phase0 { source: out, .. } = run_against_source( + let (out, _, _) = run_against_source( src, repeated_underscore::WrittenNames::new(src), &Config::test_default(), diff --git a/crates/by_transforms/src/transforms/auto_quote.rs b/crates/by_transforms/src/transforms/auto_quote.rs index d294d7445e..1be8adbe83 100644 --- a/crates/by_transforms/src/transforms/auto_quote.rs +++ b/crates/by_transforms/src/transforms/auto_quote.rs @@ -21,25 +21,34 @@ //! ([`crate::annotations_evaluated`]): on python >= 3.14 they are deferred //! natively (PEP 649), and a `from __future__ import annotations` — written by //! the user, opted into, or written for a target below 3.10 — defers every one, -//! where a quote would reach `__annotations__` as a string inside a string. a class -//! *base*, and a value-position subscript in a class body (`list[A]()`), -//! evaluates while the class is being built on every version, so a -//! self-reference there is always quoted. a direct base (`class A(A):`) is -//! left alone — that is a runtime error regardless of quoting +//! where a quote would reach `__annotations__` as a string inside a string +//! +//! the checker reads two more kinds of type expression as deferred, though python +//! evaluates them where they stand on every version: the type arguments of a +//! specialization written as a value (`list[F]()`, `Alias = dict[str, F]`, a base +//! `Foo[F]`), and the type arguments of a typing construct assigned to a name +//! (`T = TypeVar("T", bound=F)`). which ones it read that way is its answer +//! ([`TypeInfo::is_read_deferred`]), and a forward reference in one is quoted too: +//! +//! `a = list[F](); class F: ...` → `a = list["F"]()` +//! +//! a position the checker reads where it is written — a base itself +//! (`class A(A)`), a reified class's type arguments — reports a name not yet bound +//! there, and is left alone //! //! an annotation holding a forward reference is quoted whole, as one wrapper //! template passing its source through. no lowering reaches past the annotation //! it lowers, so whatever the annotation becomes — a callable arrow, a `T?`, a //! renamed type parameter, even when a lowering rewrites the whole annotation -//! as text — ends up between the quotes. a base or a value-position subscript -//! cannot be a string, so there only the self-reference itself is quoted +//! as text — ends up between the quotes. a specialization cannot be a string, so +//! there only the type arguments holding a reference are quoted -use ruff_python_ast::visitor::{Visitor, walk_expr}; -use ruff_python_ast::{AnyParameterRef, Expr, ExprName, Stmt, StmtClassDef}; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{AnyParameterRef, Expr, ExprName, Stmt, TypeParams}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; -use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr}; +use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr_with}; use crate::type_info::TypeInfo; pub(crate) struct AutoQuote<'src> { @@ -67,7 +76,15 @@ impl TypeAwarePass for AutoQuote<'_> { edits: Vec::new(), }; walk.block(stmts, Scope::Module); - ctx.template_edits.extend(walk.edits); + let mut deferred = DeferredReads { + source: self.source, + types, + edits: walk.edits, + }; + for stmt in stmts { + deferred.visit_stmt(stmt); + } + ctx.template_edits.extend(deferred.edits); } } @@ -95,7 +112,7 @@ impl Walk<'_> { fn stmt(&mut self, stmt: &Stmt, scope: Scope) { match stmt { - Stmt::ClassDef(class) => self.class(class), + Stmt::ClassDef(class) => self.block(&class.body, Scope::Class), Stmt::FunctionDef(function) => { if self.quote_annotations { for parameter in function @@ -162,44 +179,92 @@ impl Walk<'_> { quoter.quote(annotation.range()); } } +} + +/// quotes the forward references in what the checker reads as deferred though python +/// evaluates it where it stands: the type arguments of a specialization written as a value +/// (`list[F]()`, `Alias = dict[str, F]`, a base `Foo[F]`), and the type-expression arguments of +/// a typing construct assigned to a name (`T = TypeVar("T", bound=F)`). which those are is the +/// checker's answer ([`TypeInfo::is_read_deferred`]), so a position it reads where it is +/// written — a base itself, a reified class's type arguments, the type an `is` tests for — is +/// left alone, and a name in it that is not yet bound stays the error the checker reports +struct DeferredReads<'a> { + source: &'a str, + types: &'a dyn TypeInfo, + /// every quote written so far, the annotations' included: nothing inside one is evaluated + /// where it stands any more + edits: Vec<(TextRange, Vec)>, +} + +impl DeferredReads<'_> { + fn quoted_already(&self, range: TextRange) -> bool { + self.edits + .iter() + .any(|(quoted, _)| quoted.contains_range(range)) + } - /// a class's bases and the value-position subscripts in its body run while - /// the class is being built, whatever the version, and the class's own name - /// is not bound until it is - fn class(&mut self, class: &StmtClassDef) { - let class_name = class.name.id.as_str(); - let is_self = |name: &ExprName| name.id.as_str() == class_name; - if let Some(arguments) = &class.arguments { - for base in &arguments.args { - // a direct `class A(A)` base must not be quoted — that would - // mask a runtime error rather than fix it - let mut quoter = Quoter { - source: self.source, - is_forward: &is_self, - edits: &mut self.edits, - skip_root: true, - }; - walk_one_type_expr(base, &mut quoter); + /// quotes the type expression `expr` whole if it holds a forward reference. with + /// `skip_root`, `expr` is the specialization, which is evaluated where it stands, and each + /// of its type arguments is quoted that way instead + fn quote_forward_references(&mut self, expr: &Expr, skip_root: bool) { + let types = self.types; + let is_forward = |name: &ExprName| types.is_forward_reference(name) == Some(true); + let mut quoter = Quoter { + source: self.source, + is_forward: &is_forward, + edits: &mut self.edits, + skip_root, + }; + walk_one_type_expr_with(expr, Some(self.types), &mut quoter); + } +} + +impl<'a> Visitor<'a> for DeferredReads<'_> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + // a `type` statement's value is evaluated lazily, and the type parameters are the + // generics polyfill's to write + Stmt::TypeAlias(_) => {} + // a `TypedDict`'s `extra_items=` is a type expression python evaluates as the class + // is built + Stmt::ClassDef(class) => { + for keyword in class.keywords() { + if self.types.is_read_deferred(&keyword.value) { + self.quote_forward_references(&keyword.value, false); + } + } + walk_stmt(self, stmt); } + _ => walk_stmt(self, stmt), } - for stmt in &class.body { - let value = match stmt { - Stmt::Expr(node) => Some(node.value.as_ref()), - Stmt::Assign(node) => Some(node.value.as_ref()), - Stmt::AnnAssign(node) => node.value.as_deref(), - _ => None, - }; - if let Some(value) = value { - let mut quoter = Quoter { - source: self.source, - is_forward: &is_self, - edits: &mut self.edits, - skip_root: false, - }; - quoter.value_subscripts(value); + } + + // an annotation is quoted whole where python evaluates it, and is never evaluated + // elsewhere + fn visit_annotation(&mut self, _expr: &'a Expr) {} + + fn visit_type_params(&mut self, _type_params: &'a TypeParams) {} + + fn visit_expr(&mut self, expr: &'a Expr) { + if self.quoted_already(expr.range()) { + return; + } + match expr { + Expr::Subscript(subscript) if self.types.is_read_deferred(&subscript.slice) => { + self.quote_forward_references(expr, true); + } + Expr::Call(call) => { + for argument in self.types.call_type_expression_arguments(call) { + if self.types.is_read_deferred(argument) { + self.quote_forward_references(argument, false); + } + } } + _ => {} } - self.block(&class.body, Scope::Class); + // what a quote left out may hold more: the metadata of an `Annotated`, a call's + // other arguments + walk_expr(self, expr); } } @@ -208,38 +273,23 @@ struct Quoter<'a> { source: &'a str, is_forward: &'a dyn Fn(&ExprName) -> bool, edits: &'a mut Vec<(TextRange, Vec)>, - /// when walking a class base, the root expression must not be quoted even - /// if it's a bare self-reference — that would mask the runtime error + /// the root is the specialization whose type arguments are deferred, which is + /// evaluated where it stands itself skip_root: bool, } impl TypeExprVisitor for Quoter<'_> { + /// a type expression holding a forward reference is quoted whole, so whatever it lowers + /// to — a callable arrow, a `T?`, a tuple type, a literal type — ends up between the + /// quotes. quoting one name inside it would leave the construct around it evaluated fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { if std::mem::take(&mut self.skip_root) { - return match expr { - Expr::Subscript(_) | Expr::BinOp(_) => Recurse::Descend, - _ => Recurse::Stop, - }; - } - if !self.contains_forward_reference(expr) { - return Recurse::Stop; + return Recurse::Descend; } - match expr { - Expr::Tuple(_) => Recurse::Descend, - // a generic whose base is not itself a forward reference: quote the - // references inside its arguments at their own level - Expr::Subscript(subscript) if !self.contains_forward_reference(&subscript.value) => { - Recurse::Descend - } - // a union is quoted whole: quoting one arm alone would evaluate - // `str | NoneType` at runtime. anything else that holds a reference - // — a name, a generic rooted at one, an arrow type — is quoted whole - // as the reference it is - _ => { - self.quote(expr.range()); - Recurse::Stop - } + if self.contains_forward_reference(expr) { + self.quote(expr.range()); } + Recurse::Stop } } @@ -271,20 +321,6 @@ impl Quoter<'_> { finder.found } - /// `list[A]()` and similar — quote a self-reference inside a value-position - /// subscript on the LHS of a call. doesn't descend into call arguments - fn value_subscripts(&mut self, expr: &Expr) { - match expr { - Expr::Subscript(subscript) => { - walk_one_type_expr(subscript.slice.as_ref(), self); - self.value_subscripts(&subscript.value); - } - Expr::Call(call) => self.value_subscripts(&call.func), - Expr::Attribute(attribute) => self.value_subscripts(&attribute.value), - _ => {} - } - } - fn quote(&mut self, range: TextRange) { self.edits.push((range, quoted(self.source, range))); } @@ -388,8 +424,280 @@ mod tests { #[test] fn nested_self_reference() { check( + indoc! {" + class Node[T]: ... + class Tree(Node[Tree]): ... + "}, + indoc! {" + from typing import TypeVar, Generic + _T = TypeVar(\"_T\") + class Node(Generic[_T]): ... + class Tree(Node[\"Tree\"]): ... + "}, + ); + } + + /// what the checker cannot resolve it cannot read as a specialization, so a + /// name in its subscript is left as it is + #[test] + fn an_unresolved_generic_quotes_nothing() { + check( + "class Tree(Node[Tree]): ...\n", "class Tree(Node[Tree]): ...\n", - "class Tree(Node[\"Tree\"]): ...\n", + ); + } + + /// a specialization written as a value reads its type arguments when it runs, + /// so a class defined further down is quoted wherever one stands + #[test] + fn a_later_class_in_a_value_specialization_is_quoted() { + check( + indoc! {" + from typing import Callable, Union + items = list[F]() + Alias = dict[str, F] + Maybe = Union[F, None] + Handler = Callable[[F], F] + nested = dict[str, list[F | None]]() + class F: ... + "}, + indoc! {" + from typing import Callable, Union + items = list[\"F\"]() + Alias = dict[str, \"F\"] + Maybe = Union[\"F\", None] + Handler = Callable[[\"F\"], \"F\"] + nested = dict[str, \"list[F | None]\"]() + class F: ... + "}, + ); + } + + /// a base's type arguments are a specialization like any other + #[test] + fn a_later_class_in_a_base_type_argument_is_quoted() { + check( + indoc! {" + class Base[T]: ... + class Bar(Base[F]): ... + class F: ... + "}, + indoc! {" + from typing import TypeVar, Generic + _T = TypeVar(\"_T\") + class Base(Generic[_T]): ... + class Bar(Base[\"F\"]): ... + class F: ... + "}, + ); + } + + /// the types a typing construct assigned to a name takes as call arguments + #[test] + fn a_later_class_in_a_typing_construct_is_quoted() { + check( + indoc! {" + from typing import NamedTuple, NewType, TypedDict, TypeVar + T = TypeVar(\"T\", bound=list[F]) + Id = NewType(\"Id\", F) + Pair = NamedTuple(\"Pair\", [(\"first\", F)]) + Movie = TypedDict(\"Movie\", {\"lead\": F}) + class F: ... + "}, + indoc! {" + from typing import NamedTuple, NewType, TypedDict, TypeVar + T = TypeVar(\"T\", bound=\"list[F]\") + Id = NewType(\"Id\", \"F\") + Pair = NamedTuple(\"Pair\", [(\"first\", \"F\")]) + Movie = TypedDict(\"Movie\", {\"lead\": \"F\"}) + class F: ... + "}, + ); + } + + /// a type argument holding a reference is quoted whole, so the tuple type it + /// lowers to is inside the quote + #[test] + fn a_tuple_type_argument_is_quoted_whole() { + check( + indoc! {" + from typing import TypeVar + Pairs = list[(F, int)] + T = TypeVar(\"T\", bound=(F, int)) + class F: ... + "}, + indoc! {" + from typing import TypeVar + Pairs = list[\"tuple[F, int]\"] + T = TypeVar(\"T\", bound=\"tuple[F, int]\") + class F: ... + "}, + ); + } + + /// a `Callable` parameter list written as a tuple becomes python's list, and + /// each parameter holding a reference is quoted on its own, as a parameter of + /// a list is + #[test] + fn a_callable_tuple_parameter_list_quotes_each_parameter() { + check( + indoc! {" + from typing import Callable + Both = Callable[(F, int), F] + Returns = Callable[(int, str), F] + Nested = list[Callable[(F, int), bool]] + class F: ... + "}, + indoc! {" + from typing import Callable + Both = Callable[[\"F\", int], \"F\"] + Returns = Callable[[int, str], \"F\"] + Nested = list[\"Callable[[F, int], bool]\"] + class F: ... + "}, + ); + } + + /// basedpython type syntax around a reference lowers inside the quote + #[test] + fn a_basedpython_type_argument_is_quoted_with_its_lowering() { + check( + indoc! {" + Maybe = list[F?] + Handler = list[(F) -> None] + Method = list[int.() -> F] + class F: ... + "}, + indoc! {" + from typing import Callable + Maybe = list[\"F | None\"] + Handler = list[\"Callable[[F], None]\"] + Method = list[\"Callable[[int], F]\"] + class F: ... + "}, + ); + } + + /// a `TypedDict`'s `extra_items=` is a type python evaluates as the class is + /// built, and a keyword like `metaclass=` is an object it is built with + #[test] + fn extra_items_is_quoted_and_a_metaclass_is_not() { + check( + indoc! {" + from typing_extensions import TypedDict + class Movie(TypedDict, extra_items=F): ... + class Meta(type): ... + class Plain(metaclass=Meta): ... + class F: ... + "}, + indoc! {" + from typing_extensions import TypedDict + class Movie(TypedDict, extra_items=\"F\"): ... + class Meta(type): ... + class Plain(metaclass=Meta): ... + class F: ... + "}, + ); + } + + /// a string in a type argument is a literal type, and lands in the quote + /// with the reference beside it + #[test] + fn a_literal_type_beside_a_quoted_reference_is_lowered_inside_the_quote() { + check( + indoc! {" + items = list[\"a\" | F]() + class F: ... + "}, + indoc! {" + from typing import Literal + items = list['Literal[\"a\"] | F']() + class F: ... + "}, + ); + } + + /// the specialization itself is evaluated where it stands, so a generic bound + /// only on some paths, which counts as a forward reference, is left as it is + #[test] + fn a_possibly_unbound_generic_is_not_quoted() { + check( + indoc! {" + import sys + if len(sys.argv) > 0: + Box = list + items = Box[F]() + class F: ... + "}, + indoc! {" + import sys + if len(sys.argv) > 0: + Box = list + items = Box[\"F\"]() + class F: ... + "}, + ); + } + + /// a specialization inside a reference already quoted is quoted with it, + /// and not a second time inside the quote + #[test] + fn a_specialization_inside_a_quote_is_not_quoted_again() { + check( + indoc! {" + from typing import Annotated + items = list[F | Annotated[int, dict[str, G]()]]() + class F: ... + class G: ... + "}, + indoc! {" + from typing import Annotated + items = list[\"F | Annotated[int, dict[str, G]()]\"]() + class F: ... + class G: ... + "}, + ); + } + + /// a type argument is quoted whole, as an annotation is, so the metadata of + /// an `Annotated` in it is read with it + #[test] + fn an_annotated_type_argument_is_quoted_whole() { + check( + indoc! {" + from typing import Annotated + items = list[Annotated[F, len]]() + class F: ... + "}, + indoc! {" + from typing import Annotated + items = list[\"Annotated[F, len]\"]() + class F: ... + "}, + ); + } + + /// a name a function binds further down is a forward reference within it, + /// and one the module binds is bound by the time the function runs + #[test] + fn a_function_body_quotes_only_its_own_later_names() { + check( + indoc! {" + def f(): + x = list[G]() + y = list[F]() + G = int + return x, y + class F: ... + "}, + indoc! {" + def f(): + x = list[\"G\"]() + y = list[F]() + G = int + return x, y + class F: ... + "}, ); } @@ -405,7 +713,7 @@ mod tests { fn self_ref_in_nested_subscript() { check( "class A(dict[str, list[A]]): ...\n", - "class A(dict[str, list[\"A\"]]): ...\n", + "class A(dict[str, \"list[A]\"]): ...\n", ); } diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index d6a1cc7a4a..c94faf4cfc 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -1060,6 +1060,16 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> { return crate::transforms::type_expr_walker::Recurse::Stop; } + // a generic's type arguments are lowered one by one, each where it stands, so a + // pass that writes one of them whole, a forward reference quoted where python reads + // it lazily, wraps that argument's lowering rather than landing inside a rewrite of + // the whole subscript. a `typeof` and a decorated type are syntax of their own + if let Expr::Subscript(s) = expr + && !s.is_typeof + && !s.is_type_decoration + { + return crate::transforms::type_expr_walker::Recurse::Descend; + } // `rewrite` is the single type-expression lowerer — it owns every // structural type-form (callable arrows, `&` / `and`, `or`, `not`, // `typeof`, subscripts) and composes leaves through `lower_leaf`. it diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index af4e4fcecf..594375bc84 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -70,10 +70,6 @@ pub(crate) struct GenericPolyfill<'src> { /// ranges of `pending_edits` this pass re-rendered; the driver drops them so /// the stale un-renamed text cannot win the overlap race superseded: Vec, - /// the names a bound or default reads that is written as a string — a forward - /// reference, which python evaluates against the module's namespace when anything - /// evaluates it at all - forward_reference_reads: Vec, /// the `TypeVar` definitions a polyfilled `class` / `def` needs, keyed on the /// start of the line its definition begins on /// @@ -240,7 +236,6 @@ impl<'src> GenericPolyfill<'src> { symbolic_substitutions, pending_edits, superseded: Vec::new(), - forward_reference_reads: Vec::new(), statement_prefixes: Vec::new(), } } @@ -479,13 +474,6 @@ impl<'src> GenericPolyfill<'src> { self.needed_imports.paramspec = true; } - /// record that `expr` is written as a forward reference, a string python evaluates - /// against the module's namespace when anything evaluates it - fn note_forward_reference(&mut self, expr: &Expr) { - self.forward_reference_reads - .extend(names_read(expr).into_iter().map(str::to_owned)); - } - /// lower `params`, each declared under the name at the same position in `names` fn process_type_params( &mut self, @@ -495,16 +483,12 @@ impl<'src> GenericPolyfill<'src> { ) -> ProcessedTypeParams { // a bound that reads a class's name through the class is written ahead of the class, // so it is a string, which `TypeVar` keeps as a forward reference - // — and whether it is one let through_classes = |text: String| { let read_through = apply_renames_to_rendered(&text, qualified); if read_through == text { - (text, false) + text } else { - ( - super::source_util::python_string_literal(&read_through), - true, - ) + super::source_util::python_string_literal(&read_through) } }; let mut generic_args: Vec = Vec::new(); @@ -603,22 +587,18 @@ impl<'src> GenericPolyfill<'src> { ) .unwrap_or_else(|| self.src(bound.range()).to_owned()) }; - let (written, quoted) = - through_classes(apply_renames_to_rendered(&bound_src, &visible)); - if quoted { - self.note_forward_reference(bound); - } - extra_args.push(format!("bound={written}")); + extra_args.push(format!( + "bound={}", + through_classes(apply_renames_to_rendered(&bound_src, &visible)) + )); } } if let Some(default) = &tv.default { - let (written, quoted) = - through_classes(self.default_arg(default, &visible)); - if quoted { - self.note_forward_reference(default); - } - extra_args.push(format!("default={written}")); + extra_args.push(format!( + "default={}", + through_classes(self.default_arg(default, &visible)) + )); } // basedpython variance keywords: forward `out`/`in`/`in out` @@ -2138,8 +2118,6 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { .map(|(index, _)| index) .collect(); ctx.withdrawn_text_edits.extend(withdrawn); - ctx.forward_reference_reads - .extend(std::mem::take(&mut inner.forward_reference_reads)); let emits_any = inner.needed_imports_any; for line in std::mem::take(&mut inner.needed_imports).into_lines(&inner.constructors, self.written) diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index 28cccec184..9cded327ba 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -145,8 +145,17 @@ pub(crate) fn walk_type_positions_skipping( /// callers that already know the expression is in a type position (e.g. /// `generics.rs` re-rewriting the body of a polyfilled type alias) pub(crate) fn walk_one_type_expr(expr: &Expr, visitor: &mut dyn TypeExprVisitor) { + walk_one_type_expr_with(expr, None, visitor); +} + +/// [`walk_one_type_expr`], with `types` to tell which value is `Callable` +pub(crate) fn walk_one_type_expr_with( + expr: &Expr, + types: Option<&dyn TypeInfo>, + visitor: &mut dyn TypeExprVisitor, +) { let mut walker = TypePosWalker { - types: None, + types, claimed: &[], visitor, }; @@ -220,7 +229,7 @@ impl TypePosWalker<'_> { } return; } - if name_matches(&s.value, "Callable") { + if self.is_callable(&s.value) { self.descend_callable(&s.slice); return; } @@ -263,18 +272,38 @@ impl TypePosWalker<'_> { self.visit_type_expr(slice, TypePos::Nested); } + /// whether `value` is `Callable`: the checker's answer where the walk has one, and the + /// spelling where it has none + fn is_callable(&self, value: &Expr) -> bool { + match self.types { + Some(types) => types.is_callable_form(value), + None => name_matches(value, "Callable"), + } + } + fn descend_callable(&mut self, slice: &Expr) { // `Callable[[P1, P2], R]` — slice is an unparenthesized 2-tuple // `(List[P1, P2], R)`. fall back to generic descent for any other // shape (e.g. `Callable[..., R]`, `Callable[P, R]` with ParamSpec) if let Expr::Tuple(t) = slice { if !t.parenthesized && t.elts.len() == 2 { - if let Expr::List(pl) = &t.elts[0] { - for p in &pl.elts { - self.visit_type_expr(p, TypePos::CallableParam); + match &t.elts[0] { + Expr::List(pl) => { + for p in &pl.elts { + self.visit_type_expr(p, TypePos::CallableParam); + } + } + // basedpython: `Callable[(int, name: str), R]` writes the parameter list + // as a tuple, which lowers to python's list. each parameter's type is a + // type position of its own; a variadic one has no place in the list + Expr::Tuple(pt) if pt.parenthesized => { + for p in &pt.elts { + if let Some(ty) = callable_parameter_type(p) { + self.visit_type_expr(ty, TypePos::CallableParam); + } + } } - } else { - self.visit_type_expr(&t.elts[0], TypePos::Nested); + params => self.visit_type_expr(params, TypePos::Nested), } self.visit_type_expr(&t.elts[1], TypePos::Nested); return; @@ -483,6 +512,17 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { } } +/// the type of one parameter of a `Callable` parameter list written as a tuple, or `None` for +/// a variadic one, which python's list form has no place for: `int` for `int` and `name: int` +pub(crate) fn callable_parameter_type(parameter: &Expr) -> Option<&Expr> { + match parameter { + Expr::Named(named) if matches!(named.target.as_ref(), Expr::Starred(_)) => None, + Expr::Named(named) => Some(&named.value), + Expr::Starred(_) => None, + parameter => Some(parameter), + } +} + fn name_matches(expr: &Expr, ident: &str) -> bool { match expr { Expr::Name(n) => n.id.as_str() == ident, diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 504a008140..35a76e5e32 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -270,6 +270,10 @@ pub(crate) trait TypeInfo { /// whether `annotation` declares a PEP 613 type alias, whose value is a type expression fn is_type_alias_annotation(&self, annotation: &Expr) -> bool; + /// whether `expr` is `typing.Callable` or `collections.abc.Callable`, however it is spelled + /// — see [`SemanticModel::is_callable_form`] + fn is_callable_form(&self, expr: &Expr) -> bool; + /// the arguments of `call` that are type expressions, because the call is one of the /// typing constructs that spells a type through a call — `NewType("D", int)`, /// `TypeVar("T", bound=int)`, the functional `NamedTuple` and `TypedDict` @@ -480,6 +484,11 @@ pub(crate) trait TypeInfo { /// annotations`. `None` when ty did not index the name fn is_forward_reference(&self, name: &ExprName) -> Option; + /// whether the checker reads the type expression `expr` as deferred though python evaluates + /// it where it stands, so that a name in it may be a forward reference — see + /// [`SemanticModel::is_read_deferred`] + fn is_read_deferred(&self, expr: &Expr) -> bool; + /// whether every name `expr` reads holds, where it is written, the binding the checker /// resolved when the program runs, so `expr` can be written into code that runs there fn reads_names_bound_where_written(&self, expr: &Expr) -> bool; @@ -917,6 +926,10 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::is_type_alias_annotation(self, annotation) } + fn is_callable_form(&self, expr: &Expr) -> bool { + SemanticModel::is_callable_form(self, expr) + } + fn call_type_expression_arguments<'ast>(&self, call: &'ast ExprCall) -> Vec<&'ast Expr> { call.func .inferred_type(self) @@ -1251,6 +1264,10 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::is_forward_reference(self, name) } + fn is_read_deferred(&self, expr: &Expr) -> bool { + SemanticModel::is_read_deferred(self, expr) + } + fn class_body_member_name(&self, name: &ExprName) -> Option { SemanticModel::class_body_member_name(self, name) } diff --git a/crates/by_transforms/tests/forward_reference_runtime.rs b/crates/by_transforms/tests/forward_reference_runtime.rs index 90cf287857..d4be287603 100644 --- a/crates/by_transforms/tests/forward_reference_runtime.rs +++ b/crates/by_transforms/tests/forward_reference_runtime.rs @@ -1,4 +1,5 @@ -//! Runtime test for forward references in annotations python evaluates as the definition runs. +//! Runtime test for forward references in annotations python evaluates as the definition runs, +//! and in the types written outside an annotation that it evaluates where they stand //! //! basedpython resolves every annotation as deferred, so a signature may name a class defined //! further down, the class it sits in, or a name imported only for the checker. Before 3.14 @@ -6,8 +7,9 @@ //! forward reference raises `NameError` at import — which asserting on the lowered text cannot //! see. //! -//! Needs an interpreter older than 3.14. From 3.14 annotations are deferred natively, so there -//! is nothing to observe, and the test skips rather than pass without checking anything. +//! The annotation test needs an interpreter older than 3.14: from 3.14 annotations are deferred +//! natively, so there is nothing to observe, and it skips rather than pass without checking +//! anything //! //! Below 3.10 the output defers every annotation with `from __future__ import annotations` //! instead, and there a quote would reach `__annotations__` as a string inside a string, which @@ -15,7 +17,7 @@ use std::process::Command; -use by_transforms::{Config, PythonVersion, transpile}; +use by_transforms::{Config, PythonVersion, reverse_transpile, transpile}; mod common; use common::python; @@ -167,3 +169,170 @@ fn a_forward_reference_under_the_future_import_is_not_quoted() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); interpreters::ran(&interpreter, target); } + +/// a type written outside an annotation that python evaluates where it stands, naming classes +/// and an import bound only further down. the checker reads the type arguments as deferred, so +/// the transpiler quotes them, and a name a quote reads is bound where it is imported rather +/// than when the module first reads it, since a quote is evaluated against the module's +/// namespace +const OUTSIDE_AN_ANNOTATION: &str = r#" +from typing import Callable, NamedTuple, TypeVar, Union, get_args + +items = list[Later]() +Alias = dict[str, Later] +Maybe = Union[Later, None] +Handler = Callable[[Later], Later] +T = TypeVar("T", bound=Later) +Pair = NamedTuple("Pair", [("first", Later)]) +Counts = list[OrderedDict] +Pairs = list[(Later, int)] +pair_map = dict[str, (Later, int)]() +P = TypeVar("P", bound=(Later, int)) +MaybeList = list[Later?] +maybes = list[Later?]() +HandlerList = list[(Later) -> None] +Method = list[int.() -> Later] +TupleParams = Callable[(Later, int), Later] + + +class Base[S]: ... + + +class Bar(Base[Later]): ... + + +class PairBar(Base[(Later, int)]): ... + + +class MaybeBar(Base[Later?]): ... + + +class Later: ... + + +from collections import OrderedDict + + +# a builtin generic keeps a string argument as it is, where `typing` makes it a `ForwardRef` +def resolve(reference: object) -> object: + text = reference if isinstance(reference, str) else reference.__forward_arg__ + return eval(text, globals()) + + +assert resolve(get_args(Alias)[1]) == Later, get_args(Alias) +assert resolve(get_args(Maybe)[0]) == Later, get_args(Maybe) +assert resolve(get_args(Handler)[1]) == Later, get_args(Handler) +assert resolve(T.__bound__) == Later, T.__bound__ +assert resolve(get_args(Bar.__orig_bases__[0])[0]) == Later, Bar.__orig_bases__ +assert resolve(get_args(Counts)[0]) == OrderedDict, get_args(Counts) +assert resolve(get_args(Pairs)[0]) == tuple[Later, int], get_args(Pairs) +assert resolve(P.__bound__) == tuple[Later, int], P.__bound__ +assert resolve(get_args(PairBar.__orig_bases__[0])[0]) == tuple[Later, int], PairBar.__orig_bases__ +assert resolve(get_args(MaybeList)[0]) == Later | None, get_args(MaybeList) +assert resolve(get_args(MaybeBar.__orig_bases__[0])[0]) == Later | None, MaybeBar.__orig_bases__ +assert resolve(get_args(HandlerList)[0]) == Callable[[Later], None], get_args(HandlerList) +assert resolve(get_args(Method)[0]) == Callable[[int], Later], get_args(Method) +params, returns = get_args(TupleParams) +assert [resolve(params[0]), params[1], resolve(returns)] == [Later, int, Later], get_args(TupleParams) +print("ok") +"#; + +#[test] +fn a_forward_reference_outside_an_annotation_resolves() { + let Some(python) = python() else { + return; + }; + let config = Config { + min_version: PythonVersion::PY310, + ..Config::default() + }; + let transpiled = transpile(OUTSIDE_AN_ANNOTATION, &config).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +/// python that spells a forward reference outside an annotation as a string, in each place the +/// checker reads one as deferred: an alias's type argument, a union's, a callable's, an +/// optional's, a type variable's bound and a base's. the reverse transpile writes each as the +/// name it spells, and the forward transpile quotes it again, so python sees the same types +const QUOTED_OUTSIDE_AN_ANNOTATION: &str = r#" +from typing import Callable, Generic, List, Optional, TypeVar, Union, get_args, get_type_hints + +T = TypeVar("T") + + +class Foo(Generic[T]): + pass + + +Alias = List["Later"] +UAlias = Union["Later", int] +CAlias = Callable[["Later"], "Later"] +MaybeLater = Optional["Later"] +B = TypeVar("B", bound="Later") + + +class Bar(Foo["Later"]): + pass + + +class Later: + pass + + +def uses(a: Alias, u: UAlias, c: CAlias, m: MaybeLater, b: B) -> Bar: ... + + +print(get_args(Alias), get_args(UAlias), get_args(CAlias), get_args(MaybeLater)) +print(B.__bound__, Bar.__orig_bases__) +print(get_type_hints(uses)) +"#; + +/// what `program` prints on `python` +fn printed(python: &str, program: &str) -> String { + let output = Command::new(python) + .arg("-c") + .arg(program) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "program failed on {python}:\n--- stderr ---\n{}\n--- program ---\n{program}", + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +#[test] +fn a_forward_reference_outside_an_annotation_survives_the_round_trip() { + let Some(python) = python() else { + return; + }; + let config = Config { + min_version: PythonVersion::PY310, + ..Config::default() + }; + let by = reverse_transpile(QUOTED_OUTSIDE_AN_ANNOTATION, &config) + .expect("reverse transpile should succeed"); + assert!( + !by.contains("\"Later\""), + "a forward reference should be the name it spells:\n{by}" + ); + let back = transpile(&by, &config).expect("transpile should succeed"); + assert_eq!( + printed(&python, &back), + printed(&python, QUOTED_OUTSIDE_AN_ANNOTATION), + "--- basedpython ---\n{by}\n--- transpiled back ---\n{back}" + ); +} diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_class_self_ref.md b/crates/ty_python_semantic/resources/mdtest/basedpython_class_self_ref.md index b37deb172d..a015f03f79 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_class_self_ref.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_class_self_ref.md @@ -39,11 +39,12 @@ reveal_type(a["k"]) # revealed: list[A] ## direct base self-reference is still an error -a bare self-reference as a direct base is a genuine runtime error; the transpiler doesn't auto-quote -it, and ty reports the cyclic inheritance +a bare base is resolved where it is written, since the class is built on the object itself, so the +`A` not yet bound there is an unresolved reference, as python's `NameError` for it shows. a type +argument inside a base (`class A(list[A])`) is a forward reference the transpiler quotes ```by -class A(A): # error: [cyclic-class-definition] +class A(A): # error: [unresolved-reference] pass ``` @@ -53,3 +54,37 @@ class A(A): # error: [cyclic-class-definition] class A(list[B]): # error: [unresolved-reference] pass ``` + +## a cycle across modules is a cyclic definition + +each base is bound where it is written, by an import of the other module, so the cycle is between +two classes rather than a name read before it is bound + +`cycle_a.by`: + +```by +from cycle_b import B + +class A(B): # error: [cyclic-class-definition] + pass +``` + +`cycle_b.by`: + +```by +from cycle_a import A + +class B(A): # error: [cyclic-class-definition] + pass +``` + +## a cycle in a stub is a cyclic definition + +a stub is never run, so its bases are read as deferred and may name a class defined further down + +`cycle_stub.byi`: + +```byi +class C(D): ... # error: [cyclic-class-definition] +class D(C): ... # error: [cyclic-class-definition] +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_conformance.md b/crates/ty_python_semantic/resources/mdtest/basedpython_conformance.md index f15ee50ebd..16af69ca0b 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_conformance.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_conformance.md @@ -408,6 +408,25 @@ extension Widget(Show): class Widget: ... ``` +## a conformance declared above the protocol it names is reported + +the protocol is named where the conformance registers itself, so it has to be declared first too + +```by +# error: [invalid-conformance] "`Show` is declared after this conformance" +extension str(Show): + override def show(self) -> str: + return self + +protocol Show: + def show(self) -> str + +def render(value: Show) -> str: + return value.show() + +render("hi") +``` + ## a conformance's own member overrides the interface's default ```by diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md b/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md new file mode 100644 index 0000000000..e057577356 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md @@ -0,0 +1,270 @@ +# basedpython: forward references outside an annotation + +an annotation is read as deferred, so it may name a class the module defines further down. the same +holds for the type arguments of a specialization written as a value, `list[F]()` or +`Alias = dict[str, F]`, and for the types a typing construct such as `TypeVar` takes as call +arguments: python evaluates these where they stand, so the transpiler writes such a name as the +quoted forward reference python reads lazily (`list["F"]()`) + +```toml +[environment] +python-version = "3.12" +``` + +## a specialization's type arguments may name a later class + +```by +from typing import Callable, Union + +items = list[F]() +Alias = dict[str, F] +Maybe = Union[F, None] +Handler = Callable[[F], F] +nested = dict[str, list[F | None]]() + +def use(alias: Alias, maybe: Maybe, handler: Handler): + reveal_type(items) # revealed: final list[F] + reveal_type(alias) # revealed: dict[str, F] + reveal_type(maybe) # revealed: F | None + reveal_type(handler) # revealed: (F, /) -> F + reveal_type(nested) # revealed: final dict[str, list[F | None]] + +class F: ... +``` + +## a base's type arguments may name a later class + +a base is the object the class is built on, so it is read where it is written, but its type +arguments are a specialization like any other + +```by +class Base[T]: + def get(self) -> T: + raise NotImplementedError + +class Bar(Base[F]): ... + +def use(bar: Bar): + reveal_type(bar.get()) # revealed: F + +class F: ... +``` + +## a tuple type argument may name a later class + +a type argument holding the reference is quoted whole, so the tuple type it lowers to is read lazily +with it + +```by +from typing import TypeVar + +class Foo[T]: + def get(self) -> T: + raise NotImplementedError + +Alias = list[(F, int)] +pairs = dict[str, (F, int)]() +T = TypeVar("T", bound=(F, int)) + +class Bar(Foo[(F, int)]): ... + +def use(alias: Alias, bar: Bar): + reveal_type(alias) # revealed: list[(F, int)] + reveal_type(pairs) # revealed: final dict[str, (F, int)] + reveal_type(bar.get()) # revealed: (F, int) + +class F: ... +``` + +## basedpython type syntax around a later class + +an optional, an intersection or a callable type holding the reference is quoted whole, with what it +lowers to inside the quote + +```by +class Foo[T]: + def get(self) -> T: + raise NotImplementedError + +class G: ... + +Maybe = list[F?] +Both = list[F & G] +Handler = list[(F) -> None] +Method = list[int.() -> F] +maybes = list[F?]() + +class Bar(Foo[F?]): ... + +def use(maybe: Maybe, both: Both, handler: Handler, method: Method, bar: Bar): + reveal_type(maybe) # revealed: list[F | None] + reveal_type(both) # revealed: list[F & G] + reveal_type(handler) # revealed: list[(F, /) -> None] + reveal_type(method) # revealed: list[(int, /) -> F] + reveal_type(maybes) # revealed: final list[F | None] + reveal_type(bar.get()) # revealed: F | None + +class F: ... +``` + +## a `Callable` parameter list written as a tuple may name a later class + +each parameter of the list python's `Callable` takes is a type of its own + +```by +from typing import Callable + +Both = Callable[(F, int), F] +Returns = Callable[(int, str), F] + +def use(both: Both, returns: Returns): + reveal_type(both) # revealed: (F, int, /) -> F + reveal_type(returns) # revealed: (int, str, /) -> F + +class F: ... +``` + +## a typing construct's type arguments may name a later class + +```by +from typing import NamedTuple, NewType, TypedDict, TypeVar + +T = TypeVar("T", bound=F) +Id = NewType("Id", F) +Pair = NamedTuple("Pair", [("first", F)]) +Movie = TypedDict("Movie", {"lead": F}) + +def use(pair: Pair, movie: Movie, t: T) -> T: + reveal_type(pair.first) # revealed: F + reveal_type(movie["lead"]) # revealed: F + return t + +class F: ... +``` + +## a name nothing binds is still unresolved + +```by +items = list[Missing]() # error: [unresolved-reference] +``` + +## a base itself is read where it is written + +the class is built on the object, so a quote could not stand in for it, whether the class is generic +or not + +```by +class Bar(F): # error: [unresolved-reference] + pass + +class GenericBar[T](F): # error: [unresolved-reference] + pass + +class Shaped((*: F)): # error: [unresolved-reference] + pass + +class F: ... +``` + +## a keyword a class is built with is read where it is written + +a metaclass, or a keyword `__init_subclass__` takes, is an object the class is built with + +```by +class Tagged: + def __init_subclass__(cls, tag: object = None): + pass + +class WithMeta(metaclass=Meta): # error: [unresolved-reference] + pass + +class WithTag(Tagged, tag=F): # error: [unresolved-reference] + pass + +class GenericWithMeta[T](metaclass=Meta): # error: [unresolved-reference] + pass + +class Meta(type): ... +class F: ... +``` + +## a `TypedDict`'s `extra_items` may name a later class + +`extra_items=` is the type of the keys a `TypedDict` does not list, so it is a type like a +specialization's arguments + +```by +from typing_extensions import TypedDict + +class Movie(TypedDict, extra_items=F): ... +class Pair[T](TypedDict, extra_items=list[F]): ... + +class F: ... +``` + +## the bases of a class built by a call are read where they are written + +`type()` builds the class on the base objects, as a `class` statement does + +```by +import types + +Built = type("Built", (F,), dict()) # error: [unresolved-reference] +NewBuilt = types.new_class("NewBuilt", (F,)) # error: [unresolved-reference] + +class F: ... +``` + +## a reified class's type arguments are read where they are written + +a reified class keeps its type arguments as runtime values, which a forward reference cannot stand +in for + +```by +class Box[reified T]: + pass + +box = Box[F]() # error: [unresolved-reference] +boxes = list[Box[F]]() # error: [unresolved-reference] + +class F: ... +``` + +## the type a test or a cast checks for is read where it is written + +`is`, `cast!` and `cast?` check the value against the type when they run, and `cast` reads its type +as a type expression rather than a specialization + +```by +from typing import cast + +def make() -> object: + return [] + +value = make() +is_list = value is list[F] # error: [unresolved-reference] +checked = value cast? list[F] # error: [unresolved-reference] +cast_list = cast(list[F], value) # error: [unresolved-reference] + +class F: ... +``` + +## a value inside a specialization is read where it is written + +a `Literal`'s parameters and the metadata of an `Annotated` are values python evaluates while it +builds the specialization, even where the type around them is read as deferred + +```by +from enum import Enum +from typing import Annotated, Literal + +# error: [invalid-type-form] +# error: [unresolved-reference] +literals = list[Literal[E.A]]() +annotated = list[Annotated[int, F()]]() # error: [unresolved-reference] + +class E(Enum): + A = 1 + +class F: ... +``` diff --git a/crates/ty_python_semantic/src/assumed.rs b/crates/ty_python_semantic/src/assumed.rs index 5b9b4618cb..7b855fab4a 100644 --- a/crates/ty_python_semantic/src/assumed.rs +++ b/crates/ty_python_semantic/src/assumed.rs @@ -474,16 +474,22 @@ pub(crate) fn seeded_place<'db>( scope: ScopeId<'db>, place_expr: PlaceExprRef, ) -> Option> { - scope.program(db).assumptions(db)?; - let seeds = seeds(db, scope); - if seeds.is_empty() { + let assumptions = scope.program(db).assumptions(db)?; + let index = semantic_index(db, scope.program_file(db)); + let table = index.place_table(scope.file_scope_id(db)); + let place_id = table.place_id(place_expr)?; + // only a place the debugger observed can have a seed, and that is asked before the seeds are + // built: building them types every observation, and typing an enum member reads its class's + // bases, which are name loads in this same scope — `class Color(Enum)` loading `Enum` would + // otherwise wait on the seeds it is part of building + if !assumptions + .observations(db) + .iter() + .any(|observation| place_id_of(table, &observation.name) == Some(place_id)) + { return None; } - let index = semantic_index(db, scope.program_file(db)); - let place_id = index - .place_table(scope.file_scope_id(db)) - .place_id(place_expr)?; - seeds.get(&place_id).copied() + seeds(db, scope).get(&place_id).copied() } /// whether a use is at or below the line the program is stopped on diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 7a7b6e19ff..798a7fb3e8 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -20,6 +20,7 @@ use crate::place::implicit_globals::all_implicit_module_globals; use crate::place::imported_symbol; use crate::place::{builtins_module_scope, implicit_builtins_symbol_scope}; use crate::types::AnonNamedTupleClass; +use crate::types::TypeExpressionFlags; use crate::types::ide_support::{ImportAliasResolution, definition_for_name}; use crate::types::implicit_names::implicit_name; use crate::types::list_members::{all_members, all_reachable_members}; @@ -27,7 +28,8 @@ use crate::types::runtime_spelling::RuntimeSite; use crate::types::{ ClassPatternPositionalResult, ClassPatternPositionalSource, CycleDetector, ProgramEnvironment, SpecialFormType, Type, TypeQualifiers, binding_type, class_pattern_positional_result, - class_pattern_positional_sources, infer_complete_scope_types, inferred_declaration, + class_pattern_positional_sources, infer_complete_scope_types, infer_scope_types, + inferred_declaration, }; use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_core::node_key::NodeKey; @@ -69,6 +71,21 @@ impl<'db> SemanticModel<'db> { self.db } + /// basedpython: infers every scope of the file, each before the scopes inside it, as + /// checking the file does. a reader about to ask for the types of most of the file asks + /// this first: a scope inferred before the scope around it reads that scope's names + /// before that scope has inferred the conditions deciding whether they are bound there, + /// and each of those conditions is then inferred inside the inference of the next + pub fn infer_in_check_order(&self) { + let index = semantic_index(self.db, self.file); + for scope in index.scope_ids() { + // a scope that takes type context is inferred inside the scope around it + if !scope.accepts_type_context(self.db) { + infer_scope_types(self.db, scope, crate::types::TypeContext::default()); + } + } + } + pub fn file(&self) -> File { self.file.file(self.db) } @@ -964,21 +981,42 @@ impl<'db> SemanticModel<'db> { } /// basedpython: whether subscripting `value` specializes it, so that the checker infers - /// the slice as type expressions: a generic class, `tuple`, `type`, a generic alias or a - /// special form. a class that is not generic is subscripted through its metaclass's - /// `__getitem__` or its `__class_getitem__` — `Colour["Red"]` looks an enum member up — - /// and a slice read there is a value. `false` for a value the checker could not resolve + /// the slice as type expressions — the same answer inference acts on (see + /// `types::subscript::specializes_on_subscript`). `false` for a value the checker could + /// not resolve pub fn subscript_specializes(&self, value: &Expr) -> bool { - let db = self.db; - match value.inferred_type(self) { - Some(Type::ClassLiteral(class)) => { - class.generic_context(db).is_some() - || class.is_tuple(db) - || class.is_known(db, crate::types::KnownClass::Type) - } - Some(Type::GenericAlias(_) | Type::SpecialForm(_) | Type::KnownInstance(_)) => true, - _ => false, - } + value + .inferred_type(self) + .is_some_and(|ty| crate::types::subscript::specializes_on_subscript(self.db, ty)) + } + + /// basedpython: whether `expr` is `typing.Callable` or `collections.abc.Callable`, whatever + /// name it is reached by — an alias (`from typing import Callable as Fn`), an assignment + /// (`C = typing.Callable`) — and not a class of the program's own that is named `Callable` + pub fn is_callable_form(&self, expr: &Expr) -> bool { + matches!( + expr.inferred_type(self), + Some(Type::SpecialForm( + SpecialFormType::TypingCallable | SpecialFormType::CollectionsAbcCallable + )) + ) + } + + /// basedpython: whether inference reads the type expression `expr` as deferred, as it + /// does an annotation, though python evaluates it where it stands — the slice of a + /// specialization written as a value (`list[F]()`), or a type-expression argument of a + /// typing construct assigned to a name (`TypeVar("T", bound=F)`). a name in one may be + /// bound only further on, and the transpiler writes it as the forward reference python + /// reads lazily + pub fn is_read_deferred(&self, expr: &Expr) -> bool { + let file = self.program_file(); + let index = semantic_index(self.db, file); + let Some(file_scope) = index.try_expression_scope_id(&ast::ExprRef::from(expr)) else { + return false; + }; + infer_complete_scope_types(self.db, file_scope.to_scope_id(self.db, file)) + .type_expression_flags(expr) + .contains(TypeExpressionFlags::READ_DEFERRED) } /// basedpython: whether an attribute access resolves through an *implicit diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index d73e765b58..40271a0afa 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -44,9 +44,10 @@ pub use self::diagnostic::{ }; use self::infer::infer_function_default_types; pub(crate) use self::infer::{ - ArgumentContextOrigin, InferredDeclaration, TypeContext, infer_complete_scope_types, - infer_deferred_types, infer_definition_types, infer_expression_type, infer_expression_types, - infer_same_file_expression_type, infer_scope_types, is_discarded_dict_key_assignment, + ArgumentContextOrigin, InferredDeclaration, TypeContext, TypeExpressionFlags, + infer_complete_scope_types, infer_deferred_types, infer_definition_types, + infer_expression_type, infer_expression_types, infer_same_file_expression_type, + infer_scope_types, is_discarded_dict_key_assignment, }; pub(crate) use self::iteration::{ extract_fixed_length_iterable_element_types, report_iteration_over_character, diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index 0760fb8e0f..268478f11e 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -97,6 +97,14 @@ bitflags::bitflags! { /// The expression refers to a `TypeVarTuple` without unpacking it. const INVALID_BARE_TYPE_VAR_TUPLE = 1 << 2; + + /// basedpython: a type expression python evaluates where it stands, which inference + /// reads as deferred all the same, as it does an annotation: the type arguments of a + /// specialization written as a value (`list[F]()`), and a type-expression argument of + /// a typing construct assigned to a name (`T = TypeVar("T", bound=F)`). a name in one + /// may be bound only further on, and the transpiler writes it as the forward reference + /// python reads lazily + const READ_DEFERRED = 1 << 3; } } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 3b46689c7e..febe55d921 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -530,6 +530,12 @@ pub(super) struct TypeInferenceBuilder<'db, 'ast> { /// is a stub file but we're still in a non-deferred region. deferred_state: DeferredExpressionState, + /// basedpython: the state a specialization written as a value is itself evaluated in, while + /// its type arguments are inferred as deferred (see `infer_subscript_load`). what is a value + /// inside them, `Literal[E.A]` or the metadata of `Annotated[T, m]`, python still evaluates + /// where it is written, so it is inferred in this state instead + specialization_evaluation_state: Option, + /// For decorated function or class definitions, the type before applying decorators. undecorated_type: Option>, @@ -675,6 +681,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return_types_and_ranges: vec![], called_functions: FxIndexSet::default(), deferred_state: DeferredExpressionState::None, + specialization_evaluation_state: None, expressions: FxHashMap::default(), comparison_truthiness: FxHashMap::default(), expression_cache: None, @@ -1219,6 +1226,19 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { previous } + /// infers what is a value inside a specialization's type arguments in the state the + /// specialization itself is evaluated in, when those arguments are being read as deferred + fn infer_as_evaluated(&mut self, infer: impl FnOnce(&mut Self) -> T) -> T { + let Some(state) = self.specialization_evaluation_state.take() else { + return infer(self); + }; + let previous = self.replace_deferred_state(state); + let result = infer(self); + self.deferred_state = previous; + self.specialization_evaluation_state = Some(state); + result + } + /// Returns `true` if `expr` is a call to a known diagnostic function /// (e.g., `reveal_type` or `assert_type`) whose return value should not /// trigger the `unused-awaitable` lint. @@ -4811,6 +4831,37 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } fn infer_assignment_deferred(&mut self, target: &ast::Expr, value: &'ast ast::Expr) { + // basedpython: the type expressions a typing construct takes as call arguments — + // `TypeVar("T", bound=F)` — are read as deferred, as an annotation is, so they may name + // a class the module binds further on: the transpiler writes such a name as the forward + // reference python reads lazily. the bases of `type("C", bases, ns)` are classes the + // call needs, so they are not + let deferred_form = if self.is_basedpython_file() + && let ast::Expr::Call(call) = value + && let callee = self + .try_expression_type(&call.func) + .unwrap_or_else(|| self.infer_expression(&call.func, TypeContext::default())) + && let Some(form) = CallTypeForm::of(self.db(), callee) + { + Some((form, call)) + } else { + None + }; + let previous_deferred_state = deferred_form + .is_some() + .then(|| self.replace_deferred_state(DeferredExpressionState::Deferred)); + self.infer_assignment_deferred_impl(target, value); + if let Some(previous) = previous_deferred_state { + self.deferred_state = previous; + } + if let Some((form, call)) = deferred_form { + for argument in form.type_expressions(&call.arguments) { + self.store_type_expression_flags(argument, TypeExpressionFlags::READ_DEFERRED); + } + } + } + + fn infer_assignment_deferred_impl(&mut self, target: &ast::Expr, value: &'ast ast::Expr) { let db = self.db(); let env = self.program_environment(); // Infer deferred bounds/constraints/defaults of a legacy TypeVar / ParamSpec / NewType, @@ -17113,6 +17164,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { reachability_cache: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, called_functions, index: _, region: _, @@ -17185,6 +17237,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { slice_materialization: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, index: _, region: _, } = self; @@ -17313,6 +17366,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { discards_dict_key_assignments: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, index: _, region: _, cycle_recovery: _, @@ -17377,6 +17431,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { slice_materialization: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, index: _, region: _, return_types_and_ranges: _, @@ -17540,6 +17595,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { slice_materialization: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, called_functions: _, index: _, region: _, @@ -17597,6 +17653,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { index, cycle_recovery, deferred_state, + specialization_evaluation_state, typevar_binding_context, ref expression_cache, ref reachability_cache, @@ -17648,6 +17705,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // Ensure the speculative builder has the same inference context as the current one. builder.cycle_recovery = cycle_recovery; builder.deferred_state = deferred_state; + builder.specialization_evaluation_state = specialization_evaluation_state; builder.typevar_binding_context = typevar_binding_context; builder.context.inference_flags = self.inference_flags(); builder.expression_cache.clone_from(expression_cache); @@ -17709,6 +17767,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { reachability_cache: _, typevar_binding_context: _, deferred_state: _, + specialization_evaluation_state: _, called_functions, index: _, region: _, diff --git a/crates/ty_python_semantic/src/types/infer/builder/class.rs b/crates/ty_python_semantic/src/types/infer/builder/class.rs index fb8e6c5c50..50a7355ea6 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/class.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/class.rs @@ -2,6 +2,7 @@ use crate::Db; use crate::ProgramEnvironment; use crate::reified::{UnansweredReason, reified_class_reads}; use crate::types::diagnostic::{INVALID_VARIANCE_DECLARATION, REIFIED_WITHOUT_RECEIVER}; +use crate::types::infer::TypeExpressionFlags; use crate::types::{ CallArguments, ClassLiteralFlags, DataclassFlags, DataclassParams, KnownClass, KnownInstanceType, SpecialFormType, StaticClassLiteral, SubclassOfType, Type, TypeContext, @@ -115,8 +116,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { self.report_reified_class_errors(class); if class.arguments.is_some() { - let defer_class_args = self.in_stub() || self.is_basedpython_file(); - let previous_deferred_state = self.replace_deferred_state(defer_class_args.into()); + let defer_bases = self.reads_bases_deferred(class); + let previous_deferred_state = self.replace_deferred_state(defer_bases.into()); // PEP 695 class headers are inferred in the type-parameter scope, before the completed // class type is available. Infer the bases first because `extra_items=T` is an @@ -125,8 +126,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { let mut is_typed_dict = false; for base in class.bases() { - let ty = if let Some(ty) = - self.infer_parameter_shape_class_base(base, defer_class_args) + let ty = if let Some(ty) = self.infer_parameter_shape_class_base(base, defer_bases) { ty } else if let ast::Expr::Starred(starred) = base { @@ -144,9 +144,10 @@ impl<'db> TypeInferenceBuilder<'db, '_> { }; } + self.replace_deferred_state(self.in_stub().into()); for keyword in class.keywords() { if is_typed_dict && keyword.arg.as_deref() == Some("extra_items") { - self.infer_extra_items_kwarg(&keyword.value); + self.infer_typed_dict_extra_items(&keyword.value); } else { self.infer_expression(&keyword.value, TypeContext::default()); } @@ -527,10 +528,9 @@ impl<'db> TypeInferenceBuilder<'db, '_> { // if there are type parameters, then the keywords and bases are within that scope // and we don't need to run inference here if type_params.is_none() { - // In stub files (and basedpython files, where self-refs are auto-quoted by the - // transpiler), keyword values may reference names that are defined later. - let defer_class_args = self.in_stub() || self.is_basedpython_file(); - let previous_deferred_state = self.replace_deferred_state(defer_class_args.into()); + // a keyword is an object the class is built with, read where it is written as a + // base is, except in a stub, which is never run + let previous_deferred_state = self.replace_deferred_state(self.in_stub().into()); for keyword in class_node.keywords() { if keyword.arg.as_deref() != Some("extra_items") { self.infer_expression(&keyword.value, TypeContext::default()); @@ -539,7 +539,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { self.deferred_state = previous_deferred_state; // Inference of bases deferred in stubs/basedpython, or if any are string literals. - if defer_class_args + if self.in_stub() + || self.is_basedpython_file() || class_node .bases() .iter() @@ -615,13 +616,39 @@ impl<'db> TypeInferenceBuilder<'db, '_> { Some(base_ty) } + /// a `TypedDict`'s `extra_items=`, which is a type expression. in a basedpython file it is + /// read as deferred, as an annotation is, though python evaluates it as the class is built, + /// so the transpiler writes a forward reference in it as one python reads lazily + fn infer_typed_dict_extra_items(&mut self, value: &ast::Expr) { + let previous_deferred_state = + self.replace_deferred_state((self.in_stub() || self.is_basedpython_file()).into()); + self.infer_extra_items_kwarg(value); + self.deferred_state = previous_deferred_state; + if self.is_basedpython_file() { + self.store_type_expression_flags(value, TypeExpressionFlags::READ_DEFERRED); + } + } + + /// whether a class's bases are read as deferred rather than where they are written + /// + /// a base is the object the class is built on, so outside a stub it is read where it is + /// written, and a name not yet bound there is unresolved, as python's `NameError` for it is + /// + /// a basedpython base's type arguments are deferred all the same, as every specialization + /// written as a value is — see `infer_subscript_load`. the protocols a basedpython extension + /// lists (`extension str(Show)`) are not bases: the conformance check reads them, and reports + /// one declared below the extension itself, as `invalid-conformance` + fn reads_bases_deferred(&self, class: &ast::StmtClassDef) -> bool { + self.in_stub() || class.is_extension() + } + pub(super) fn infer_class_deferred( &mut self, definition: Definition<'db>, class: &ast::StmtClassDef, ) { let previous_typevar_binding_context = self.typevar_binding_context.replace(definition); - let defer_class_args = self.in_stub() || self.is_basedpython_file(); + let defer_bases = self.reads_bases_deferred(class); // resolve bases with `IN_TYPE_EXPRESSION` set so the basedpython // `dynamic` keyword resolves to `Any` in a base list (`class C(dynamic)`), // matching the transpiler's `dynamic → Any` rewrite. only `dynamic` name @@ -632,12 +659,12 @@ impl<'db> TypeInferenceBuilder<'db, '_> { .replace(InferenceFlags::IN_TYPE_EXPRESSION, true); for base in class.bases() { if self - .infer_parameter_shape_class_base(base, defer_class_args) + .infer_parameter_shape_class_base(base, defer_bases) .is_some() { continue; } - if defer_class_args { + if defer_bases { self.infer_expression_with_state( base, TypeContext::default(), @@ -658,8 +685,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { if original_class_type(self.db(), definition) .is_some_and(|class_literal| class_literal.is_typed_dict(self.db())) { - self.infer_extra_items_kwarg(&extra_items_keyword.value); - } else if defer_class_args { + self.infer_typed_dict_extra_items(&extra_items_keyword.value); + } else if self.in_stub() { self.infer_expression_with_state( &extra_items_keyword.value, TypeContext::default(), diff --git a/crates/ty_python_semantic/src/types/infer/builder/subscript.rs b/crates/ty_python_semantic/src/types/infer/builder/subscript.rs index ec2f58c6bb..d0c0d371c8 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/subscript.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/subscript.rs @@ -5,7 +5,7 @@ use ruff_python_ast::{self as ast, ArgOrKeyword, ExprContext, ParameterBorrow}; use ruff_text_size::Ranged; use ty_module_resolver::file_to_module; -use super::TypeInferenceBuilder; +use super::{DeferredExpressionState, TypeInferenceBuilder}; use crate::place::{DefinedPlace, Definedness, Place}; use crate::types::call::Argument; use crate::types::call::CallErrorKind; @@ -483,13 +483,27 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // let the `None` ride out to the end of the chain let (value_ty, in_chain) = self.basedpython_chain_receiver(&subscript.value, value_ty); - // If we have an implicit type alias like `MyList = list[T]`, and if `MyList` is being - // used in another implicit type alias like `Numbers = MyList[int]`, then we infer the - // right hand side as a value expression, and need to handle the specialization here. - if value_ty.is_generic_alias() { - return Ok(self.infer_explicit_type_alias_specialization(subscript, value_ty, false)); + // basedpython: the type arguments of a specialization written as a value — `list[F]()`, + // `Alias = dict[str, F]`, `Foo[F]` among a class's bases — are read as deferred, as an + // annotation is, so they may name a class the module binds further on: the transpiler + // writes such a name as the forward reference python reads lazily + let defer = self.is_basedpython_file() + && crate::types::subscript::defers_type_arguments(self.db(), value_ty); + let previous_states = defer.then(|| { + let evaluated = self.replace_deferred_state(DeferredExpressionState::Deferred); + let previous_evaluation = self.specialization_evaluation_state.replace(evaluated); + (evaluated, previous_evaluation) + }); + let loaded = self.infer_subscript_load_specialization(subscript, value_ty, tcx); + if let Some((evaluated, previous_evaluation)) = previous_states { + self.deferred_state = evaluated; + self.specialization_evaluation_state = previous_evaluation; + self.store_type_expression_flags(&*subscript.slice, TypeExpressionFlags::READ_DEFERRED); } - let loaded = self.infer_subscript_load_impl(value_ty, subscript, tcx); + let loaded = match loaded { + Ok(loaded) => loaded, + Err(specialized) => return Ok(specialized), + }; let ty = loaded.unwrap_or_else(|recovery_ty| recovery_ty); // `m[1]` goes through `Match.__getitem__`, whose stub can only say @@ -502,6 +516,23 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { loaded.map(|_| ty).map_err(|_| ty) } + /// the subscript `value_ty[...]` loads: `Err` with the result of re-specializing an implicit + /// type alias, which is final, or `Ok` with the ordinary subscript's + fn infer_subscript_load_specialization( + &mut self, + subscript: &ast::ExprSubscript, + value_ty: Type<'db>, + tcx: TypeContext<'db>, + ) -> Result, Type<'db>>, Type<'db>> { + // If we have an implicit type alias like `MyList = list[T]`, and if `MyList` is being + // used in another implicit type alias like `Numbers = MyList[int]`, then we infer the + // right hand side as a value expression, and need to handle the specialization here. + if value_ty.is_generic_alias() { + return Err(self.infer_explicit_type_alias_specialization(subscript, value_ty, false)); + } + Ok(self.infer_subscript_load_impl(value_ty, subscript, tcx)) + } + /// The type of `m[key]` for a `re.Match` whose capture groups are known. fn refine_regex_subscript( &self, @@ -936,12 +967,26 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .replace(InferenceFlags::DISABLE_INT_FLOAT_SPECIAL_CASE, true) }); - let result = self.infer_explicit_callable_specialization( - subscript, - value_ty, - generic_context, - specialize, - ); + // a reified class keeps its type arguments as runtime values, so they are read where + // they are written even inside a specialization whose type arguments are deferred: + // `list[Box[F]]` hands `Box` the object `F`, not a forward reference to it + let result = if crate::types::subscript::defers_type_arguments(db, value_ty) { + self.infer_explicit_callable_specialization( + subscript, + value_ty, + generic_context, + specialize, + ) + } else { + self.infer_as_evaluated(|builder| { + builder.infer_explicit_callable_specialization( + subscript, + value_ty, + generic_context, + specialize, + ) + }) + }; if let Some(previously_disabled_int_float_special_case) = previously_disabled_int_float_special_case @@ -3159,9 +3204,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .context .inference_flags .replace(InferenceFlags::IN_TYPE_ALIAS, false); - for metadata_element in &arguments[1..] { - self.infer_expression(metadata_element, TypeContext::default()); - } + self.infer_as_evaluated(|builder| { + for metadata_element in &arguments[1..] { + builder.infer_expression(metadata_element, TypeContext::default()); + } + }); self.context .inference_flags .set(InferenceFlags::IN_TYPE_ALIAS, previous_in_type_alias); diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index fd456868c4..ab329460ec 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -4558,6 +4558,15 @@ impl<'db> TypeInferenceBuilder<'db, '_> { pub(crate) fn infer_literal_parameter_type<'param>( &mut self, parameters: &'param ast::Expr, + ) -> Result, Vec<&'param ast::Expr>> { + // a literal's parameters are values python evaluates where they are written, an enum + // member's class included + self.infer_as_evaluated(|builder| builder.infer_literal_parameter_type_impl(parameters)) + } + + fn infer_literal_parameter_type_impl<'param>( + &mut self, + parameters: &'param ast::Expr, ) -> Result, Vec<&'param ast::Expr>> { let db = self.db(); let env = self.program_environment(); @@ -4565,7 +4574,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { ast::Expr::Subscript(ast::ExprSubscript { value, slice, .. }) => { let value_ty = self.infer_expression(value, TypeContext::default()); if matches!(value_ty, Type::SpecialForm(SpecialFormType::Literal)) { - let ty = self.infer_literal_parameter_type(slice)?; + let ty = self.infer_literal_parameter_type_impl(slice)?; // This branch deals with annotations such as `Literal[Literal[1]]`. // Here, we store the type for the inner `Literal[1]` expression: @@ -4582,7 +4591,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { let mut errors = vec![]; let mut builder = UnionBuilder::new(db, env); for elt in tuple { - match self.infer_literal_parameter_type(elt) { + match self.infer_literal_parameter_type_impl(elt) { Ok(ty) => { builder = builder.add(ty); } diff --git a/crates/ty_python_semantic/src/types/subscript.rs b/crates/ty_python_semantic/src/types/subscript.rs index 2193a37afc..90cb4b4c55 100644 --- a/crates/ty_python_semantic/src/types/subscript.rs +++ b/crates/ty_python_semantic/src/types/subscript.rs @@ -27,6 +27,43 @@ use super::{ TypeVarBoundOrConstraints, TypedDictType, UnionBuilder, UnsafeUnionType, todo_type, }; +/// basedpython: whether subscripting a value of type `ty` specializes it, so that its slice +/// holds type arguments, which inference reads as type expressions: a generic class, `tuple`, +/// `type`, a generic alias or a special form. a class that is not generic is subscripted +/// through its metaclass's `__getitem__` or its `__class_getitem__` — `Colour["Red"]` looks an +/// enum member up — and its slice is a value +pub(crate) fn specializes_on_subscript<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { + match ty { + Type::ClassLiteral(class) => { + class.generic_context(db).is_some() + || class.is_tuple(db) + || class.is_known(db, KnownClass::Type) + } + Type::GenericAlias(_) | Type::SpecialForm(_) | Type::KnownInstance(_) => true, + _ => false, + } +} + +/// basedpython: whether the type arguments subscripting a value of type `ty` are read as +/// deferred in a `.by` file, as an annotation is: a name in them may be one the module binds +/// only further on, and the transpiler writes it as the forward reference python reads lazily +/// +/// a reified class keeps its type arguments as runtime values, which a forward reference +/// cannot stand in for, so its are read where they are written +pub(crate) fn defers_type_arguments<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { + if !specializes_on_subscript(db, ty) { + return false; + } + match ty { + Type::ClassLiteral(class) => !class.generic_context(db).is_some_and(|context| { + context + .variables(db) + .any(|typevar| typevar.is_reified_class_typevar(db)) + }), + _ => true, + } +} + /// basedpython: whether subscripting a value of type `ty` is a runtime /// `__getitem__` call rather than a type specialization. /// diff --git a/docs/basedpython/features/differences-from-python.md b/docs/basedpython/features/differences-from-python.md index b9753517b5..e5add2c339 100644 --- a/docs/basedpython/features/differences-from-python.md +++ b/docs/basedpython/features/differences-from-python.md @@ -107,9 +107,10 @@ x: "Foo" ``` python reads that as a deferred reference to the name `Foo`; basedpython reads -it as the string-literal type. there is no manual forward-reference syntax -because none is needed — the transpiler -[quotes a self-reference for you](forward-references.md) when the +it as the string-literal type, in an annotation or a type argument +(`list["a"]()`) alike. there is no manual forward-reference syntax because none +is needed — the transpiler +[quotes a forward reference for you](forward-references.md) when the runtime requires it ### `float` means `float` diff --git a/docs/basedpython/features/forward-references.md b/docs/basedpython/features/forward-references.md index 42f7995221..cf6b7fc9e7 100644 --- a/docs/basedpython/features/forward-references.md +++ b/docs/basedpython/features/forward-references.md @@ -46,13 +46,60 @@ in its lowered form (`"Tree | None"`, `"Callable[[Tree], None]"`) a name that is already bound is left alone, as is a local variable's annotation, which python never evaluates -a class's *subscript bases* evaluate while the class is being built, and so do -value-position subscripts in its body (`list[Tree]()`). a self-reference there -is quoted where it stands: `class Tree(list["Tree"])`. a direct base — -`class A(A)` — is *not* quoted. that is always a runtime error, and quoting it -would only mask the bug. method *bodies* are not rewritten either: by the time -a body runs, the class is bound, and quoting would produce a string instead of -a value +method *bodies* are not rewritten: by the time a body runs, the class is bound, +and quoting would produce a string instead of a value + +## outside an annotation + +a type written as a value can name a later class too, wherever python only +records it as a type: the type arguments of a generic (`list[Tree]()`, +`Alias = dict[str, Tree]`, a base `Node[Tree]`), and the types a typing +construct assigned to a name takes as arguments (`TypeVar`, `NewType`, the +functional `NamedTuple` and `TypedDict`). a type argument that names one is +quoted whole, as an annotation is, with whatever it lowers to inside the quote: + +```by +from typing import Callable, TypeVar + +registry = dict[str, Plugin]() +Handler = Callable[[Plugin], None] +P = TypeVar("P", bound=Plugin) + + +class Plugin: ... +``` + +python evaluates these where they stand, on every version, so the reference is +quoted where it is written: + +```python +from typing import Callable, TypeVar + +registry = dict[str, "Plugin"]() +Handler = Callable[["Plugin"], None] +P = TypeVar("P", bound="Plugin") + + +class Plugin: ... +``` + +a string there is a [literal type](literal-types.md), as it is in an +annotation, so you write the forward reference as the bare name and never as a +string + +some positions python reads as an object rather than as a type, and a quote +cannot stand in for one. a name not bound by then is an `unresolved-reference` +there: + +- a base itself: `class A(A)`, or `class Bar(Plugin)` above `Plugin`. its type + arguments are quoted as above (`class Tree(list[Tree])`) +- a keyword the class is built with, such as `metaclass=Meta`. a `TypedDict`'s + `extra_items=` is a type, and is quoted as above +- the type arguments of a [reified](reified-class-generics.md) class, which keeps + them as runtime values, wherever it stands: `list[Box[Plugin]]` too +- the type an `is`, `cast!` or `cast?` checks for, and the type `cast` reads +- a value inside a type: the arguments of `Literal[…]` (an enum member) and + the metadata of `Annotated[…]` ## why automatic @@ -66,9 +113,17 @@ transpiler quotes exactly the references that need it in python a string annotation is a forward reference. in basedpython a string in an annotation is a [literal type](literal-types.md), so `by transpile --reverse` writes each one as the expression it spells: -`-> "Tree"` becomes `-> Tree`, and `"Tree | None"` becomes `Tree?`. the -arguments of `Literal[…]` and the metadata of `Annotated[…]` are values, and -stay strings +`-> "Tree"` becomes `-> Tree`, and `"Tree | None"` becomes `Tree?`. the same +goes for a string outside an annotation that names a type: `List["Tree"]` +becomes `List[Tree]`, and `TypeVar("T", bound="Tree")` becomes +`TypeVar("T", bound=Tree)`. the arguments of `Literal[…]` and the metadata of +`Annotated[…]` are values, and stay strings, as does a subscript that is a +lookup rather than a generic (`Colour["Red"]` on an enum) + +the type `cast` reads is read where it is written, so it has no spelling for a +class defined further down: `cast("Later", o)` above `class Later` comes back +as `cast(Later, o)`, which the checker reports as an `unresolved-reference`. +move the cast below the class ## when quoting is skipped @@ -82,5 +137,5 @@ evaluated eagerly. it is skipped when: opted into the blanket injection, or target a version below 3.10, which always gets it -a subscript base evaluates eagerly on every target, so its self-reference is -quoted either way +a type outside an annotation evaluates where it stands on every target, so a +forward reference in one is quoted either way diff --git a/docs/basedpython/features/literal-types.md b/docs/basedpython/features/literal-types.md index f3a4608f7b..e659eabe78 100644 --- a/docs/basedpython/features/literal-types.md +++ b/docs/basedpython/features/literal-types.md @@ -68,9 +68,17 @@ the output reports the argument as invalid ## scope promotion fires only in syntactic type contexts: annotations, return types, -type aliases, and generic subscript slices whose target is a type. value -expressions are untouched. `Annotated[T, metadata]` does not treat its -metadata slice as a type context, so literal metadata is preserved +type aliases, and the type arguments of a generic, written as a value or not. +`list["a"]()` is a list of the string `"a"`, transpiled to +`list[Literal["a"]]()`, where python would read `"a"` as a forward reference +to a name `a`. a forward reference is written as the bare name, which the +transpiler quotes where python needs it (see +[forward references](forward-references.md#outside-an-annotation)) + +value expressions are untouched, and so is a subscript that looks a value up +rather than naming a generic: `Colour["Red"]` on an enum keeps its string. +`Annotated[T, metadata]` does not treat its metadata slice as a type context, +so literal metadata is preserved ## polyfill From c08473dd650a8edb7dc36da1120a4c3d40ba5de1 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:45:36 +1000 Subject: [PATCH 39/47] quote an anonymous named tuple's field type when it names something the module binds the class an anonymous named tuple lowers to is written ahead of everything the module binds, so a field naming one of those (`x: list[(a: F, b: int)]` with `class F` further down, or a `typeof y` of a module variable) raised `NameError` as the module was imported. such a field type is now written as a string, as a renamed type variable's already was, with the lowered type inside it (`a: "F"`), in an annotation and in a type written as a value alike --- .../src/transforms/anon_named_tuple.rs | 60 +++++++++++++++++-- .../src/transforms/ast_driver.rs | 13 +++- .../src/transforms/auto_quote.rs | 26 +++++--- .../tests/forward_reference_runtime.rs | 6 +- .../basedpython_value_forward_references.md | 15 +++++ 5 files changed, 105 insertions(+), 15 deletions(-) diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index 3ecb1eaaca..1b6525637d 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -226,15 +226,41 @@ impl<'src> AnonNamedTuple<'src> { fn render_field_type(&mut self, expr: &Expr) -> String { let lowered = self.callable.lower_type_expr(expr); let renamed = self.apply_typevar_renames(&lowered); - // a renamed typevar is declared below the hoisted class, so the annotation - // must stay a string - if renamed != lowered && !lowered.is_empty() { - format!("\"{renamed}\"") + // the class is hoisted ahead of everything the module binds, a renamed typevar + // among them, so a field naming one is a forward reference there and is quoted + // as one + let forward = renamed != lowered || self.reads_a_module_binding(expr); + if forward && !lowered.is_empty() { + super::auto_quote::quoted_text(&renamed) } else { renamed } } + /// whether `expr` reads a name the module binds + fn reads_a_module_binding(&self, expr: &Expr) -> bool { + struct Finder<'a> { + types: &'a dyn TypeInfo, + found: bool, + } + impl<'ast> Visitor<'ast> for Finder<'_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && self.types.is_bound_globally(&name.id) + { + self.found = true; + } + walk_expr(self, expr); + } + } + let mut finder = Finder { + types: self.types, + found: false, + }; + finder.visit_expr(expr); + finder.found + } + /// Active typevar rename map, flattened from the stack with inner scopes /// shadowing outer ones. Empty when not inside a generic scope or when /// `min_version` is 3.12+ (where PEP 695 stays native and no renames @@ -825,6 +851,29 @@ mod tests { ); } + /// the class is hoisted ahead of everything the module binds, so a field naming + /// one of those is a forward reference there, and is quoted as one + #[test] + fn a_field_naming_a_module_binding_is_quoted() { + check( + indoc! {" + x: list[(a: F, b: int)] = [] + Alias = list[(a: F, b: int)] + class F: ... + "}, + indoc! {" + from typing import NamedTuple + class _AnonNamedTuple_58e90127(NamedTuple): + a: \"F\" + b: int + + x: \"list[_AnonNamedTuple_58e90127]\" = [] + Alias = list[\"_AnonNamedTuple_58e90127\"] + class F: ... + "}, + ); + } + /// the synthesized class body leaves the statement the field type was /// written in, so nothing else lowers it — `dynamic` reaching the output /// verbatim is a `NameError` at import, and a wrapped type is a syntax error @@ -854,7 +903,8 @@ mod tests { 1, "expected a single class def, got: {out}" ); - assert!(out.contains(" m: TypeOf[b]"), "got: {out}"); + // `b` is bound below the hoisted class, so the field reads it as a forward reference + assert!(out.contains(" m: \"TypeOf[b]\""), "got: {out}"); } #[test] diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 1d9e37b5e0..7889dd4870 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -2762,12 +2762,23 @@ mod subsumed_lowerings { Lowering::FloatConst, Lowering::Typeof, ]; + // the class is hoisted ahead of the module's own bindings, so a field reading one — + // the `y` a `typeof` names — is written as a forward reference + let (reads_binding, reads_none): (Vec, Vec) = fields + .iter() + .partition(|lowering| **lowering == Lowering::Typeof); cases.extend(leaves( "AnonNamedTuplePass", - &fields, + &reads_none, |leaf| format!("def f(x: (a: {leaf}, b: str)) -> None: ..."), |lowered| format!("a: {lowered}"), )); + cases.extend(leaves( + "AnonNamedTuplePass", + &reads_binding, + |leaf| format!("def f(x: (a: {leaf}, b: str)) -> None: ..."), + |lowered| format!("a: \"{lowered}\""), + )); cases.extend(leaves( "ProtocolTypePass", &fields, diff --git a/crates/by_transforms/src/transforms/auto_quote.rs b/crates/by_transforms/src/transforms/auto_quote.rs index 1be8adbe83..06a360e40d 100644 --- a/crates/by_transforms/src/transforms/auto_quote.rs +++ b/crates/by_transforms/src/transforms/auto_quote.rs @@ -330,8 +330,23 @@ impl Quoter<'_> { /// the lowerings inside it land between the quotes. the delimiter is one the source it /// wraps does not already use pub(crate) fn quoted(source: &str, range: TextRange) -> Vec { - let text = &source[range]; - let delimiter = ["\"", "'", "\"\"\"", "'''"] + let delimiter = delimiter_for(&source[range]); + vec![ + Fragment::Lit(delimiter.to_owned()), + Fragment::Src(range), + Fragment::Lit(delimiter.to_owned()), + ] +} + +/// `text`, a lowered type expression, written as the forward reference python reads lazily +pub(crate) fn quoted_text(text: &str) -> String { + let delimiter = delimiter_for(text); + format!("{delimiter}{text}{delimiter}") +} + +/// a string delimiter `text` does not already use +fn delimiter_for(text: &str) -> &'static str { + ["\"", "'", "\"\"\"", "'''"] .into_iter() .find(|delimiter| { !text.contains(delimiter) @@ -340,12 +355,7 @@ pub(crate) fn quoted(source: &str, range: TextRange) -> Vec { .next() .is_none_or(|quote| !text.ends_with(quote)) }) - .unwrap_or("\""); - vec![ - Fragment::Lit(delimiter.to_owned()), - Fragment::Src(range), - Fragment::Lit(delimiter.to_owned()), - ] + .unwrap_or("\"") } #[cfg(test)] diff --git a/crates/by_transforms/tests/forward_reference_runtime.rs b/crates/by_transforms/tests/forward_reference_runtime.rs index d4be287603..d4546f9530 100644 --- a/crates/by_transforms/tests/forward_reference_runtime.rs +++ b/crates/by_transforms/tests/forward_reference_runtime.rs @@ -176,7 +176,7 @@ fn a_forward_reference_under_the_future_import_is_not_quoted() { /// than when the module first reads it, since a quote is evaluated against the module's /// namespace const OUTSIDE_AN_ANNOTATION: &str = r#" -from typing import Callable, NamedTuple, TypeVar, Union, get_args +from typing import Callable, NamedTuple, TypeVar, Union, get_args, get_type_hints items = list[Later]() Alias = dict[str, Later] @@ -192,7 +192,9 @@ MaybeList = list[Later?] maybes = list[Later?]() HandlerList = list[(Later) -> None] Method = list[int.() -> Later] +Fields = list[(first: Later, count: int)] TupleParams = Callable[(Later, int), Later] +annotated: list[(first: Later, count: int)] = [] class Base[S]: ... @@ -234,6 +236,8 @@ assert resolve(get_args(HandlerList)[0]) == Callable[[Later], None], get_args(Ha assert resolve(get_args(Method)[0]) == Callable[[int], Later], get_args(Method) params, returns = get_args(TupleParams) assert [resolve(params[0]), params[1], resolve(returns)] == [Later, int, Later], get_args(TupleParams) +fields = resolve(get_args(Fields)[0]) +assert get_type_hints(fields) == {"first": Later, "count": int}, get_type_hints(fields) print("ok") "#; diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md b/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md index e057577356..98a527f32a 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_value_forward_references.md @@ -124,6 +124,21 @@ def use(both: Both, returns: Returns): class F: ... ``` +## an anonymous named tuple may name a later class + +its class is written ahead of everything the module binds, so a field naming a later class is a +forward reference there + +```by +Fields = list[(first: F, count: int)] +fields: list[(first: F, count: int)] = [] + +def use(entries: Fields): + reveal_type(entries[0].first) # revealed: F + +class F: ... +``` + ## a typing construct's type arguments may name a later class ```by From 7a5b186734e9ecea4d5f504e1d3dbafa3bf7d4ef Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:42:48 +1000 Subject: [PATCH 40/47] record each type the transpiler writes as a string where it writes it, and keep the names it reads ordinary imports below python 3.15 the lazy-import polyfill defers a `from` import's names, but python evaluates a string type (`list["F | JSONDecoder"]`) against the module's namespace, where a deferred name is not bound yet: `typing.get_type_hints` raised `NameError` for a name read only inside such a string a quote can now only be written through `WrittenQuotes` (`transforms/quotes.rs`): `quote(text)` writes a lowered type as a string and records it, and `quote_span(range)` gives a `Fragment::Quote` the driver writes as a string, recording what it lowered to. every producer holds the registry: the forward-reference quoting, a `cast` target, the fields of a hoisted anonymous named tuple, typed dict and inline or callable protocol, and a type variable's bound read through its classes. code that only picks names never sees it. the lazy-import phase keeps every name the recorded quotes read an ordinary import --- crates/by_transforms/src/lib.rs | 46 ++++-- crates/by_transforms/src/source_map.rs | 17 +- .../src/transforms/annotation.rs | 14 +- .../src/transforms/anon_named_tuple.rs | 22 ++- .../src/transforms/ast_driver.rs | 150 ++++++++++++++---- .../src/transforms/auto_quote.rs | 58 ++----- .../by_transforms/src/transforms/callable.rs | 50 +++--- .../src/transforms/checked_cast.rs | 15 +- .../by_transforms/src/transforms/generics.rs | 22 ++- .../src/transforms/init_method.rs | 10 +- .../src/transforms/lazy_import.rs | 46 ++++++ crates/by_transforms/src/transforms/mod.rs | 1 + .../src/transforms/protocol_type.rs | 26 +-- crates/by_transforms/src/transforms/quotes.rs | 63 ++++++++ .../by_transforms/src/transforms/rerender.rs | 2 + .../src/transforms/source_util.rs | 12 +- .../src/transforms/typed_dict_literal.rs | 57 ++++--- .../tests/lazy_import_runtime.rs | 96 +++++++++++ 18 files changed, 537 insertions(+), 170 deletions(-) create mode 100644 crates/by_transforms/src/transforms/quotes.rs diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 3c9008ac33..2995c7da1b 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -22,6 +22,7 @@ pub mod entry_point { }; } +use transforms::quotes::WrittenQuotes; use transforms::repeated_underscore::{CodeNames, PrivateNames, SynthesizedClasses}; use std::collections::{BTreeSet, HashSet}; @@ -252,6 +253,7 @@ fn transpile_with_report( ty_python_semantic::extension_backing_functions(&local_db, local_file), ); let synthesized_classes = SynthesizedClasses::default(); + let quotes = WrittenQuotes::default(); let written_names = written_names .with_private(&private_names) .with_synthesized(&synthesized_classes); @@ -263,6 +265,7 @@ fn transpile_with_report( let (source, ast_errors, _phase0_map) = transforms::ast_driver::run_against_source( source, written_names, + "es, config, if unchanged { transforms::ast_driver::SemanticSource::Project(&local_db, local_file) @@ -298,12 +301,13 @@ fn transpile_with_report( // --- Phase 2: import-redirect, surface-syntax cleanup, lazy-import marking --- let (final_output, requirements) = run_import_redirect_phase(output, config); - let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; + let final_output = run_anon_named_tuple_cleanup(final_output, written_names, "es, config)?; let final_output = run_lazy_import_phase( final_output, config, &model.eagerly_imported_modules(), &model.eagerly_imported_names(), + "es.texts(), )?; let final_output = run_version_polyfill_phase(final_output, config); @@ -473,6 +477,7 @@ pub fn transpile_typed_with_report( ty_python_semantic::extension_backing_functions(db, file), ); let synthesized_classes = SynthesizedClasses::default(); + let quotes = WrittenQuotes::default(); let written_names = written_names .with_private(&private_names) .with_synthesized(&synthesized_classes); @@ -509,8 +514,13 @@ pub fn transpile_typed_with_report( let eager_model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); let eager_imports = eager_model.eagerly_imported_modules(); let eager_names = eager_model.eagerly_imported_names(); - let (spliced, ast_errors, phase0_map) = - transforms::ast_driver::run_against_source(working_source, written_names, config, project); + let (spliced, ast_errors, phase0_map) = transforms::ast_driver::run_against_source( + working_source, + written_names, + "es, + config, + project, + ); if let Some(first) = ast_errors.first() { return Err(first.clone().into()); } @@ -551,8 +561,14 @@ pub fn transpile_typed_with_report( } let (final_output, requirements) = run_import_redirect_phase(output, config); - let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; - let final_output = run_lazy_import_phase(final_output, config, &eager_imports, &eager_names)?; + let final_output = run_anon_named_tuple_cleanup(final_output, written_names, "es, config)?; + let final_output = run_lazy_import_phase( + final_output, + config, + &eager_imports, + &eager_names, + "es.texts(), + )?; let final_output = run_version_polyfill_phase(final_output, config); // phases 1-2c only prepend preambles at the top and edit within lines, so @@ -627,6 +643,7 @@ fn line_count(s: &str) -> usize { fn run_anon_named_tuple_cleanup( mut source: String, written: WrittenNames, + quotes: &WrittenQuotes, config: &Config, ) -> Result { use ruff_python_ast::visitor::Visitor; @@ -648,6 +665,7 @@ fn run_anon_named_tuple_cleanup( let mut anon = transforms::anon_named_tuple::AnonNamedTuple::new( src, written, + quotes, &model, config.clone(), module.suite(), @@ -659,8 +677,14 @@ fn run_anon_named_tuple_cleanup( if let Some(err) = anon.errors.first() { return Err(err.clone()); } - let protocol = - transforms::protocol_type::cleanup(src, written, &model, module.suite(), config)?; + let protocol = transforms::protocol_type::cleanup( + src, + written, + quotes, + &model, + module.suite(), + config, + )?; if anon.edits.is_empty() && !anon.needs_import && protocol.is_none() { return Ok(source); @@ -768,8 +792,10 @@ fn run_import_redirect_phase(source: String, config: &Config) -> (String, Runtim /// /// `eager_names` names the *bindings* that must be bound to the real object where /// the import is written: what `except` catches, a special form, and a name an -/// annotation or a decorator reads, whose readers look them up in the module's -/// namespace or tell them apart by identity as the module is executing +/// annotation, a decorator or a forward reference written as a string reads, whose +/// readers look them up in the module's namespace or tell them apart by identity as +/// the module is executing. `quotes` are the type expressions inside the forward +/// references the lowering wrote as strings, recorded as it wrote them /// /// A stub defers nothing: it is never executed, and deferring its imports would /// hand a checker a call result where the stub declares a module or a class @@ -778,6 +804,7 @@ fn run_lazy_import_phase( config: &Config, eager: &[String], eager_names: &[String], + quotes: &[String], ) -> Result { if !config.lazy_imports { return Ok(source); @@ -804,6 +831,7 @@ fn run_lazy_import_phase( module.suite(), src, )); + eager_names.extend(transforms::lazy_import::forward_reference_names(quotes)); eager_names.extend(transforms::lazy_import::decorator_names(module.suite())); let deferral = if config.is_stub { diff --git a/crates/by_transforms/src/source_map.rs b/crates/by_transforms/src/source_map.rs index 40ef4fb861..03a85e61bb 100644 --- a/crates/by_transforms/src/source_map.rs +++ b/crates/by_transforms/src/source_map.rs @@ -65,13 +65,28 @@ impl Replacement { if start >= end { return; } + self.push_copied(&source[start..end], start); + } + + /// Append `text`, copied from the source at `start`. + fn push_copied(&mut self, text: &str, start: usize) { let continues = self.runs.last().is_some_and(|&(at, origin)| { matches!(origin, Origin::Copied(from) if from + (self.text.len() - at) == start) }); if !continues { self.runs.push((self.text.len(), Origin::Copied(start))); } - self.text.push_str(&source[start..end]); + self.text.push_str(text); + } + + /// Append `other`, run by run, so each keeps where it came from. + pub(crate) fn append(&mut self, other: &Replacement) { + for (text, origin) in other.runs() { + match origin { + Origin::Generated(anchor) => self.push_generated(text, anchor), + Origin::Copied(start) => self.push_copied(text, start), + } + } } /// The replacement text. diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index 77d1c63fce..77d759a584 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -8,6 +8,7 @@ use crate::Config; use crate::transforms::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::transforms::callable::CallableSyntax; use crate::transforms::optional_type; +use crate::transforms::quotes::WrittenQuotes; use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{ Recurse, RootKind, TypeExprVisitor, TypePos, callable_parameter_type, walk_type_positions, @@ -56,11 +57,12 @@ impl<'src> TupleLiteralType<'src> { fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: &Config, symbolic_substitutions: &'src [(TextRange, String)], ) -> Self { - let mut leaves = CallableSyntax::new(source, written, config).with_types(types); + let mut leaves = CallableSyntax::new(source, written, quotes, config).with_types(types); for (range, rendered) in symbolic_substitutions { leaves.add_substitution(*range, rendered.clone()); } @@ -408,14 +410,21 @@ impl TypeExprVisitor for TupleLiteralType<'_> { pub(crate) struct TupleLiteralTypePass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, } impl<'src> TupleLiteralTypePass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + config: Config, + ) -> Self { Self { source, written, + quotes, config, } } @@ -437,6 +446,7 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { let mut inner = TupleLiteralType::new( self.source, self.written, + self.quotes, types, &self.config, &symbolic_substitutions, diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index 1b6525637d..76c99f6252 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -43,6 +43,7 @@ use ty_python_semantic::AnonNamedTupleId; use crate::transforms::callable::CallableSyntax; use crate::transforms::generics::PolyfilledTypeParams; +use crate::transforms::quotes::WrittenQuotes; use crate::transforms::repeated_underscore::WrittenNames; use crate::type_info::{AnonNamedTupleConstruction, AnonNamedTupleField, TypeInfo}; @@ -98,6 +99,8 @@ impl Shape { pub(crate) struct AnonNamedTuple<'src> { source: &'src str, written: WrittenNames<'src>, + /// where a field type naming something the module binds is written as a string + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: crate::Config, /// One lowerer drives every field type in the run, so its imports and the @@ -171,6 +174,7 @@ impl<'src> AnonNamedTuple<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: crate::Config, stmts: &[Stmt], @@ -179,8 +183,9 @@ impl<'src> AnonNamedTuple<'src> { polyfilled: PolyfilledTypeParams::decide(source, written, stmts, &config), source, written, + quotes, types, - callable: CallableSyntax::new(source, written, &config).with_types(types), + callable: CallableSyntax::new(source, written, quotes, &config).with_types(types), config, edits: Vec::new(), shapes: indexmap::IndexMap::new(), @@ -231,7 +236,7 @@ impl<'src> AnonNamedTuple<'src> { // as one let forward = renamed != lowered || self.reads_a_module_binding(expr); if forward && !lowered.is_empty() { - super::auto_quote::quoted_text(&renamed) + self.quotes.quote(&renamed) } else { renamed } @@ -693,6 +698,7 @@ impl<'ast> Visitor<'ast> for AnonNamedTuple<'_> { pub(crate) struct AnonNamedTuplePass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: crate::Config, } @@ -700,11 +706,13 @@ impl<'src> AnonNamedTuplePass<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: crate::Config, ) -> Self { Self { source, written, + quotes, config, } } @@ -735,8 +743,14 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { types: &dyn TypeInfo, ctx: &mut super::ast_driver::PassContext, ) { - let mut inner = - AnonNamedTuple::new(self.source, self.written, types, self.config.clone(), stmts); + let mut inner = AnonNamedTuple::new( + self.source, + self.written, + self.quotes, + types, + self.config.clone(), + stmts, + ); inner.annotations_evaluated = ctx.annotations_evaluated; for stmt in stmts { inner.visit_stmt(stmt); diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 7889dd4870..8317b3d074 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -34,6 +34,7 @@ use ruff_python_parser::parse_unchecked_source; use ruff_source_file::LineEnding; use ruff_text_size::{Ranged, TextRange, TextSize}; +use super::quotes::WrittenQuotes; use super::source_util::preamble_offset; use super::{ annotation, anon_named_tuple, auto_quote, build_stamps, callable, character_type, checked_cast, @@ -82,6 +83,37 @@ pub(crate) enum SemanticSource<'p> { pub(crate) enum Fragment { Lit(String), Src(TextRange), + /// a passthrough written as a string, the forward reference python reads lazily: a + /// type expression, with whatever lowers inside it lowered too. what it lowered to is + /// recorded, so the names it reads are bound where they are imported. only + /// [`WrittenQuotes::quote_span`](super::quotes::WrittenQuotes::quote_span) makes one + Quote(super::quotes::QuotedSpan), +} + +impl Fragment { + /// the source span this fragment passes through, quoted or not + fn passthrough(&self) -> Option { + match self { + Fragment::Lit(_) => None, + Fragment::Src(span) => Some(*span), + Fragment::Quote(quoted) => Some(quoted.range()), + } + } +} + +/// a string delimiter `text` does not already use, to write it as a string literal of +/// python's that holds exactly `text` +fn quote_delimiter(text: &str) -> &'static str { + ["\"", "'", "\"\"\"", "'''"] + .into_iter() + .find(|delimiter| { + !text.contains(delimiter) + && delimiter + .chars() + .next() + .is_none_or(|quote| !text.ends_with(quote)) + }) + .unwrap_or("\"") } /// Mutable state shared across every pass during a single transpile. @@ -385,10 +417,7 @@ enum SubPatch { /// whether a template spanning `start..end` only wraps it: its one passthrough /// is the whole span, and everything else it emits is text around it fn is_wrapper(frags: &[Fragment], start: usize, end: usize) -> bool { - let mut passthroughs = frags.iter().filter_map(|frag| match frag { - Fragment::Src(range) => Some(range), - Fragment::Lit(_) => None, - }); + let mut passthroughs = frags.iter().filter_map(Fragment::passthrough); passthroughs .next() .is_some_and(|range| usize::from(range.start()) == start && usize::from(range.end()) == end) @@ -427,16 +456,12 @@ fn template_claimees( // a boundary insertion at a passthrough's end is left to // `apply_within`'s `include_end`, which knows whether an adjacent // span will re-emit it - let in_src = frags.iter().any(|frag| match frag { - Fragment::Lit(_) => false, - Fragment::Src(span) => { - let (span_start, span_end) = - (usize::from(span.start()), usize::from(span.end())); - s >= span_start - && e <= span_end - && !(matches!(all[m].2, SubPatch::Relocating(_)) - && (s, e) == (span_start, span_end)) - } + let in_src = frags.iter().filter_map(Fragment::passthrough).any(|span| { + let (span_start, span_end) = (usize::from(span.start()), usize::from(span.end())); + s >= span_start + && e <= span_end + && !(matches!(all[m].2, SubPatch::Relocating(_)) + && (s, e) == (span_start, span_end)) }); in_own || in_src }) @@ -463,6 +488,8 @@ fn rewrites_covered_source(range: TextRange, frags: &[Fragment], source: &str) - for frag in frags { let piece = match frag { Fragment::Lit(text) => text.as_str(), + // a quote writes delimiters no source has there + Fragment::Quote(_) => return true, Fragment::Src(span) => { match source.get(usize::from(span.start())..usize::from(span.end())) { Some(text) => text, @@ -486,6 +513,7 @@ fn rewrites_covered_source(range: TextRange, frags: &[Fragment], source: &str) - /// is charged to that offset in the line table: a hoisted `def` header, an /// injected keyword argument, a `nonlocal` line — none of them has a line of /// its own, and the construct the edit rewrites is the one they stand for. +#[expect(clippy::too_many_arguments)] fn materialize_fragments( out: &mut Replacement, frags: &[Fragment], @@ -494,10 +522,31 @@ fn materialize_fragments( contained: &[usize], anchor: usize, emitted: &mut [bool], + record_quote: &dyn Fn(&str), ) { for (i, frag) in frags.iter().enumerate() { match frag { Fragment::Lit(s) => out.push_generated(s, anchor), + Fragment::Quote(quoted_span) => { + let span = quoted_span.range(); + let mut quoted = Replacement::default(); + apply_within( + &mut quoted, + source, + usize::from(span.start()), + usize::from(span.end()), + all, + contained, + true, + emitted, + record_quote, + ); + record_quote(quoted.text()); + let delimiter = quote_delimiter(quoted.text()); + out.push_generated(delimiter, anchor); + out.append("ed); + out.push_generated(delimiter, anchor); + } Fragment::Src(span) => { // a zero-width insertion at this span's end is normally deferred // to the *next* `Src` span (which re-emits it at its start), so @@ -520,6 +569,7 @@ fn materialize_fragments( contained, include_end, emitted, + record_quote, ); } } @@ -545,6 +595,7 @@ fn apply_within( contained: &[usize], include_end: bool, emitted: &mut [bool], + record_quote: &dyn Fn(&str), ) { let mut cursor = s0; let mut k = 0; @@ -570,7 +621,7 @@ fn apply_within( .copied() .filter(|&m| all[m].0 >= s && all[m].1 <= e && all[m].0 != e) .collect(); - materialize_fragments(out, frags, source, all, &inner, s, emitted); + materialize_fragments(out, frags, source, all, &inner, s, emitted, record_quote); } } cursor = cursor.max(e); @@ -594,6 +645,7 @@ fn splice( sub_edits: Vec<(usize, usize, SubPatch)>, origins: Vec>, authors: &[Author], + record_quote: &dyn Fn(&str), ) -> (Vec<(usize, usize, Replacement)>, Vec) { let mut edits: Vec<(usize, usize, Replacement)> = Vec::new(); let mut emitted = vec![false; sub_edits.len()]; @@ -731,6 +783,7 @@ fn splice( &contained, start, &mut emitted, + record_quote, ); } } @@ -762,6 +815,7 @@ fn splice( &contained, start, &mut emitted, + record_quote, ); out } @@ -1181,6 +1235,7 @@ fn merge_from_imports(lines: Vec) -> (Vec, Vec) { pub(crate) fn run_against_source<'a>( source: &'a str, written: repeated_underscore::WrittenNames, + quotes: &WrittenQuotes, config: &Config, project: SemanticSource<'_>, ) -> (Cow<'a, str>, Vec, Vec>) { @@ -1373,9 +1428,14 @@ pub(crate) fn run_against_source<'a>( ); let unique_loop_bindings_pass = unique_loop_bindings::UniqueLoopBindingsPass::new(source_ref, config.unique_loop_bindings); - let auto_quote_pass = auto_quote::AutoQuote::new(source_ref); - let init_method_pass = - init_method::InitMethod::new(source_ref, written, &default_markers, config.clone()); + let auto_quote_pass = auto_quote::AutoQuote { quotes }; + let init_method_pass = init_method::InitMethod::new( + source_ref, + written, + quotes, + &default_markers, + config.clone(), + ); let properties_pass = properties::PropertiesPass::new( source_ref, written, @@ -1415,6 +1475,7 @@ pub(crate) fn run_against_source<'a>( let typed_dict_literal_pass = typed_dict_literal::TypedDictLiteralPass::new( source_ref, written, + quotes, config.clone(), parsed_handle.suite(), &semantic_model, @@ -1441,12 +1502,13 @@ pub(crate) fn run_against_source<'a>( ); let template_type_pass = template_type::TemplateTypePass { written }; let tuple_types_pass = - annotation::TupleLiteralTypePass::new(source_ref, written, config.clone()); + annotation::TupleLiteralTypePass::new(source_ref, written, quotes, config.clone()); let literal_types_pass = literal_types::LiteralTypePass::new(source_ref, written, config.float_literals); - let callable_pass = callable::CallableSyntaxPass::new(source_ref, written, config.clone()); + let callable_pass = + callable::CallableSyntaxPass::new(source_ref, written, quotes, config.clone()); let protocol_type_pass = - protocol_type::ProtocolTypePass::new(source_ref, written, config.clone()); + protocol_type::ProtocolTypePass::new(source_ref, written, quotes, config.clone()); let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref, &comments); let force_unwrap_pass = force_unwrap::ForceUnwrapPass::new(source_ref); let flexible_keyword_pass = flexible_keyword::FlexibleKeywordPass; @@ -1456,12 +1518,10 @@ pub(crate) fn run_against_source<'a>( let optional_type_pass = optional_type::OptionalTypePass::new(source_ref, written, config.min_version); let runtime_union_pass = runtime_union::RuntimeUnionPass::new(written, config.min_version); - let generics_pass = generics::GenericPolyfillPass::new(source_ref, written, config.clone()); + let generics_pass = + generics::GenericPolyfillPass::new(source_ref, written, quotes, config.clone()); let soundness_pass = soundness::SoundnessPass::new(source_ref, written, config); - let checked_cast_pass = checked_cast::CheckedCastPass { - source: source_ref, - written, - }; + let checked_cast_pass = checked_cast::CheckedCastPass { written, quotes }; let module_api_pass = module_api::ModuleApiPass::new(source_ref); let trailing_lambda_pass = trailing_lambda::TrailingLambdaPass::new(source_ref); let if_let_pass = if_let::IfLetPass::new(source_ref); @@ -1479,7 +1539,7 @@ pub(crate) fn run_against_source<'a>( let frameworks_pass = frameworks::FrameworksPass::new(source_ref); let variance_pass = decl_site_variance::VarianceStripPass::new(source_ref); let anon_named_tuple_pass = - anon_named_tuple::AnonNamedTuplePass::new(source_ref, written, config.clone()); + anon_named_tuple::AnonNamedTuplePass::new(source_ref, written, quotes, config.clone()); // Order matters: passes that read source ranges via `text_edits` mode // must run BEFORE passes that mutate the AST (which zero source ranges @@ -2139,7 +2199,13 @@ pub(crate) fn run_against_source<'a>( ) })) .collect(); - let (spliced, lost) = splice(source_ref, sub_edits, origins, &authorship.authors); + let (spliced, lost) = splice( + source_ref, + sub_edits, + origins, + &authorship.authors, + &|quoted| quotes.record(quoted), + ); edits.extend(spliced); ctx.errors.extend(lost); // line table for the spliced body, built from the ascending edit list @@ -2276,7 +2342,7 @@ mod driver_tests { SubPatch::Text(edit.1.to_owned()), )); let origins = vec![Vec::new(); sub_edits.len()]; - splice(source, sub_edits, origins, &[]).1 + splice(source, sub_edits, origins, &[], &|_| {}).1 } /// the call of `stmt`, an assignment of one @@ -2379,6 +2445,7 @@ mod driver_tests { sub_edits, vec![vec![0], vec![1]], &[writer, inserter], + &|_| {}, ) .1 } @@ -2428,7 +2495,7 @@ mod driver_tests { (6, 7, SubPatch::Text(" | None".to_owned())), ]; let authors = [author(None, &[]), author(Some(Lowering::OptionalType), &[])]; - let errors = splice(source, sub_edits, vec![vec![0], vec![1]], &authors).1; + let errors = splice(source, sub_edits, vec![vec![0], vec![1]], &authors, &|_| {}).1; assert_eq!(errors, Vec::::new()); } @@ -2438,6 +2505,7 @@ mod driver_tests { let (out, _, _) = run_against_source( src, repeated_underscore::WrittenNames::new(src), + &WrittenQuotes::default(), &Config::test_default(), SemanticSource::Fresh(ruff_python_ast::PythonVersion::latest_ty()), ); @@ -2480,7 +2548,16 @@ mod driver_tests { Fragment::Lit("Y".to_owned()), ]; let mut out = Replacement::default(); - materialize_fragments(&mut out, &frags, source, &all, &[0], 0, &mut [false]); + materialize_fragments( + &mut out, + &frags, + source, + &all, + &[0], + 0, + &mut [false], + &|_| {}, + ); assert_eq!(out.text(), "[1])Y"); } @@ -2496,7 +2573,16 @@ mod driver_tests { Fragment::Src(TextRange::new(TextSize::from(3u32), TextSize::from(4u32))), ]; let mut out = Replacement::default(); - materialize_fragments(&mut out, &frags, source, &all, &[0], 0, &mut [false]); + materialize_fragments( + &mut out, + &frags, + source, + &all, + &[0], + 0, + &mut [false], + &|_| {}, + ); assert_eq!(out.text(), "[1])W"); } } diff --git a/crates/by_transforms/src/transforms/auto_quote.rs b/crates/by_transforms/src/transforms/auto_quote.rs index 06a360e40d..820fb5161d 100644 --- a/crates/by_transforms/src/transforms/auto_quote.rs +++ b/crates/by_transforms/src/transforms/auto_quote.rs @@ -48,17 +48,12 @@ use ruff_python_ast::{AnyParameterRef, Expr, ExprName, Stmt, TypeParams}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; +use super::quotes::WrittenQuotes; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_one_type_expr_with}; use crate::type_info::TypeInfo; pub(crate) struct AutoQuote<'src> { - source: &'src str, -} - -impl<'src> AutoQuote<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } - } + pub(crate) quotes: &'src WrittenQuotes, } impl TypeAwarePass for AutoQuote<'_> { @@ -70,14 +65,14 @@ impl TypeAwarePass for AutoQuote<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut walk = Walk { - source: self.source, + quotes: self.quotes, types, quote_annotations: ctx.annotations_evaluated, edits: Vec::new(), }; walk.block(stmts, Scope::Module); let mut deferred = DeferredReads { - source: self.source, + quotes: self.quotes, types, edits: walk.edits, }; @@ -97,7 +92,7 @@ enum Scope { } struct Walk<'a> { - source: &'a str, + quotes: &'a WrittenQuotes, types: &'a dyn TypeInfo, quote_annotations: bool, edits: Vec<(TextRange, Vec)>, @@ -170,7 +165,7 @@ impl Walk<'_> { let types = self.types; let is_forward = |name: &ExprName| types.is_forward_reference(name) == Some(true); let mut quoter = Quoter { - source: self.source, + quotes: self.quotes, is_forward: &is_forward, edits: &mut self.edits, skip_root: false, @@ -189,7 +184,7 @@ impl Walk<'_> { /// written — a base itself, a reified class's type arguments, the type an `is` tests for — is /// left alone, and a name in it that is not yet bound stays the error the checker reports struct DeferredReads<'a> { - source: &'a str, + quotes: &'a WrittenQuotes, types: &'a dyn TypeInfo, /// every quote written so far, the annotations' included: nothing inside one is evaluated /// where it stands any more @@ -210,7 +205,7 @@ impl DeferredReads<'_> { let types = self.types; let is_forward = |name: &ExprName| types.is_forward_reference(name) == Some(true); let mut quoter = Quoter { - source: self.source, + quotes: self.quotes, is_forward: &is_forward, edits: &mut self.edits, skip_root, @@ -270,7 +265,7 @@ impl<'a> Visitor<'a> for DeferredReads<'_> { /// quotes the forward references in one type expression struct Quoter<'a> { - source: &'a str, + quotes: &'a WrittenQuotes, is_forward: &'a dyn Fn(&ExprName) -> bool, edits: &'a mut Vec<(TextRange, Vec)>, /// the root is the specialization whose type arguments are deferred, which is @@ -322,42 +317,11 @@ impl Quoter<'_> { } fn quote(&mut self, range: TextRange) { - self.edits.push((range, quoted(self.source, range))); + self.edits + .push((range, vec![self.quotes.quote_span(range)])); } } -/// `range` of `source` wrapped in quotes, as one template passing the source through so -/// the lowerings inside it land between the quotes. the delimiter is one the source it -/// wraps does not already use -pub(crate) fn quoted(source: &str, range: TextRange) -> Vec { - let delimiter = delimiter_for(&source[range]); - vec![ - Fragment::Lit(delimiter.to_owned()), - Fragment::Src(range), - Fragment::Lit(delimiter.to_owned()), - ] -} - -/// `text`, a lowered type expression, written as the forward reference python reads lazily -pub(crate) fn quoted_text(text: &str) -> String { - let delimiter = delimiter_for(text); - format!("{delimiter}{text}{delimiter}") -} - -/// a string delimiter `text` does not already use -fn delimiter_for(text: &str) -> &'static str { - ["\"", "'", "\"\"\"", "'''"] - .into_iter() - .find(|delimiter| { - !text.contains(delimiter) - && delimiter - .chars() - .next() - .is_none_or(|quote| !text.ends_with(quote)) - }) - .unwrap_or("\"") -} - #[cfg(test)] mod tests { use crate::config::PythonVersion; diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index c94faf4cfc..80cd6ea3a6 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -33,6 +33,7 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Lowering, PassContext, TypeAwarePass}; use super::intersection::{collect_intersect, collect_union, is_intersection_node}; use super::just_float::rewrite_type_expr_with_imports; +use super::quotes::WrittenQuotes; use super::repeated_underscore::{WrittenNames, lowered_callable_parameters}; use super::type_expr_walker::RootKind; use crate::config::{Config, FloatLiteralLowering}; @@ -59,6 +60,8 @@ pub(crate) struct CallableSyntax<'src> { source: &'src str, /// the module as written, whose names a repeated `_` is numbered around written: WrittenNames<'src>, + /// where the synthesized protocols' member types are written as strings + quotes: &'src WrittenQuotes, types: Option<&'src (dyn TypeInfo + 'src)>, /// ranges already folded by `symbolic_type_op` (e.g. `1 + typeof d` → /// `Literal[3]`). a claimed sub-expression is opaque here: descending into @@ -115,10 +118,16 @@ struct ProtocolShape { } impl<'src> CallableSyntax<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: &Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + config: &Config, + ) -> Self { Self { source, written, + quotes, types: None, claimed_ranges: &[], owned_typeof: None, @@ -516,7 +525,7 @@ impl<'src> CallableSyntax<'src> { }, _ => name.to_string(), }; - let ty = quote_forward_ref(&self.rewrite_or_leaf(&named.value)); + let ty = self.quotes.quote(&self.rewrite_or_leaf(&named.value)); parts.push(format!("{name}: {ty}")); } Expr::Starred(s) => match s.value.as_ref() { @@ -526,7 +535,7 @@ impl<'src> CallableSyntax<'src> { // members are emitted one by one match self.unpacked_kwargs(&inner.value) { Some(UnpackedKwargsLowering::TypedDict) => { - let ty = quote_forward_ref(&format!( + let ty = self.quotes.quote(&format!( "{}[{}]", self.written.imported("typing", "Unpack"), self.rewrite_or_leaf(&inner.value) @@ -544,11 +553,11 @@ impl<'src> CallableSyntax<'src> { } for (name, member_ty) in members { parts - .push(format!("{name}: {}", quote_forward_ref(&member_ty))); + .push(format!("{name}: {}", self.quotes.quote(&member_ty))); } } Some(UnpackedKwargsLowering::ParameterPack) | None => { - let ty = quote_forward_ref(&self.rewrite_or_leaf(&inner.value)); + let ty = self.quotes.quote(&self.rewrite_or_leaf(&inner.value)); parts.push(format!("**kwargs: {ty}")); } } @@ -561,12 +570,13 @@ impl<'src> CallableSyntax<'src> { let unpacks = self.is_unpacked_variadic(&s.value); let inner = self.rewrite_or_leaf(&s.value); let ty = - quote_forward_ref(&if unpacks { format!("*{inner}") } else { inner }); + self.quotes + .quote(&if unpacks { format!("*{inner}") } else { inner }); parts.push(format!("*args: {ty}")); } }, _ => { - let ty = quote_forward_ref(&self.rewrite_or_leaf(arg)); + let ty = self.quotes.quote(&self.rewrite_or_leaf(arg)); parts.push(format!("{}: {ty}", names[i])); } } @@ -648,11 +658,11 @@ impl<'src> CallableSyntax<'src> { format!( "{}: {}", receiver_parameter_name(ct), - quote_forward_ref(&rendered) + self.quotes.quote(&rendered) ) }); let params = self.render_protocol_params(ct, false, "self", implicit_receiver); - let returns = quote_forward_ref(&self.render_return(ct)); + let returns = self.quotes.quote(&self.render_return(ct)); let shape = ProtocolShape { params, returns }; Some(self.class_name_for(shape)) } @@ -899,13 +909,6 @@ impl<'src> CallableSyntax<'src> { } } -/// render a synthesized annotation as a forward-reference string literal so it -/// is never evaluated at class-body time (the hoisted protocol class precedes -/// the definitions its annotations mention) -fn quote_forward_ref(ty: &str) -> String { - format!("\"{}\"", ty.replace('\\', "\\\\").replace('"', "\\\"")) -} - /// The synthesized name of a receiver parameter in a protocol `__call__`. The /// receiver is unnamed in the surface syntax, so any name works — but a callable /// may itself declare a parameter called `_receiver`, and two parameters of the @@ -965,16 +968,18 @@ fn is_named(expr: &Expr, ident: &str) -> bool { /// rendering path, which is what lets one wide edit carry rewrites it subsumes. /// /// `root` says where the text is written, which decides how a union in it is spelled +#[expect(clippy::too_many_arguments)] pub(crate) fn lower_type_expr_full( source: &str, written: WrittenNames, + quotes: &WrittenQuotes, types: &dyn TypeInfo, expr: &Expr, substitutions: &[(TextRange, String)], config: &Config, root: RootKind, ) -> Option { - let mut inner = CallableSyntax::new(source, written, config).with_types(types); + let mut inner = CallableSyntax::new(source, written, quotes, config).with_types(types); inner.set_evaluated(root == RootKind::Evaluated); for (range, name) in substitutions { inner.add_substitution(*range, name.clone()); @@ -1091,14 +1096,21 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> pub(crate) struct CallableSyntaxPass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, } impl<'src> CallableSyntaxPass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + config: Config, + ) -> Self { Self { source, written, + quotes, config, } } @@ -1167,7 +1179,7 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { // `required_imports` / `edits` uses below let claimed = ctx.claimed_type_op_ranges.clone(); let structural_typeof = ctx.structural_typeof_ranges.clone(); - let mut inner = CallableSyntax::new(self.source, self.written, &self.config) + let mut inner = CallableSyntax::new(self.source, self.written, self.quotes, &self.config) .with_types(types) .with_claimed_ranges(&claimed) .with_owned_typeof(&structural_typeof); diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 0b10c04e38..513d345aee 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -51,6 +51,7 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::parametric_is::{PredicateRuntime, ProbeStrictness, build_predicate}; +use super::quotes::WrittenQuotes; use super::repeated_underscore::WrittenNames; use crate::type_info::{CastCheck, SoundnessCheck, TypeInfo}; @@ -115,9 +116,11 @@ impl Helper { } struct CastLower<'a> { - source: &'a str, types: &'a dyn TypeInfo, written: WrittenNames<'a>, + /// where a `typing.cast` target that names something not bound there is written as a + /// string + quotes: &'a WrittenQuotes, /// the name `typing.cast` is written under cast: String, edits: Vec<(TextRange, Vec)>, @@ -127,11 +130,11 @@ struct CastLower<'a> { } impl<'a> CastLower<'a> { - fn new(source: &'a str, types: &'a dyn TypeInfo, written: WrittenNames<'a>) -> Self { + fn new(types: &'a dyn TypeInfo, written: WrittenNames<'a>, quotes: &'a WrittenQuotes) -> Self { Self { - source, types, written, + quotes, cast: written.imported("typing", "cast"), edits: Vec::new(), used: BTreeSet::new(), @@ -204,7 +207,7 @@ impl<'a> CastLower<'a> { let first = if self.types.reads_names_bound_where_written(type_arg) { vec![Fragment::Src(type_arg.range())] } else { - super::auto_quote::quoted(self.source, type_arg.range()) + vec![self.quotes.quote_span(type_arg.range())] }; (helper, first, vec![Fragment::Src(value_range)]) } else { @@ -254,8 +257,8 @@ impl<'ast> Visitor<'ast> for CastLower<'_> { } pub(crate) struct CheckedCastPass<'src> { - pub(crate) source: &'src str, pub(crate) written: WrittenNames<'src>, + pub(crate) quotes: &'src WrittenQuotes, } impl TypeAwarePass for CheckedCastPass<'_> { @@ -283,7 +286,7 @@ impl TypeAwarePass for CheckedCastPass<'_> { } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = CastLower::new(self.source, types, self.written); + let mut inner = CastLower::new(types, self.written, self.quotes); for stmt in stmts { inner.visit_stmt(stmt); } diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 594375bc84..6be0dc104f 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -8,6 +8,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::config::Config; use crate::transforms::callable::lower_type_expr_full; +use crate::transforms::quotes::WrittenQuotes; use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::RootKind; use crate::type_info::TypeInfo; @@ -21,6 +22,8 @@ use ruff_python_ast::PythonVersion; pub(crate) struct GenericPolyfill<'src> { source: &'src str, written: WrittenNames<'src>, + /// where a bound read through the classes it stands in is written as a string + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: Config, edits: Vec, @@ -208,9 +211,11 @@ struct ProcessedTypeParams { } impl<'src> GenericPolyfill<'src> { + #[expect(clippy::too_many_arguments)] fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: Config, symbolic_substitutions: Vec<(TextRange, String)>, @@ -221,6 +226,7 @@ impl<'src> GenericPolyfill<'src> { Self { source, written, + quotes, types, config, edits: Vec::new(), @@ -427,6 +433,7 @@ impl<'src> GenericPolyfill<'src> { let rendered = lower_type_expr_full( self.source, self.written, + self.quotes, self.types, default, &self.subsume_within(default.range()), @@ -483,12 +490,13 @@ impl<'src> GenericPolyfill<'src> { ) -> ProcessedTypeParams { // a bound that reads a class's name through the class is written ahead of the class, // so it is a string, which `TypeVar` keeps as a forward reference + let quotes = self.quotes; let through_classes = |text: String| { let read_through = apply_renames_to_rendered(&text, qualified); if read_through == text { text } else { - super::source_util::python_string_literal(&read_through) + quotes.quote(&read_through) } }; let mut generic_args: Vec = Vec::new(); @@ -579,6 +587,7 @@ impl<'src> GenericPolyfill<'src> { lower_type_expr_full( self.source, self.written, + self.quotes, self.types, bound, &self.subsume_within(bound.range()), @@ -1086,6 +1095,7 @@ impl<'src> GenericPolyfill<'src> { let value_src = lower_type_expr_full( self.source, self.written, + self.quotes, self.types, &alias.value, &substitutions, @@ -2054,14 +2064,21 @@ fn names_read(expr: &Expr) -> Vec<&str> { pub(crate) struct GenericPolyfillPass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, } impl<'src> GenericPolyfillPass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + config: Config, + ) -> Self { Self { source, written, + quotes, config, } } @@ -2097,6 +2114,7 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { let mut inner = GenericPolyfill::new( self.source, self.written, + self.quotes, types, self.config.clone(), ctx.symbolic_substitutions.clone(), diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index 6f03e7ad62..fca51952b4 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -34,6 +34,7 @@ use super::callable::lower_type_expr_full; use super::mutable_defaults::{ DefaultMarkers, define_markers, parameter_guards, undeclarable_error, }; +use super::quotes::WrittenQuotes; use super::repeated_underscore::WrittenNames; use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; use super::type_expr_walker::RootKind; @@ -44,6 +45,7 @@ use crate::type_info::TypeInfo; pub(crate) struct InitMethod<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, markers: &'src DefaultMarkers, config: Config, } @@ -52,12 +54,14 @@ impl<'src> InitMethod<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, markers: &'src DefaultMarkers, config: Config, ) -> Self { Self { source, written, + quotes, markers, config, } @@ -69,6 +73,7 @@ impl TypeAwarePass for InitMethod<'_> { let mut state = State { source: self.source, written: self.written, + quotes: self.quotes, types, symbolic_substitutions: ctx.symbolic_substitutions.clone(), module_private: types.private_module_symbols().into_iter().collect(), @@ -131,7 +136,7 @@ fn literal_text(frags: &[Fragment]) -> String { .iter() .map(|frag| match frag { Fragment::Lit(lit) => lit.as_str(), - Fragment::Src(_) => "", + Fragment::Src(_) | Fragment::Quote(_) => "", }) .collect() } @@ -139,6 +144,8 @@ fn literal_text(frags: &[Fragment]) -> String { struct State<'src> { source: &'src str, written: WrittenNames<'src>, + /// where a synthesized protocol's member types are written as strings + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, /// `(range, rendered)` for every symbolic fold in the module. the `__init__` /// line is fresh output, so a fold inside a parameter annotation is dropped @@ -234,6 +241,7 @@ impl State<'_> { lower_type_expr_full( self.source, self.written, + self.quotes, self.types, ann, &substitutions, diff --git a/crates/by_transforms/src/transforms/lazy_import.rs b/crates/by_transforms/src/transforms/lazy_import.rs index 7cff073d82..1d1bf452d4 100644 --- a/crates/by_transforms/src/transforms/lazy_import.rs +++ b/crates/by_transforms/src/transforms/lazy_import.rs @@ -548,6 +548,33 @@ pub(crate) fn annotation_names(suite: &[Stmt], source: &str) -> Vec { names.read } +/// every name the type expressions in `quotes` read: the forward references the lowering +/// wrote as strings, each by the text inside its quotes +/// +/// python evaluates such a string against the module's namespace whenever something asks for +/// the type, so a name in it is bound where it is imported. the lowering records each quote as +/// it writes it, with whatever lowered inside it lowered, so a name a lowering wrote into one — +/// a hoisted class, a `typing` form — is read as surely as one the source spelled. a string +/// inside the type, a literal's, is a value and names nothing +pub(crate) fn forward_reference_names(quotes: &[String]) -> Vec { + struct Names(Vec); + impl<'a> Visitor<'a> for Names { + fn visit_expr(&mut self, expr: &'a ruff_python_ast::Expr) { + if let ruff_python_ast::Expr::Name(name) = expr { + self.0.push(name.id.to_string()); + } + walk_expr(self, expr); + } + } + let mut names = Names(Vec::new()); + for quote in quotes { + if let Ok(parsed) = ruff_python_parser::parse_expression(quote) { + names.visit_expr(parsed.expr()); + } + } + names.0 +} + /// The names every decorator in `suite` reads, which are bound eagerly. /// /// A decorator is the other thing read by identity rather than by value. `typing.overload` @@ -1557,6 +1584,25 @@ mod tests { ); } + /// every name a recorded quote reads is collected, and a string inside one is a value + #[test] + fn forward_reference_names_reads_every_name_a_quote_reads() { + let quotes = [ + "A".to_owned(), + "B | None".to_owned(), + "Callable[[C], D]".to_owned(), + "tuple[E, Literal[\"NotAName\"]]".to_owned(), + "not an expression(".to_owned(), + ]; + let mut names = super::forward_reference_names("es); + names.sort(); + names.dedup(); + assert_eq!( + names, + ["A", "B", "C", "Callable", "D", "E", "Literal", "tuple"] + ); + } + #[test] fn annotation_names_reads_the_annotations_python_keeps() { use ruff_python_parser::parse_module; diff --git a/crates/by_transforms/src/transforms/mod.rs b/crates/by_transforms/src/transforms/mod.rs index b0b8323674..ceb26d7734 100644 --- a/crates/by_transforms/src/transforms/mod.rs +++ b/crates/by_transforms/src/transforms/mod.rs @@ -64,6 +64,7 @@ pub(crate) mod propagate; pub(crate) mod properties; pub(crate) mod protocol_type; pub(crate) mod python_scopes; +pub(crate) mod quotes; pub(crate) mod raises_clause; pub(crate) mod reified_class; pub(crate) mod reified_generic; diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index 4c2e0c3d29..30d1d7710a 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -43,6 +43,7 @@ use crate::type_info::TypeInfo; use super::ast_driver::{PassContext, TypeAwarePass}; use super::callable::CallableSyntax; +use super::quotes::WrittenQuotes; use super::repeated_underscore::{WrittenNames, protocol_method_receiver}; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping}; @@ -78,6 +79,7 @@ impl Shape { pub(crate) struct ProtocolTypePass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: crate::Config, } @@ -85,11 +87,13 @@ impl<'src> ProtocolTypePass<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: crate::Config, ) -> Self { Self { source, written, + quotes, config, } } @@ -98,6 +102,8 @@ impl<'src> ProtocolTypePass<'src> { struct ProtocolTypeLowering<'src> { source: &'src str, written: WrittenNames<'src>, + /// where the member types are written as strings + quotes: &'src WrittenQuotes, /// One lowerer drives every member type in the run, so its imports and the /// `_Callable_*` classes it synthesizes can be collected at the end. The /// `callable` pass deliberately does not descend into an inline protocol, @@ -119,6 +125,7 @@ impl<'src> ProtocolTypeLowering<'src> { fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, claimed: &'src [TextRange], config: &Config, @@ -126,7 +133,8 @@ impl<'src> ProtocolTypeLowering<'src> { Self { source, written, - callable: CallableSyntax::new(source, written, config) + quotes, + callable: CallableSyntax::new(source, written, quotes, config) .with_types(types) .with_claimed_ranges(claimed), edits: Vec::new(), @@ -192,7 +200,7 @@ impl<'src> ProtocolTypeLowering<'src> { let line = format!( "def {name}({params}) -> {returns}: ...", name = method.name.id, - returns = quote_forward_ref(&returns), + returns = self.quotes.quote(&returns), ); Some(self.apply_typevar_renames(method.range(), &line)) } @@ -205,7 +213,7 @@ impl<'src> ProtocolTypeLowering<'src> { let Some(name) = named.target.as_name_expr() else { continue; }; - let ty = quote_forward_ref(&self.render_type(&named.value)); + let ty = self.quotes.quote(&self.render_type(&named.value)); // a member that only has to be readable is a getter-only property, // which is how a protocol class body says read-only members.push(if named.is_let { @@ -318,11 +326,6 @@ impl TypeExprVisitor for ProtocolTypeLowering<'_> { } } -/// Wrap a rendered member type in a forward-reference string. -fn quote_forward_ref(ty: &str) -> String { - format!("\"{}\"", ty.replace('\\', "\\\\").replace('"', "\\\"")) -} - /// Lowers inline protocols that appear in a value position (`x = protocol(a: /// int)`). `protocol(...)` is never valid python wherever it appears, so it has /// to be replaced everywhere, not just in the annotation slots the type-position @@ -376,12 +379,13 @@ impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for TypevarScopeWalker<'_> { fn lower<'src>( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, claimed: &'src [TextRange], stmts: &[Stmt], config: &crate::Config, ) -> ProtocolTypeLowering<'src> { - let mut inner = ProtocolTypeLowering::new(source, written, types, claimed, config); + let mut inner = ProtocolTypeLowering::new(source, written, quotes, types, claimed, config); let polyfilled = super::generics::PolyfilledTypeParams::decide(source, written, stmts, config); { let mut walker = TypevarScopeWalker { @@ -410,11 +414,12 @@ fn lower<'src>( pub(crate) fn cleanup( source: &str, written: WrittenNames, + quotes: &WrittenQuotes, types: &dyn TypeInfo, stmts: &[Stmt], config: &crate::Config, ) -> Result, String)>, String> { - let mut inner = lower(source, written, types, &[], stmts, config); + let mut inner = lower(source, written, quotes, types, &[], stmts, config); if let Some(error) = inner.errors.first() { return Err(error.clone()); } @@ -479,6 +484,7 @@ impl TypeAwarePass for ProtocolTypePass<'_> { let mut inner = lower( self.source, self.written, + self.quotes, types, &claimed, stmts, diff --git a/crates/by_transforms/src/transforms/quotes.rs b/crates/by_transforms/src/transforms/quotes.rs new file mode 100644 index 0000000000..715d197b6f --- /dev/null +++ b/crates/by_transforms/src/transforms/quotes.rs @@ -0,0 +1,63 @@ +//! the forward references the transpiler writes as strings +//! +//! python evaluates such a string against the module's namespace whenever something asks for +//! the type, so the lazy-import polyfill binds every name one reads where the name is +//! imported. writing a quote is a capability: only a holder of [`WrittenQuotes`] can write +//! one, as text ([`WrittenQuotes::quote`]) or as a passthrough of source the driver lowers in +//! place ([`WrittenQuotes::quote_span`]), and each is recorded as it is written. code that only +//! picks names never holds one + +use std::cell::RefCell; + +use ruff_text_size::TextRange; + +use super::ast_driver::Fragment; +use super::source_util::python_string_literal; + +/// every forward reference the module's lowerings write as a string, by the type expression +/// inside the quotes +#[derive(Debug, Default)] +pub(crate) struct WrittenQuotes { + texts: RefCell>, +} + +impl WrittenQuotes { + /// `ty`, a lowered type expression, written as the forward reference python reads lazily + pub(crate) fn quote(&self, ty: &str) -> String { + self.record(ty); + python_string_literal(ty) + } + + /// `range` written as a string, with whatever lowers inside it lowered too. the driver + /// records what it lowered to once it has written it + #[expect( + clippy::unused_self, + reason = "taking the registry is what makes writing a quote a capability of its holder" + )] + pub(crate) fn quote_span(&self, range: TextRange) -> Fragment { + Fragment::Quote(QuotedSpan { range }) + } + + /// record `ty`, the type expression a quote the driver wrote holds + pub(super) fn record(&self, ty: &str) { + self.texts.borrow_mut().push(ty.to_owned()); + } + + /// the type expressions the recorded quotes hold + pub(crate) fn texts(&self) -> Vec { + self.texts.borrow().clone() + } +} + +/// the source a [`Fragment::Quote`] writes as a string, which only +/// [`WrittenQuotes::quote_span`] makes +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct QuotedSpan { + range: TextRange, +} + +impl QuotedSpan { + pub(crate) fn range(self) -> TextRange { + self.range + } +} diff --git a/crates/by_transforms/src/transforms/rerender.rs b/crates/by_transforms/src/transforms/rerender.rs index fff310b77c..c4e47db861 100644 --- a/crates/by_transforms/src/transforms/rerender.rs +++ b/crates/by_transforms/src/transforms/rerender.rs @@ -2018,6 +2018,7 @@ mod tests { .map(|fragment| match fragment { Fragment::Lit(text) => text.clone(), Fragment::Src(range) => source[*range].to_string(), + Fragment::Quote(quoted) => source[quoted.range()].to_string(), }) .collect() }) @@ -2071,6 +2072,7 @@ mod tests { .map(|fragment| match fragment { Fragment::Lit(text) => text.clone(), Fragment::Src(range) => source[*range].to_string(), + Fragment::Quote(quoted) => source[quoted.range()].to_string(), }) .collect() }) diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index 121db15bd7..0c26b544a7 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -36,13 +36,11 @@ pub(crate) fn standalone_expression(source: &str, range: TextRange) -> Vec String { - standalone_expression(source, range) - .iter() - .map(|fragment| match fragment { - Fragment::Lit(text) => text.as_str(), - Fragment::Src(span) => &source[*span], - }) - .collect() + if source[range].contains(['\n', '\r']) { + format!("({})", &source[range]) + } else { + source[range].to_owned() + } } /// Whether `expr`, written as it is, has to be parenthesized to stand as one argument diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index c388373d6b..6a009bad82 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -27,7 +27,7 @@ //! b: _TypedDict_ //! ``` //! -//! field types are emitted as forward references — see [`quote_type`] +//! field types are emitted as forward references — see `Shape::class_def` use std::collections::BTreeSet; use std::fmt::Write as _; @@ -43,6 +43,7 @@ use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_posit use super::ast_driver::{AstPass, PassContext}; use super::callable::CallableSyntax; +use super::quotes::WrittenQuotes; use super::repeated_underscore::WrittenNames; use crate::config::Config; use crate::type_info::TypeInfo; @@ -66,9 +67,15 @@ impl Shape { /// the class declaring this shape as `name`, `typing_extensions.TypedDict` written as /// `typed_dict` - fn class_def(&self, name: &str, typed_dict: &str) -> String { + /// + /// every field type is written as a string. the hoisted classes are emitted in the + /// preamble, ahead of everything the module defines, and a shape can name a class declared + /// later or a type parameter that only exists inside the generic class the literal was + /// written in. neither resolves where the class lands, and `TypedDict` evaluates its + /// class-body annotations eagerly, so every field is a forward reference + fn class_def(&self, name: &str, typed_dict: &str, quotes: &WrittenQuotes) -> String { let bases = match &self.extra_items { - Some(ty) => format!("{typed_dict}, extra_items={}", quote_type(ty)), + Some(ty) => format!("{typed_dict}, extra_items={}", quotes.quote(ty)), None => format!("{typed_dict}, closed=True"), }; let mut out = format!("class {name}({bases}):\n"); @@ -82,38 +89,17 @@ impl Shape { // attribute annotations. Such typed dicts are still valid // (TypedDict's functional form supports them), but we leave // them un-rewritten and let the user fall back to that form. - let _ = writeln!(out, " {field_name}: {}", quote_type(field_type)); + let _ = writeln!(out, " {field_name}: {}", quotes.quote(field_type)); } out } } -/// Wrap a rendered field type in a string annotation. -/// -/// The hoisted classes are emitted in the preamble, ahead of everything the -/// module defines, and a shape can name a class declared later or a type -/// parameter that only exists inside the generic class the literal was written -/// in. Neither resolves where the class lands, and `TypedDict` evaluates its -/// class-body annotations eagerly, so every field is a forward reference. -fn quote_type(type_source: &str) -> String { - let mut out = String::with_capacity(type_source.len() + 2); - out.push('"'); - for c in type_source.chars() { - match c { - '\\' => out.push_str("\\\\"), - '"' => out.push_str("\\\""), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - _ => out.push(c), - } - } - out.push('"'); - out -} - pub(crate) struct TypedDictLiteral<'src> { source: &'src str, written: WrittenNames<'src>, + /// where the field types are written as strings + quotes: &'src WrittenQuotes, edits: Vec, errors: Vec, /// Insertion-ordered map of shape → synthesized class name so emitted @@ -139,13 +125,15 @@ impl<'src> TypedDictLiteral<'src> { fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: &Config, types: &'src dyn TypeInfo, ) -> Self { Self { source, written, - callable: CallableSyntax::new(source, written, config).with_types(types), + quotes, + callable: CallableSyntax::new(source, written, quotes, config).with_types(types), edits: Vec::new(), errors: Vec::new(), shapes: IndexMap::new(), @@ -160,7 +148,7 @@ impl<'src> TypedDictLiteral<'src> { let mut out = String::new(); let typed_dict = self.written.imported("typing_extensions", "TypedDict"); for (shape, name) in &self.shapes { - out.push_str(&shape.class_def(name, &typed_dict)); + out.push_str(&shape.class_def(name, &typed_dict, self.quotes)); out.push('\n'); } out @@ -503,7 +491,13 @@ impl AstPass for TypedDictLiteralPass<'_> { } fn run(&self, _module: &mut ModModule, ctx: &mut PassContext) { - let mut inner = TypedDictLiteral::new(self.source, self.written, &self.config, self.types); + let mut inner = TypedDictLiteral::new( + self.source, + self.written, + self.quotes, + &self.config, + self.types, + ); // the module as parsed, not the one the passes before this rewrote: the type // information the field lowerer asks is about those nodes, and the edits are keyed // on the source ranges both share @@ -555,6 +549,7 @@ impl AstPass for TypedDictLiteralPass<'_> { pub(crate) struct TypedDictLiteralPass<'src> { source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, /// the module as parsed, which is what `types` answers for suite: &'src [Stmt], @@ -565,6 +560,7 @@ impl<'src> TypedDictLiteralPass<'src> { pub(crate) fn new( source: &'src str, written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, config: Config, suite: &'src [Stmt], types: &'src dyn TypeInfo, @@ -572,6 +568,7 @@ impl<'src> TypedDictLiteralPass<'src> { Self { source, written, + quotes, config, suite, types, diff --git a/crates/by_transforms/tests/lazy_import_runtime.rs b/crates/by_transforms/tests/lazy_import_runtime.rs index a8a22de7da..38fd171daf 100644 --- a/crates/by_transforms/tests/lazy_import_runtime.rs +++ b/crates/by_transforms/tests/lazy_import_runtime.rs @@ -27,6 +27,58 @@ use interpreters::Interpreter; /// a soundness check in a function on a class the module imports further down is made once /// the import has run. the import is deferred, so the name is not in the module's namespace /// then, and the check asks the module's `_LazyFrom` whether its statement has run +/// a forward reference the transpiler quotes in a type argument whose generic comes from a +/// sibling module. the quote reads `JSONDecoder` when something asks for the type, against the +/// module's namespace, so the import is bound where it is written however the module that +/// declares `Box` is resolved +const QUOTED_THROUGH_A_SIBLING: &[(&str, &str)] = &[ + ( + "/by_boxes_mod.by", + r#" +class Box[T]: + pass +"#, + ), + ( + "/main.by", + r#" +import typing +from json import JSONDecoder +from by_boxes_mod import Box + +Alias = Box[F | JSONDecoder] + +def h(x: Alias): + pass + +class F: + pass + +hints = typing.get_type_hints(h) +assert typing.get_args(hints["x"]) == (F | JSONDecoder,), hints +print("ok") +"#, + ), +]; + +/// the classes a lowering hoists ahead of the module write a field type naming one of the +/// module's imports as a string, which reads the import when something asks for the field's +/// type, so the import is bound where it is written +const HOISTED_FIELDS: &str = r#" +import typing +from json import JSONDecoder, JSONEncoder + +Pair = list[(first: JSONDecoder, count: int)] +Shaped = list[protocol(encoder: JSONEncoder)] + +def hints(alias: object) -> dict[str, object]: + return typing.get_type_hints(typing.get_args(alias)[0]) + +assert hints(Pair)["first"] == JSONDecoder, hints(Pair) +assert hints(Shaped)["encoder"] == JSONEncoder, hints(Shaped) +print("ok") +"#; + const ONCE_BOUND: &[(&str, &str)] = &[ ( "/by_once_mod.by", @@ -1008,6 +1060,50 @@ fn a_check_on_a_deferred_class_is_made_once_its_import_has_run() { run_main(&python, &dir); } +/// transpile each of `files` against a project db holding all of them, into `case`'s directory +fn build_project(case: &str, files: &[(&str, &str)]) -> PathBuf { + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(case); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + for (path, _) in files { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = transpile_typed(&db, file, &Config::default(), None) + .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + dir +} + +#[test] +fn a_hoisted_class_s_quoted_field_binds_what_it_reads() { + let Some(python) = python() else { + return; + }; + let dir = build_case("lazy_hoisted_fields", &[("main", HOISTED_FIELDS)]); + run_main(&python, &dir); +} + +#[test] +fn a_quote_in_a_type_argument_of_a_sibling_module_s_generic_binds_what_it_reads() { + let Some(python) = python() else { + return; + }; + let dir = build_project("lazy_quoted_through_a_sibling", QUOTED_THROUGH_A_SIBLING); + let main = fs::read_to_string(dir.join("main.py")).expect("read main"); + assert!(main.contains("Box[\"F | JSONDecoder\"]"), "{main}"); + run_main(&python, &dir); +} + #[test] fn a_relative_import_of_the_package_reads_its_attribute_or_its_submodule() { let Some(python) = python() else { From f5d623c162814488d60c2a6f0f94da968f96622d Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:23:50 +1000 Subject: [PATCH 41/47] =?UTF-8?q?`by=20build`,=20`by=20run`,=20`by=20trans?= =?UTF-8?q?pile=20`=20and=20`by=20compile`=20transpile=20a=20project'?= =?UTF-8?q?s=20files=20in=20parallel.=20each=20rayon=20job=20gets=20its=20?= =?UTF-8?q?own=20handle=20on=20the=20project=20db=20without=20attaching=20?= =?UTF-8?q?it=20for=20the=20whole=20job:=20the=20`Cannot=20change=20databa?= =?UTF-8?q?se=20mid-query`=20panic=20came=20from=20`map=5Fwith=5Fdb`=20att?= =?UTF-8?q?aching=20the=20job's=20handle=20across=20the=20closure,=20not?= =?UTF-8?q?=20from=20a=20database=20built=20inside=20a=20salsa=20query,=20?= =?UTF-8?q?so=20a=20type-aware=20pass's=20own=20single-file=20and=20rebuil?= =?UTF-8?q?t=20databases=20attach=20normally.=20modules=20are=20handed=20t?= =?UTF-8?q?o=20the=20caller=20in=20source=20order,=20so=20a=20build=20repo?= =?UTF-8?q?rts=20every=20module=20it=20could=20not=20emit=20in=20that=20or?= =?UTF-8?q?der,=20and=20under=20the=20`run`=20gate=20the=20hand-over=20sto?= =?UTF-8?q?ps=20at=20the=20first=20failure=20in=20source=20order=20and=20n?= =?UTF-8?q?o=20module=20after=20it=20is=20started.=20a=20panic=20in=20one?= =?UTF-8?q?=20module's=20transpile=20is=20reported=20as=20that=20module's?= =?UTF-8?q?=20`panic`=20diagnostic=20through=20`ty=5Fproject::catch=5Fpani?= =?UTF-8?q?c`,=20the=20helper=20a=20check=20uses=20(now=20public,=20and=20?= =?UTF-8?q?naming=20what=20was=20being=20done=20through=20`PanicError::to?= =?UTF-8?q?=5Fdiagnostic=5Fmessage`),=20so=20a=20cancellation=20goes=20bac?= =?UTF-8?q?k=20to=20the=20caller=20unchanged=20and=20a=20panic=20propagate?= =?UTF-8?q?d=20from=20a=20query=20another=20thread=20was=20running=20is=20?= =?UTF-8?q?reported.=20`by=20build`,=20`by=20run`,=20`by=20compile`=20and?= =?UTF-8?q?=20`by=20transpile=20`=20exit=20101=20when=20a=20check=20o?= =?UTF-8?q?r=20a=20transpile=20panicked,=20as=20`by=20check`=20does.=20on?= =?UTF-8?q?=20neuromorph=20(213=20files,=20interleaved,=204=20rounds)=20th?= =?UTF-8?q?e=20build=20goes=20from=2024.3=E2=80=9327.2s=20wall=20/=2029.4?= =?UTF-8?q?=E2=80=9330.9s=20cpu=20/=201.66=20GB=20to=205.6=E2=80=936.1s=20?= =?UTF-8?q?wall=20/=2035.5=E2=80=9336.5s=20cpu=20/=202.06=E2=80=932.18=20G?= =?UTF-8?q?B,=20with=20byte-identical=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/by_stage/src/emit.rs | 281 ++++++++++++++++-- crates/by_stage/src/restage.rs | 2 +- crates/by_stage/src/runtime.rs | 69 +++-- crates/ruff_db/src/panic.rs | 26 +- crates/ruff_linter/src/message/mod.rs | 7 +- crates/ty/src/by_commands.rs | 43 +-- crates/ty/src/lib.rs | 13 +- crates/ty/tests/by_e2e.rs | 57 ++++ crates/ty_project/src/lib.rs | 61 +++- .../server/api/requests/detached_transpile.rs | 7 +- docs/basedpython/cli-reference.md | 6 +- 11 files changed, 474 insertions(+), 98 deletions(-) diff --git a/crates/by_stage/src/emit.rs b/crates/by_stage/src/emit.rs index db3fa14bc4..1da5bbcee7 100644 --- a/crates/by_stage/src/emit.rs +++ b/crates/by_stage/src/emit.rs @@ -10,17 +10,22 @@ //! them to a terminal and a language server hands them to an editor, and a //! pipeline that wrote to stderr could only ever serve the first. +use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; use by_transforms::config::Config; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use rayon::iter::{ + IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, +}; use ruff_db::diagnostic::{Annotation, Diagnostic, DiagnosticId, Severity, Span}; +use ruff_db::panic::PanicActivity; use ruff_db::source::source_text; use ty_project::ProjectDatabase; use ty_project::parallel::ParallelIteratorExt; use crate::project::Rebuilder; -use crate::runtime::RuntimeLayout; +use crate::runtime::{RuntimeLayout, RuntimePackage}; use crate::staging::transpiled_destination; /// How much of the check outcome blocks emitting output. @@ -87,6 +92,34 @@ pub struct Emit<'a> { pub root: &'a Path, } +/// one module the transpile fans out over +struct Job<'a> { + by_path: &'a Path, + file: ruff_db::files::File, + /// the package whose runtime copy the module imports, when the build writes + /// one + runtime_package: Option, +} + +/// what one module's transpile came to +enum Outcome { + Transpiled { + python: String, + line_map: Vec>, + requirements: by_transforms::RuntimeRequirements, + }, + /// the transpiler declined to emit the module, for whatever reason it gave + Failed(by_transforms::TranspileError), + /// the transpiler panicked, reported the way a check reports a panic + Panicked(Diagnostic), +} + +impl Outcome { + fn is_failure(&self) -> bool { + !matches!(self, Outcome::Transpiled { .. }) + } +} + /// Check every file, then for each non-blocked file call `consume` with the /// transpiled Python. pub fn check_and_transpile( @@ -170,45 +203,86 @@ pub fn check_and_transpile( } let mut ok = unusable.is_empty(); - // the transpile stays serial. it is the expensive half and the files are - // independent, so it looks like it should fan out — but a type-aware pass - // builds databases of its own (a single-file db for the rewritten source, - // and the project rebuilt over it), and salsa refuses to attach one while a - // query on another is running. parallelising this needs the transpiler to - // stop making databases mid-query first - let rebuild = || Some(rebuilder.rebuild()); - for (bpy, file) in handles { - if unusable.contains(file) { - continue; - } - // the helpers this module calls come from a copy written beside it, and - // which copy depends on where it lands — so the config is settled per - // file rather than once for the build - let relative = transpiled_destination(roots, root, bpy); + // the helpers a module calls come from a copy written beside it, and which + // copy depends on where it lands, so the config is settled per file rather + // than once for the build + let system = ruff_db::Db::system(db); + let runtime_wanted = runtime.is_some(); + let jobs: Vec> = handles + .iter() + .filter(|(_, file)| !unusable.contains(file)) + .map(|(by_path, file)| Job { + by_path, + file: *file, + runtime_package: if runtime_wanted { + RuntimePackage::of( + &transpiled_destination(roots, root, by_path), + by_path, + system, + ) + } else { + None + }, + }) + .collect(); + let finished = fan_out(jobs, gate == CheckGate::AllErrors, db.clone(), |db, job| { let config = Config { - runtime_module: runtime - .as_deref_mut() - .and_then(|layout| layout.claim(&relative, bpy, ruff_db::Db::system(db))), + runtime_module: job.runtime_package.as_ref().map(RuntimePackage::module), ..config.clone() }; - match by_transforms::transpile_typed_with_report(db, *file, &config, Some(&rebuild)) { - Ok((out, line_map, needed)) => { + let rebuild = || Some(rebuilder.rebuild()); + // caught on the worker that ran it, because the hook that records a + // panic's location and backtrace is armed per thread + let transpiled = ty_project::catch_panic( + &*db, + job.file, + PanicActivity::Transpiling, + AssertUnwindSafe(|| { + by_transforms::transpile_typed_with_report(&*db, job.file, &config, Some(&rebuild)) + }), + ); + match transpiled { + Ok(Ok((python, line_map, requirements))) => Outcome::Transpiled { + python, + line_map, + requirements, + }, + Ok(Err(error)) => Outcome::Failed(error), + Err(panic) => Outcome::Panicked(panic), + } + }); + for (job, outcome) in finished { + // a job is skipped only after an earlier one failed, and the hand-over + // has already stopped there + let Some(outcome) = outcome else { + break; + }; + if let (Some(layout), Some(package)) = (runtime.as_deref_mut(), &job.runtime_package) { + layout.record(package); + } + let failure = match outcome { + Outcome::Transpiled { + python, + line_map, + requirements: needed, + } => { requirements.merge(needed); - let by_source = source_text(db, *file); + let by_source = source_text(db, job.file); consume(&Transpiled { - by_path: bpy, + by_path: job.by_path, by_source: by_source.as_str(), - python: &out, + python: &python, line_map: &line_map, })?; + continue; } - Err(e) => { - all_diagnostics.push(transpile_failure_diagnostic(*file, &e)); - ok = false; - if gate == CheckGate::AllErrors { - break; - } - } + Outcome::Failed(error) => transpile_failure_diagnostic(job.file, &error), + Outcome::Panicked(diagnostic) => diagnostic, + }; + all_diagnostics.push(failure); + ok = false; + if gate == CheckGate::AllErrors { + break; } } @@ -222,6 +296,46 @@ pub fn check_and_transpile( }) } +/// Run `transpile` over every job in parallel, and hand each job back with what +/// it came to, in the jobs' own order whatever order they finished in. +/// +/// With `stop_at_first_failure`, a job is not started once a job before it has +/// failed: the caller stops at the first failure in order, so nothing after it is +/// needed. It comes back with no outcome. +/// +/// Each job gets its own copy of `handle`, as the check's jobs each get their own +/// handle on the project db, and none is attached for the whole job: a type-aware +/// pass analyses rewritten source in databases of its own, and salsa refuses to +/// attach a second database on a thread that already has one attached. That holds +/// only because a rayon worker has nothing attached between jobs, so a rayon job +/// that runs attached must never block on nested parallelism, where its worker +/// could steal one of these jobs and run it under that attachment. +fn fan_out( + jobs: Vec, + stop_at_first_failure: bool, + handle: H, + transpile: impl Fn(&mut H, &J) -> Outcome + Sync + Send, +) -> Vec<(J, Option)> +where + J: Send, + H: Clone + Send, +{ + let first_failure = AtomicUsize::new(usize::MAX); + jobs.into_par_iter() + .enumerate() + .map_with(handle, |handle, (index, job)| { + if index > first_failure.load(Ordering::Relaxed) { + return (job, None); + } + let outcome = transpile(handle, &job); + if stop_at_first_failure && outcome.is_failure() { + first_failure.fetch_min(index, Ordering::Relaxed); + } + (job, Some(outcome)) + }) + .collect() +} + /// Whether this diagnostic says the file has no source to transpile: it could /// not be parsed, or it could not be read at all (an encoding ty does not speak, /// a permission error). Either way the transpiler sees an empty module, so @@ -254,3 +368,106 @@ pub fn transpile_failure_diagnostic( } diag } + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::mpsc; + use std::time::Duration; + + use ruff_db::diagnostic::{Diagnostic, DiagnosticId, Severity}; + + use super::{Outcome, fan_out}; + + fn transpiled() -> Outcome { + Outcome::Transpiled { + python: String::new(), + line_map: Vec::new(), + requirements: by_transforms::RuntimeRequirements::default(), + } + } + + fn panicked() -> Outcome { + Outcome::Panicked(Diagnostic::new( + DiagnosticId::Panic, + Severity::Fatal, + "the transpiler is broken", + )) + } + + fn pool(threads: usize) -> rayon::ThreadPool { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("a thread pool") + } + + /// the first job cannot finish until the second has, so the order they finish in is the + /// reverse of theirs, and the jobs still come back in their own + #[test] + fn jobs_come_back_in_their_own_order_whatever_order_they_finish_in() { + let (second_finished, wait_for_second) = mpsc::channel::<()>(); + let wait_for_second = Mutex::new(wait_for_second); + let second_finished = Mutex::new(second_finished); + let saw_the_second_finish = AtomicUsize::new(0); + + let finished = pool(2).install(|| { + fan_out(vec![0, 1], false, (), |(), job: &usize| { + if *job == 0 { + let waited = wait_for_second + .lock() + .expect("the receiver") + .recv_timeout(Duration::from_secs(30)); + if waited.is_ok() { + saw_the_second_finish.fetch_add(1, Ordering::Relaxed); + } + } else { + let _ = second_finished.lock().expect("the sender").send(()); + } + transpiled() + }) + }); + + assert_eq!( + saw_the_second_finish.load(Ordering::Relaxed), + 1, + "the two jobs did not run at the same time" + ); + let order: Vec = finished.iter().map(|(job, _)| *job).collect(); + assert_eq!(order, [0, 1]); + assert!(finished.iter().all(|(_, outcome)| outcome.is_some())); + } + + /// one thread runs the jobs in their order, so every job after the failed first one is + /// one that had not started when it failed + #[test] + fn a_job_after_a_failure_is_not_started_when_the_caller_stops_there() { + let started = AtomicUsize::new(0); + let finished = pool(1).install(|| { + fan_out(vec![0, 1, 2, 3], true, (), |(), job: &usize| { + started.fetch_add(1, Ordering::Relaxed); + if *job == 0 { panicked() } else { transpiled() } + }) + }); + + assert_eq!(started.load(Ordering::Relaxed), 1); + let ran: Vec = finished + .iter() + .map(|(_, outcome)| outcome.is_some()) + .collect(); + assert_eq!(ran, [true, false, false, false]); + } + + /// a build reports every module it could not emit, so a failure stops nothing there + #[test] + fn every_job_runs_when_the_caller_goes_on_past_a_failure() { + let finished = pool(1).install(|| { + fan_out(vec![0, 1, 2], false, (), |(), job: &usize| { + if *job == 0 { panicked() } else { transpiled() } + }) + }); + + assert!(finished.iter().all(|(_, outcome)| outcome.is_some())); + } +} diff --git a/crates/by_stage/src/restage.rs b/crates/by_stage/src/restage.rs index c7af1e95bb..a075ff8587 100644 --- a/crates/by_stage/src/restage.rs +++ b/crates/by_stage/src/restage.rs @@ -375,7 +375,7 @@ fn restage_transpiled( rebuilder: &tree.rebuilder, requirements: &mut by_transforms::RuntimeRequirements::default(), // a re-stage writes modules back into a tree the build already laid - // out, so what it claims is thrown away: whatever copy of the runtime + // out, so what it records is thrown away: whatever copy of the runtime // a module imports is already sitting where the build put it runtime: Some(&mut RuntimeLayout::default()), roots: &tree.record.module_roots, diff --git a/crates/by_stage/src/runtime.rs b/crates/by_stage/src/runtime.rs index 07d418fed3..91672dd869 100644 --- a/crates/by_stage/src/runtime.rs +++ b/crates/by_stage/src/runtime.rs @@ -28,7 +28,7 @@ use ruff_db::system::{System, SystemPath}; /// the copies a build writes /// -/// claimed for every module rather than only those that turned out to call a +/// recorded for every module rather than only those that turned out to call a /// helper: `by restage` recomputes modules against a tree an earlier build /// wrote, and an edit that newly reaches for a helper has to find the copy there #[derive(Debug, Default)] @@ -38,15 +38,29 @@ pub struct RuntimeLayout { } impl RuntimeLayout { - /// record that `source`, landing at `relative` in the module tree, needs the - /// runtime, and answer the module it imports it from — or `None` for a module - /// in no package, which gets the definitions pasted in - pub(crate) fn claim( - &mut self, - relative: &Path, - source: &Path, - system: &dyn System, - ) -> Option { + /// record that a module importing `package`'s copy was transpiled, whether or + /// not the transpile succeeded + pub(crate) fn record(&mut self, package: &RuntimePackage) { + self.packages.insert(package.0.clone()); + } + + /// where each copy goes within the module tree + pub fn files(&self) -> impl Iterator + '_ { + self.packages + .iter() + .map(|package| Path::new(package).join(by_transforms::runtime::FILE_NAME)) + } +} + +/// the package whose copy of the runtime a module imports +#[derive(Debug)] +pub(crate) struct RuntimePackage(String); + +impl RuntimePackage { + /// the package `source`, landing at `relative` in the module tree, imports the + /// runtime from — or `None` for a module in no package, which gets the + /// definitions pasted in + pub(crate) fn of(relative: &Path, source: &Path, system: &dyn System) -> Option { let mut components = relative.components(); let first = components.next()?; // a `relative` of one component is the module's own file name: it sits at @@ -59,15 +73,12 @@ impl RuntimeLayout { if !is_package(&module_tree_base(relative, source).join(package), system) { return None; } - self.packages.insert(package.to_owned()); - Some(format!("{package}.{}", by_transforms::runtime::MODULE_NAME)) + Some(Self(package.to_owned())) } - /// where each copy goes within the module tree - pub fn files(&self) -> impl Iterator + '_ { - self.packages - .iter() - .map(|package| Path::new(package).join(by_transforms::runtime::FILE_NAME)) + /// the module the copy is imported as + pub(crate) fn module(&self) -> String { + format!("{}.{}", self.0, by_transforms::runtime::MODULE_NAME) } } @@ -117,14 +128,16 @@ mod tests { Self { root, system } } - fn claim(&self, layout: &mut RuntimeLayout, relative: &str) -> Option { + fn record(&self, layout: &mut RuntimeLayout, relative: &str) -> Option { let source = self .root .path() .join("src") .join(relative) .with_extension("by"); - layout.claim(Path::new(relative), &source, &self.system) + let package = RuntimePackage::of(Path::new(relative), &source, &self.system)?; + layout.record(&package); + Some(package.module()) } } @@ -133,7 +146,7 @@ mod tests { fn a_root_module_gets_the_definitions_pasted_in() { let project = Project::with(&["src/main.by"]); let mut layout = RuntimeLayout::default(); - assert_eq!(project.claim(&mut layout, "main.py"), None); + assert_eq!(project.record(&mut layout, "main.py"), None); assert_eq!(layout.files().count(), 0); } @@ -142,7 +155,7 @@ mod tests { let project = Project::with(&["src/pkg/__init__.py", "src/pkg/a.by"]); let mut layout = RuntimeLayout::default(); assert_eq!( - project.claim(&mut layout, "pkg/a.py").as_deref(), + project.record(&mut layout, "pkg/a.py").as_deref(), Some("pkg._by_runtime") ); assert_eq!( @@ -157,7 +170,7 @@ mod tests { let project = Project::with(&["src/pkg/__init__.by", "src/pkg/a.by"]); let mut layout = RuntimeLayout::default(); assert_eq!( - project.claim(&mut layout, "pkg/a.py").as_deref(), + project.record(&mut layout, "pkg/a.py").as_deref(), Some("pkg._by_runtime") ); } @@ -168,7 +181,7 @@ mod tests { fn a_module_in_no_package_gets_the_definitions_pasted_in() { let project = Project::with(&["src/scripts/tool.by"]); let mut layout = RuntimeLayout::default(); - assert_eq!(project.claim(&mut layout, "scripts/tool.py"), None); + assert_eq!(project.record(&mut layout, "scripts/tool.py"), None); assert_eq!(layout.files().count(), 0); } @@ -178,7 +191,7 @@ mod tests { let project = Project::with(&["src/pkg/__init__.py", "src/pkg/sub/deep/a.by"]); let mut layout = RuntimeLayout::default(); assert_eq!( - project.claim(&mut layout, "pkg/sub/deep/a.py").as_deref(), + project.record(&mut layout, "pkg/sub/deep/a.py").as_deref(), Some("pkg._by_runtime") ); assert_eq!( @@ -191,9 +204,9 @@ mod tests { fn each_package_carries_its_own() { let project = Project::with(&["src/one/__init__.py", "src/two/__init__.py"]); let mut layout = RuntimeLayout::default(); - project.claim(&mut layout, "one/a.py"); - project.claim(&mut layout, "two/b.py"); - project.claim(&mut layout, "one/c.py"); + project.record(&mut layout, "one/a.py"); + project.record(&mut layout, "two/b.py"); + project.record(&mut layout, "one/c.py"); assert_eq!( layout.files().collect::>(), vec![ @@ -204,7 +217,7 @@ mod tests { } #[test] - fn an_unclaimed_layout_writes_nothing() { + fn an_unrecorded_layout_writes_nothing() { assert_eq!(RuntimeLayout::default().files().count(), 0); } } diff --git a/crates/ruff_db/src/panic.rs b/crates/ruff_db/src/panic.rs index 6d12e5bd0f..fb8de34328 100644 --- a/crates/ruff_db/src/panic.rs +++ b/crates/ruff_db/src/panic.rs @@ -35,12 +35,34 @@ impl std::fmt::Display for Payload { } } +/// What was being done to a file when it panicked, as a panic diagnostic says it +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PanicActivity { + Checking, + Transpiling, +} + +impl std::fmt::Display for PanicActivity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + PanicActivity::Checking => "checking", + PanicActivity::Transpiling => "transpiling", + }) + } +} + impl PanicError { pub fn resume_unwind(self) -> ! { std::panic::resume_unwind(self.payload.0) } - pub fn to_diagnostic_message(&self, path: Option) -> String { + /// The message a panic diagnostic carries, saying what was being done to `path` + /// when it panicked + pub fn to_diagnostic_message( + &self, + activity: PanicActivity, + path: Option, + ) -> String { use std::fmt::Write; let mut message = String::new(); @@ -51,7 +73,7 @@ impl PanicError { } if let Some(path) = path { - let _ = write!(&mut message, " when checking `{path}`"); + let _ = write!(&mut message, " when {activity} `{path}`"); } let _ = write!(&mut message, ": `{payload}`", payload = self.payload); diff --git a/crates/ruff_linter/src/message/mod.rs b/crates/ruff_linter/src/message/mod.rs index 55cb3e6ad4..290c0a874d 100644 --- a/crates/ruff_linter/src/message/mod.rs +++ b/crates/ruff_linter/src/message/mod.rs @@ -3,7 +3,7 @@ use std::fmt::Display; use std::io::Write; use std::path::Path; -use ruff_db::panic::PanicError; +use ruff_db::panic::{PanicActivity, PanicError}; use rustc_hash::FxHashMap; use ruff_db::diagnostic::{ @@ -31,7 +31,10 @@ pub fn create_panic_diagnostic(error: &PanicError, path: Option<&Path>) -> Diagn let mut diagnostic = Diagnostic::new( DiagnosticId::Panic, Severity::Fatal, - error.to_diagnostic_message(path.as_ref().map(|path| path.display())), + error.to_diagnostic_message( + PanicActivity::Checking, + path.as_ref().map(|path| path.display()), + ), ); diagnostic.sub(SubDiagnostic::new( diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index eabf23b338..2fa3fc7d54 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -238,7 +238,7 @@ pub(crate) fn cmd_run( let mut traceback_entries: Vec = Vec::new(); let mut staging = Staging::new(tmp.path()); let mut layout = RuntimeLayout::default(); - let ok = render_check_and_transpile( + let status = render_check_and_transpile( &db, &handles, &mut Emit { @@ -256,8 +256,8 @@ pub(crate) fn cmd_run( Ok(()) }, )?; - if !ok { - return Ok(ExitStatus::Failure); + if !matches!(status, ExitStatus::Success) { + return Ok(status); } // a program is its data as much as its modules: a `.py` module it imports, a // json file it opens, a template it renders. running out of a directory @@ -960,7 +960,7 @@ pub(crate) fn cmd_build( let mut entries: Vec = Vec::new(); let mut requirements = by_transforms::RuntimeRequirements::default(); let mut layout = RuntimeLayout::default(); - let ok = render_check_and_transpile( + let status = render_check_and_transpile( &db, &handles, &mut Emit { @@ -1003,8 +1003,9 @@ pub(crate) fn cmd_build( } staging.finish()?; - if !ok? { - return Ok(ExitStatus::Failure); + let status = status?; + if !matches!(status, ExitStatus::Success) { + return Ok(status); } eprintln!("\nbuild complete ({file_count} files)"); Ok(ExitStatus::Success) @@ -1458,8 +1459,9 @@ pub(crate) fn cmd_compile( // the transpile's verdict is answered after the tree is finished, the way // `build` answers it: a diagnostic must not cost the tree, because a `build/` // a debugger cannot read is worse than one built from a partial check - if !transpiled? { - return Ok(ExitStatus::Failure); + let transpiled = transpiled?; + if !matches!(transpiled, ExitStatus::Success) { + return Ok(transpiled); } eprintln!("\ncompiled {compiled} module(s)"); @@ -1719,7 +1721,7 @@ fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { } let roots = module_roots(&db, &root); - let ok = render_check_and_transpile( + render_check_and_transpile( &db, &handles, &mut Emit { @@ -1739,12 +1741,7 @@ fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { fs::write(&py, emitted.python).with_context(|| format!("{}", py.display()))?; Ok(()) }, - )?; - Ok(if ok { - ExitStatus::Success - } else { - ExitStatus::Failure - }) + ) } /// Every first-party `.py`/`.pyi` file under `root`, skipping non-source @@ -2275,9 +2272,9 @@ fn compilable_files(root: &Path) -> Vec { } /// Check every file, render diagnostics, then for each non-blocked file call -/// `consume` with the transpiled Python. Returns `Ok(false)` if the check -/// outcome blocks per `gate`, or a transpiler bug occurred (caller should -/// propagate failure). +/// `consume` with the transpiled Python. Answers the exit status the command +/// ends with: a failure when something was reported, and an internal error when +/// that was `by` itself panicking, the way `by check` tells the two apart /// /// The deciding is [`by_stage::emit::check_and_transpile`], and only the /// **rendering** is here. That is the whole difference between the two callers @@ -2291,12 +2288,18 @@ fn render_check_and_transpile( handles: &[(PathBuf, ruff_db::files::File)], emit: &mut Emit<'_>, consume: impl FnMut(&Transpiled<'_>) -> anyhow::Result<()>, -) -> anyhow::Result { +) -> anyhow::Result { let emitted = by_stage::emit::check_and_transpile(db, handles, emit, consume)?; if !emitted.diagnostics.is_empty() { render_diagnostics(db, &emitted.diagnostics)?; } - Ok(emitted.ok) + Ok(if emitted.ok { + ExitStatus::Success + } else if emitted.diagnostics.iter().any(|d| d.severity().is_fatal()) { + ExitStatus::InternalError + } else { + ExitStatus::Failure + }) } /// Render diagnostics to stderr in the same format as `by check`. The diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index ac19499a8a..8a58a3b049 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -69,12 +69,13 @@ where // type inference recurses with the shape of the program it is checking, so // how deep a file it can survive is decided by the stack it runs on. the // rayon pool asks for `STACK_SIZE`, but the thread a process starts on gets a - // platform default — 1 MiB on windows — and the commands that check on the - // calling thread rather than through the pool (`run`, `build`, `transpile`, - // `compile`) were overflowing it there. so the whole command runs on a thread - // this codebase has sized for the job, wherever it is dispatched to. a command - // does more per term than a check does, since it lowers the expression as well - // as inferring it, and `STACK_SIZE` is sized to leave room for that + // platform default — 1 MiB on windows — and a command that infers on the + // calling thread rather than through the pool (a single-file `transpile`, + // `compile`'s check and lowering) overflowed it there. so the whole command + // runs on a thread this codebase has sized for the job, wherever it is + // dispatched to. a command does more per term than a check does, since it + // lowers the expression as well as inferring it, and `STACK_SIZE` is sized to + // leave room for that std::thread::scope(|scope| { let command = std::thread::Builder::new() .stack_size(STACK_SIZE) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 84b9fb0876..7a24d1e7c4 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1,3 +1,4 @@ +use std::fmt::Write as _; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; @@ -3434,6 +3435,62 @@ fn build_emits_every_file_it_can_past_a_broken_one() { assert!(stderr.contains("broken.by"), "got:\n{stderr}"); } +/// the modules are transpiled in parallel and handed over in source order. a +/// build reports every module it could not emit, in that order, while a run +/// stops at the first one: the program cannot run without it, so nothing after +/// it is reported or run +#[test] +fn transpile_failures_are_reported_in_source_order() { + let dir = tempfile::tempdir().expect("tempdir"); + // each module takes over the name the runtime's `Some(...)` wrapper is + // provided under, which the transpile refuses. `by check` has nothing to say + // about it, so a run gets as far as transpiling + let refused = "from typing import Optional\n\nx = Some(3)\nprint(x!)\n"; + // the first module is the larger, so it is the last to finish: an order + // the transpiles finished in would put it after `b.by` + let mut larger = String::new(); + for i in 0..400 { + let _ = write!(larger, "\ndef f{i}(x: int) -> int:\n return x + {i}\n"); + } + fs::write(dir.path().join("a.by"), format!("{refused}{larger}")).unwrap(); + fs::write(dir.path().join("b.by"), refused).unwrap(); + fs::write(dir.path().join("main.by"), "print('main ran')\n").unwrap(); + let by = |args: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(args) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by") + }; + let refusals = |stderr: &str| -> Vec { + let lines: Vec<&str> = stderr.lines().collect(); + lines + .windows(2) + .filter(|pair| pair[0].starts_with("error[transpile-refused]")) + .map(|pair| pair[1].trim().to_owned()) + .collect() + }; + + let build = by(&["build"]); + let stderr = String::from_utf8_lossy(&build.stderr); + assert!(!build.status.success(), "{stderr}"); + assert_eq!( + refusals(&stderr), + ["--> a.by:1:20", "--> b.by:1:20"], + "got:\n{stderr}" + ); + + let run = by(&["run", "main"]); + let stderr = String::from_utf8_lossy(&run.stderr); + assert!(!run.status.success(), "{stderr}"); + assert_eq!(refusals(&stderr), ["--> a.by:1:20"], "got:\n{stderr}"); + assert!( + !String::from_utf8_lossy(&run.stdout).contains("main ran"), + "the program ran past a module it could not transpile" + ); +} + /// `by build` walks the project's own file set, so `src.exclude` applies to it /// exactly as it does to `by check` #[test] diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index 8f3cd91631..74d2288f3a 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -19,6 +19,7 @@ use ruff_db::diagnostic::{ Annotation, Diagnostic, DiagnosticId, Severity, Span, SubDiagnostic, SubDiagnosticSeverity, }; use ruff_db::files::{File, system_path_to_file}; +use ruff_db::panic::PanicActivity; use ruff_db::parsed::parsed_module; use ruff_db::system::{SystemPath, SystemPathBuf, deduplicate_nested_paths}; use rustc_hash::FxHashSet; @@ -1092,7 +1093,7 @@ pub(crate) fn check_file_impl( let source_file = file.file(db); { let db = AssertUnwindSafe(db); - match catch(&**db, source_file, || { + match catch_panic(&**db, source_file, PanicActivity::Checking, || { let script = Script::for_file(*db, source_file); if let Some(script) = script && !script.has_valid_settings(*db) @@ -1196,7 +1197,17 @@ impl Iterator for ProjectFilesIter<'_> { impl FusedIterator for ProjectFilesIter<'_> {} -fn catch(db: &dyn Db, file: File, f: F) -> Result +/// Run `f`, the work of `activity` on `file`, and report a panic in it as a `panic` diagnostic +/// against the file rather than unwinding. +/// +/// A cancellation is not a panic and is resumed, except for a panic propagated from a query +/// another thread was running, which is reported against this file too. +pub fn catch_panic( + db: &dyn Db, + file: File, + activity: PanicActivity, + f: F, +) -> Result where F: FnOnce() -> R + UnwindSafe, { @@ -1227,7 +1238,7 @@ where } } - let message = error.to_diagnostic_message(Some(file.path(db))); + let message = error.to_diagnostic_message(activity, Some(file.path(db))); let mut diagnostic = Diagnostic::new(DiagnosticId::Panic, Severity::Fatal, message); diagnostic.add_bug_sub_diagnostics("%5Bpanic%5D"); @@ -1269,17 +1280,59 @@ where #[cfg(test)] mod tests { - use crate::check_file_impl; use crate::db::Db as _; use crate::db::testing::TestDb; use crate::{IncludeResult, ProjectMetadata}; + use crate::{catch_panic, check_file_impl}; + use ruff_db::diagnostic::DiagnosticId; use ruff_db::files::system_path_to_file; + use ruff_db::panic::PanicActivity; use ruff_db::source::source_text; use ruff_db::system::{DbWithTestSystem, DbWithWritableSystem as _, SystemPath, SystemPathBuf}; use ruff_db::testing::assert_function_query_was_not_run; use ty_python_semantic::Db as _; use ty_python_semantic::types::check_types; + /// a query this file's work waited on panicked on another thread: that is a panic to + /// report against this file too, not a cancellation to hand back to the caller + #[test] + fn a_panic_propagated_from_another_thread_is_reported() -> ruff_db::system::Result<()> { + let project = ProjectMetadata::new("test", SystemPathBuf::from("/")); + let mut db = TestDb::new(project); + db.write_file("test.py", "")?; + let file = system_path_to_file(&db, "test.py").unwrap(); + + let caught = catch_panic(&db, file, PanicActivity::Transpiling, || { + std::panic::resume_unwind(Box::new(salsa::Cancelled::PropagatedPanic)) + }); + + let diagnostic = caught.expect_err("the propagated panic is reported"); + assert_eq!(diagnostic.id(), DiagnosticId::Panic); + let message = diagnostic.headline_message(); + assert!(message.contains("when transpiling"), "{message}"); + Ok(()) + } + + /// a pending write cancels the work, and the caller is the one that knows what to do about + /// that, so it goes back unchanged + #[test] + fn a_cancellation_is_handed_back() -> ruff_db::system::Result<()> { + let project = ProjectMetadata::new("test", SystemPathBuf::from("/")); + let mut db = TestDb::new(project); + db.write_file("test.py", "")?; + let file = system_path_to_file(&db, "test.py").unwrap(); + + let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + catch_panic(&db, file, PanicActivity::Transpiling, || { + std::panic::resume_unwind(Box::new(salsa::Cancelled::PendingWrite)) + }) + })); + + let payload = unwound.expect_err("the cancellation unwound past the catch"); + assert!(payload.downcast_ref::().is_some()); + Ok(()) + } + #[test] fn check_file_skips_type_checking_when_file_cant_be_read() -> ruff_db::system::Result<()> { let project = ProjectMetadata::new("test", SystemPathBuf::from("/")); diff --git a/crates/ty_server/src/server/api/requests/detached_transpile.rs b/crates/ty_server/src/server/api/requests/detached_transpile.rs index 3bcb5643a7..fe72a9d09d 100644 --- a/crates/ty_server/src/server/api/requests/detached_transpile.rs +++ b/crates/ty_server/src/server/api/requests/detached_transpile.rs @@ -15,8 +15,11 @@ //! This is a workaround and worth naming as one. The nesting belongs to the //! transpiler, not to any request, so the fix that would remove this is for a //! type-aware pass to be handed a database rather than construct one. Until then -//! every caller that already holds one attached has to arrange not to be — the same -//! reason `by_stage::emit`'s transpile loop cannot be parallelised. +//! every caller that already holds one attached has to arrange not to be. A +//! re-stage also reads its own clone of the project database on the thread it is +//! called from, which inside a handler would be a second database attached over +//! the session's. `by_stage::emit` arranges the same thing for its parallel +//! transpile, by not attaching a job's database for the whole job. use std::panic::AssertUnwindSafe; diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index fe592550a2..60db177a3f 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -136,7 +136,11 @@ artifacts and exit status answer different questions. everything that could be emitted is emitted, sourcemap and package markers included; the exit status says whether anything was *reported*. so a build that prints an error exits 1 while still leaving a usable `build/`, and `by build && pytest build/tests` runs the tests -only against a tree the checker had nothing to say about +only against a tree the checker had nothing to say about. when a reported +diagnostic is fatal the exit status is 101 instead, as it is for `by check`. the +fatal one is `panic`, reported when `by` itself panicked while checking or +transpiling a file, so a script can tell a crash in `by` from a problem in the +code. `by run`, `by compile` and `by transpile ` answer the same way writes the transpiled python to the project root's `build/` — wherever in the project it is run, as `by compile` does too; `--out DIR` names another directory, From 291cc9fb70698c514e612abfcb712c05b52a110d Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:27:11 +1000 Subject: [PATCH 42/47] =?UTF-8?q?the=20db=20`by=20build`,=20`by=20run`,=20?= =?UTF-8?q?`by=20transpile=20`,=20`by=20compile`=20and=20`by=20restag?= =?UTF-8?q?e`=20read=20a=20project=20through=20is=20set=20up=20the=20way?= =?UTF-8?q?=20`by=20check`=20sets=20up=20its=20own:=20parsed-module=20lru?= =?UTF-8?q?=20disabled,=20open=20files=20frozen=20and=20the=20project=20an?= =?UTF-8?q?d=20file=20inputs=20frozen,=20before=20the=20sources=20are=20in?= =?UTF-8?q?dexed=20so=20the=20freeze=20reaches=20every=20file.=20without?= =?UTF-8?q?=20the=20freeze=20salsa=20recorded=20a=20dependency=20on=20each?= =?UTF-8?q?=20file=20input,=20and=20the=20check=20inside=20a=20build=20cos?= =?UTF-8?q?t=20more=20than=20a=20standalone=20`by=20check`.=20on=20neuromo?= =?UTF-8?q?rph=20(213=20files,=20ABBA,=204=20rounds)=20the=20build's=20che?= =?UTF-8?q?ck=20phase=20goes=20from=207.9=E2=80=938.7s=20cpu=20/=201.04?= =?UTF-8?q?=E2=80=931.11=20GB=20to=205.2=E2=80=935.8s=20cpu=20/=20312?= =?UTF-8?q?=E2=80=93317=20MB,=20and=20the=20whole=20build=20from=2036.3?= =?UTF-8?q?=E2=80=9340.2s=20cpu=20/=202.02=E2=80=932.19=20GB=20peak=20to?= =?UTF-8?q?=2033.5=E2=80=9336.2s=20cpu=20/=201.20=E2=80=931.38=20GB,=20wit?= =?UTF-8?q?h=20byte-identical=20output=20and=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 + crates/by_stage/Cargo.toml | 1 + crates/by_stage/src/project.rs | 89 ++++++++++++++++++++++++++++++++-- crates/ty/src/by_commands.rs | 8 +-- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 289a8d0292..8f2bd37d66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -441,6 +441,7 @@ dependencies = [ "dunce", "rayon", "ruff_db", + "salsa", "serde", "serde_json", "sha2", diff --git a/crates/by_stage/Cargo.toml b/crates/by_stage/Cargo.toml index 40775522f1..7074d09240 100644 --- a/crates/by_stage/Cargo.toml +++ b/crates/by_stage/Cargo.toml @@ -29,6 +29,7 @@ sha2 = { workspace = true } walkdir = { workspace = true } [dev-dependencies] +salsa = { workspace = true } tempfile = { workspace = true } [lints] diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index 50cc973ec4..9654ad8a49 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -17,6 +17,7 @@ //! to be one implementation of each. use std::ffi::OsStr; +use std::ops::Deref; use std::path::{Path, PathBuf}; use anyhow::Context; @@ -112,12 +113,27 @@ impl Rebuilder { /// returns the `\\?\` verbatim form and the second does not — so a caller that /// re-derived it would find none of the db's paths under it. pub type ProjectBuild = ( - ProjectDatabase, + FrozenProjectDatabase, Vec<(PathBuf, ruff_db::files::File)>, Rebuilder, PathBuf, ); +/// a project db for a command that runs once, whose inputs are frozen +/// +/// it hands out only a shared [`ProjectDatabase`], because salsa panics on a write to a frozen +/// input and a write needs the db by `&mut`. a clone of that handle can still be written +/// through, and panics the same way +pub struct FrozenProjectDatabase(ProjectDatabase); + +impl Deref for FrozenProjectDatabase { + type Target = ProjectDatabase; + + fn deref(&self) -> &ProjectDatabase { + &self.0 + } +} + /// the sources `build`, `run` and `transpile` claim: a `.py` beside a `.by` /// is that file's own output, so writing beside it again would be circular pub const BY_SOURCES: &[&str] = &["by", "byi"]; @@ -276,7 +292,15 @@ pub fn build_project_db( .with_context(|| format!("non-utf8 path: {}", canonical_root.display()))?; let metadata = project_metadata.clone(); - let db = ProjectDatabase::use_defaults(project_metadata, system); + let mut db = ProjectDatabase::use_defaults(project_metadata, system); + // what `by check` does to its db, for the same reasons: the command runs once and changes + // no input, so salsa need not record a dependency on each one, and every parsed syntax + // tree is kept until it exits rather than evicted, a trade only a short-lived db should + // make. the freeze reaches only files indexed after it, so it comes before the sources + // are read + ruff_db::disable_lru(&mut db); + db.freeze_open_files(); + db.freeze(); let mut sources = project_sources(&db, extensions, &canonical_root, output); // the walk is over a hash set, so order is arbitrary; emit deterministically @@ -291,7 +315,12 @@ pub fn build_project_db( root: sys_root.to_path_buf(), included, }; - Ok((db, sources, rebuilder, canonical_root)) + Ok(( + FrozenProjectDatabase(db), + sources, + rebuilder, + canonical_root, + )) } /// The sources a build claims out of a db that already exists. @@ -356,8 +385,60 @@ pub fn module_roots(db: &ProjectDatabase, cwd: &Path) -> Vec { #[cfg(test)] mod tests { - use super::is_hidden_within; + use super::{BY_SOURCES, build_project_db, is_hidden_within}; + use ruff_db::files::File; + use salsa::Setter as _; + use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::Path; + use ty_project::{Db, ProjectDatabase}; + + /// whether salsa refuses `write` because the input it writes is frozen + /// + /// each write gets a db of its own, built outside the panic it is expected to raise, so a + /// build that panics on its own fails the test rather than passing it + fn refuses_as_frozen(write: impl FnOnce(&mut ProjectDatabase, File)) -> bool { + let project = tempfile::tempdir().expect("tempdir"); + std::fs::write(project.path().join("main.by"), "x = 1\n").expect("a source"); + let (db, sources, _, _) = + build_project_db(project.path(), BY_SOURCES, None).expect("a project db"); + assert_eq!(sources.len(), 1); + let file = sources[0].1; + let mut db = db.0; + let Err(panic) = catch_unwind(AssertUnwindSafe(|| write(&mut db, file))) else { + return false; + }; + let message = panic + .downcast_ref::() + .map(String::as_str) + .or_else(|| panic.downcast_ref::<&str>().copied()) + .unwrap_or_default(); + message.contains("never-changing inputs cannot be mutated") + } + + /// the sources are frozen as `by check` freezes them, which only holds for a file indexed + /// after the freeze: one indexed before it stays mutable, and checking it costs what an + /// incremental db does + #[test] + fn the_sources_a_build_reads_never_change() { + assert!(refuses_as_frozen(|db, file| { + file.set_source_text_override(db).to(None); + })); + } + + #[test] + fn the_project_a_build_reads_never_changes() { + assert!(refuses_as_frozen(|db, _| { + db.project().set_included_paths(db, Vec::new()); + })); + } + + #[test] + fn a_build_never_has_open_files() { + assert!(refuses_as_frozen(|db, _| { + db.project() + .set_open_files(db, std::iter::empty().collect()); + })); + } #[test] fn a_hidden_directory_is_not_project_source() { diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 2fa3fc7d54..9b2fdfb0fa 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1338,17 +1338,17 @@ pub(crate) fn cmd_compile( Err(error) => break 'compiling Err(error), }; - let program_file = ty_python_semantic::Db::program_file(&db, *file); + let program_file = ty_python_semantic::Db::program_file(&*db, *file); let parsed = - ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); - let model = ty_python_semantic::SemanticModel::new(&db, program_file); + ruff_db::parsed::parsed_module(&*db, program_file.python_file(&*db)).load(&*db); + let model = ty_python_semantic::SemanticModel::new(&*db, program_file); // a `.py` source needs no transpiling to be its own interpreted fallback let mut options = options.clone(); if path.extension().is_some_and(|x| x == "py") { options.language = by_irbuild::Language::Python; } let mut lowered = by_irbuild::build_module( - &db, + &*db, &model.program_environment(), &model, parsed.suite(), From 0c52b69aef2560b093ad586d4cb47b587e252113 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:12:21 +1000 Subject: [PATCH 43/47] a tuple type with a labelled `name: T` field and no `/`, `*` or variadic is parsed as an anonymous named tuple wherever the named field comes, as `(int, name: T)` already was: `(int, str, name: T)` and `(int, local b: T)` used to parse as parameter lists, which the checker read as anonymous named tuples while the transpiler wrote them as plain `tuple[...]` annotations. the checker now makes an anonymous named tuple class only from a tuple the parser flagged as one, and a keyword variadic (`**kw: T`, `**: T`, `**P`) in a tuple type is `invalid-type-form`: it collects keyword arguments, which a tuple has no place for, so it belongs only in a callable's parameter list. before, `(int, **kw: T)` was read as an anonymous named tuple and `(int, /, **kw: T)` as `tuple[int, T]`, while the transpiler wrote both as `tuple[int]` --- .../tests/anon_named_tuple_runtime.rs | 24 ++++ .../src/parser/expression.rs | 23 ++-- crates/ruff_python_parser/src/parser/tests.rs | 42 +++++++ .../mdtest/basedpython_parameters.md | 14 +++ .../types/infer/builder/type_expression.rs | 103 ++++++++++++------ 5 files changed, 165 insertions(+), 41 deletions(-) diff --git a/crates/by_transforms/tests/anon_named_tuple_runtime.rs b/crates/by_transforms/tests/anon_named_tuple_runtime.rs index e2a6e45226..aafef442e9 100644 --- a/crates/by_transforms/tests/anon_named_tuple_runtime.rs +++ b/crates/by_transforms/tests/anon_named_tuple_runtime.rs @@ -203,6 +203,25 @@ assert kind(built) == type(built).__name__, "a named tuple passed as a tuple sta print("ok") "#; +/// a named field after more than one positional field, or after a modifier, makes the same +/// anonymous named tuple a named field right after the first does: the annotation is the class +/// a literal under it is built as, not a plain tuple +const NAMED_AFTER_POSITIONALS_PROGRAM: &str = r#" +def later(p: (int, str, name: bytes)) -> bytes: + return p.name + + +def borrowed(p: (int, local b: str)) -> str: + return p.b + + +assert later.__annotations__["p"]._fields == ("arg0", "arg1", "name"), "after two positional fields" +assert borrowed.__annotations__["p"]._fields == ("arg0", "b"), "after a modifier" +assert later((1, "a", b"x")) == b"x", "built" +assert borrowed((1, "y")) == "y", "built" +print("ok") +"#; + fn run(program: &str) { run_with( program, @@ -281,6 +300,11 @@ fn a_nested_field_and_a_union_arm_are_built() { run(NESTED_AND_UNION_PROGRAM); } +#[test] +fn a_named_field_anywhere_makes_an_anonymous_named_tuple() { + run(NAMED_AFTER_POSITIONALS_PROGRAM); +} + #[test] fn a_named_tuple_and_a_plain_tuple_are_left_as_they_are() { run(UNCHANGED_PROGRAM); diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 66676357ce..94e3beecd5 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -4830,13 +4830,20 @@ impl<'src> Parser<'src> { self.expect(TokenKind::Rpar); - // do not lift this tuple to is_anon_named_tuple even when it has - // plain `name: T` fields — markers (`/`, `*`) signal the user - // wants parameter-spec semantics (positional-only / keyword-only), - // which conflicts with the implicit NamedTuple synthesis. anon-NT - // lifting is reserved for tuples without markers (the existing - // `(name: T, ...)` and `(int, name: T)` mixed dispatch upstream) - let is_anon_named_tuple = false; + // a tuple with no marker (`/`, `*`) and no variadic is an anonymous named tuple + // whenever it has a `name: T` field, wherever the field comes — `(int, str, + // name: T)` as much as the `(int, name: T)` dispatched upstream. a marker or a + // variadic says the tuple is a parameter list, which no named tuple can spell + let is_anon_named_tuple = slash.is_none() + && star.is_none() + && elts + .iter() + .any(|elt| matches!(elt, Expr::Named(named) if named.label().is_some())) + && !elts.iter().any(|elt| match elt { + Expr::Starred(_) => true, + Expr::Named(named) => matches!(*named.target, Expr::Starred(_)), + _ => false, + }); ast::ExprTuple { elts, @@ -4846,7 +4853,7 @@ impl<'src> Parser<'src> { parenthesized: true, is_anon_named_tuple, is_anon_named_tuple_value: false, - is_parameter_shape: true, + is_parameter_shape: !is_anon_named_tuple, callable_shape: CallableParameterShape { slash, star, diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 6f465d86aa..aa4679c606 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -1967,6 +1967,48 @@ fn test_anon_named_tuple_mixed_type() { insta::assert_debug_snapshot!(parsed.syntax()); } +/// a named field makes the tuple an anonymous named tuple wherever it comes, not only right after +/// the first positional one, while a marker or a variadic keeps it a parameter list +#[test] +fn test_anon_named_tuple_named_field_after_positionals() { + let annotation_tuple = |source: &str| { + let parsed = parse_basedpython_module(source); + let Some(Stmt::AnnAssign(assign)) = parsed.suite().first().cloned() else { + panic!("expected an annotated assignment in {source:?}"); + }; + let Expr::Tuple(tuple) = *assign.annotation else { + panic!("expected a tuple annotation in {source:?}"); + }; + tuple + }; + for source in [ + "a: (int, str, name: bytes)\n", + "a: (int, local b: str)\n", + "a: (int, str, name: bytes, int)\n", + "a: (int, (b := str), name: bytes)\n", + ] { + let tuple = annotation_tuple(source); + assert!(tuple.is_anon_named_tuple, "{source:?}"); + assert!(!tuple.is_parameter_shape, "{source:?}"); + } + // a parenthesized walrus is a positional element, not a field: it binds its target + let tuple = annotation_tuple("a: (int, (b := str), name: bytes)\n"); + let Expr::Named(walrus) = &tuple.elts[1] else { + panic!("expected the walrus as the second element"); + }; + assert!(walrus.label().is_none()); + for source in [ + "a: (int, /, name: str)\n", + "a: (int, *, name: str)\n", + "a: (int, *args: str)\n", + "a: (int, **kw: str)\n", + ] { + let tuple = annotation_tuple(source); + assert!(!tuple.is_anon_named_tuple, "{source:?}"); + assert!(tuple.is_parameter_shape, "{source:?}"); + } +} + #[test] fn test_inline_protocol_type() { let source = "a: protocol(a: int; b: str; def f(self) -> int)\n"; diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_parameters.md b/crates/ty_python_semantic/resources/mdtest/basedpython_parameters.md index ac208dc604..e35018e000 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_parameters.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_parameters.md @@ -38,6 +38,20 @@ callable usage and is dropped when the tuple is used as a tuple type. c: (int, /, name: str) ``` +## a keyword variadic is not a tuple type + +A keyword variadic `**: T` collects keyword arguments, which a tuple has no place for, so it is only +valid in a callable's parameter list. Written in a tuple type it is rejected, whether or not the +tuple also has a marker or a positional variadic. + +```by +a: (int, **kw: str) # error: [invalid-type-form] +b: (**: str) # error: [invalid-type-form] +c: (int, /, **kw: str) # error: [invalid-type-form] +d: (int, *, name: str, **kw: bytes) # error: [invalid-type-form] +e: (int, *args: str, **kw: bytes) # error: [invalid-type-form] +``` + ## variadic tuple assignment A tuple value with arbitrarily many elements is assignable to a `(*: T)` annotation since the diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index ab329460ec..df79e40699 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -1017,40 +1017,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { .unwrap_or(class_lit); } - // basedpython parameter-shape tuple in type position. three - // sub-cases: - // * variadic (`(*args: T)`, `(int, *args: T, str)`, - // `(*: T)`): emit a real variable-length tuple type so - // assignability matches `tuple[T, ...]` semantics - // * fixed with markers (`(int, /, name: T)`): the markers - // signal parameter-spec semantics — drop names and emit - // a heterogeneous `tuple[...]`. matches the forward - // transpile lowering - // * fixed without markers but with named fields - // (`(int, name: T)`): handled upstream by - // `is_anon_named_tuple` dispatch if tuple.has_parameter_shape() { - if let Some(ty) = self.lower_parameter_shape_to_tuple_type(tuple) { - return ty; - } - let has_markers = - tuple.parameter_slash().is_some() || tuple.parameter_star().is_some(); - if has_markers { - let elt_tys: Vec> = tuple - .elts - .iter() - .map(|e| match e { - ast::Expr::Named(n) => self.infer_type_expression(&n.value), - _ => self.infer_type_expression(e), - }) - .collect(); - return Type::heterogeneous_tuple(self.db(), env, elt_tys); - } - let class_lit = self - .synthesize_anon_named_tuple_class(tuple, /* is_type_form = */ true); - return class_lit - .to_instance_approximation(self.db(), env) - .unwrap_or(class_lit); + return self.infer_parameter_shape_tuple_type_expression(tuple); } // basedpython: parenthesized tuples are valid type @@ -1913,6 +1881,59 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } } + /// basedpython: a parameter-shape tuple in type position. two sub-cases: + /// * variadic (`(*args: T)`, `(int, *args: T, str)`, `(*: T)`): a real variable-length + /// tuple type, so assignability matches `tuple[T, ...]` semantics + /// * fixed with markers (`(int, /, name: T)`): the names are dropped, giving a + /// heterogeneous `tuple[...]` + /// + /// a tuple with named fields and neither is an anonymous named tuple, which the parser flags + /// and [`Self::infer_type_expression_no_store`] handles. a keyword variadic collects keyword + /// arguments, which a tuple has no place for, so it is only valid in a parameter list + /// + /// kept out of [`Self::infer_type_expression_no_store`], which recurses once per nesting + /// level of a type expression: an unoptimized build gives every local of every arm its own + /// stack slot in that one frame + // an optimizing build would inline it back into that frame + #[inline(never)] + fn infer_parameter_shape_tuple_type_expression(&mut self, tuple: &ast::ExprTuple) -> Type<'db> { + if let Some(keyword_variadic) = tuple.elts.iter().find(|elt| is_keyword_variadic(elt)) { + if let Some(builder) = self + .context + .report_lint(&INVALID_TYPE_FORM, keyword_variadic) + { + builder.into_diagnostic( + "A keyword variadic is only valid in a callable's parameter list, not in a \ + tuple type", + ); + } + for elt in &tuple.elts { + let annotation = match elt { + ast::Expr::Named(named) => &*named.value, + ast::Expr::Starred(starred) => match &*starred.value { + ast::Expr::Starred(inner) => &*inner.value, + value => value, + }, + _ => elt, + }; + self.infer_type_expression(annotation); + } + return Type::unknown(); + } + if let Some(ty) = self.lower_parameter_shape_to_tuple_type(tuple) { + return ty; + } + let elt_tys: Vec> = tuple + .elts + .iter() + .map(|e| match e { + ast::Expr::Named(n) => self.infer_type_expression(&n.value), + _ => self.infer_type_expression(e), + }) + .collect(); + Type::heterogeneous_tuple(self.db(), self.program_environment(), elt_tys) + } + /// basedpython: the callable type of a method member of an inline protocol, /// `def f(self, x: int) -> str`. /// @@ -5270,3 +5291,19 @@ fn is_narrowing_predicate_place(expr: &ast::Expr) -> bool { _ => false, } } + +/// basedpython: whether `elt`, an element of a parameter-shape tuple, is a keyword variadic — +/// `**name: T`, `**: T` or a bare `**P` — rather than the bare-star `*: *`, which shares the +/// outer shape of the last two +fn is_keyword_variadic(elt: &ast::Expr) -> bool { + match elt { + ast::Expr::Named(named) => matches!( + &*named.target, + ast::Expr::Starred(starred) if starred.value.is_starred_expr() + ), + ast::Expr::Starred(starred) => { + starred.value.is_starred_expr() && !is_top_star_marker(&starred.value) + } + _ => false, + } +} From 2744854f955f27d44e59c069ef99dd38412740ff Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:13:09 +1000 Subject: [PATCH 44/47] the checker tells an anonymous named tuple class by its own anchor, `DynamicNamedTupleAnchor::Anonymous`, which only an anonymous named tuple form makes, instead of by a `_AnonNamedTuple_` prefix on the class's name. a `NamedTuple(...)` or `namedtuple(...)` call the program makes is an ordinary named tuple whatever it names its class: it displays under its own name, is not structurally assignable to an anonymous named tuple, and a plain tuple is no longer read as an instance of it, which the transpiler used to construct as a class of its own --- .../resources/mdtest/anonymous_named_tuple.md | 19 ++++++++ .../src/types/anon_named_tuple.rs | 17 ++----- .../src/types/class/named_tuple.rs | 37 +++++++++++++-- .../ty_python_semantic/src/types/display.rs | 45 ++++++++----------- .../src/types/infer/builder.rs | 20 +++++---- .../ty_python_semantic/src/types/instance.rs | 29 +++--------- 6 files changed, 92 insertions(+), 75 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md b/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md index 2db20e4684..3250523d90 100644 --- a/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md +++ b/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md @@ -236,3 +236,22 @@ a: (name: str, age: int) = ("a", 0) b: (name: str, age: int) = ("b", 1) b = a ``` + +## A named tuple the program makes is never anonymous + +Only an anonymous named tuple form makes an anonymous named tuple. A `NamedTuple` class the program +makes itself is an ordinary named tuple whatever it is called, even when its name is the one +basedpython gives the class it synthesizes, so a plain tuple is not read as an instance of it. + +```by +from typing import NamedTuple + +def instances[T](cls: type[T]) -> list[T]: + return [] + +xs = instances(NamedTuple("_AnonNamedTuple_deadbeef", [("a", int)])) +reveal_type(xs) # revealed: list[_AnonNamedTuple_deadbeef] + +# error: [invalid-argument-type] +xs.append((1,)) +``` diff --git a/crates/ty_python_semantic/src/types/anon_named_tuple.rs b/crates/ty_python_semantic/src/types/anon_named_tuple.rs index d5d57b8db1..941d26669e 100644 --- a/crates/ty_python_semantic/src/types/anon_named_tuple.rs +++ b/crates/ty_python_semantic/src/types/anon_named_tuple.rs @@ -10,14 +10,9 @@ use ruff_python_ast::name::Name; use salsa::plumbing::AsId; use crate::Db; -use crate::types::class::{ - ClassLiteral, DynamicNamedTupleAnchor, DynamicNamedTupleLiteral, NamedTupleSpec, -}; +use crate::types::class::{ClassLiteral, DynamicNamedTupleLiteral, NamedTupleSpec}; use crate::types::{ProgramEnvironment, Type, UnionType}; -/// the prefix of every synthesized anonymous named tuple class's name -const CLASS_PREFIX: &str = "_AnonNamedTuple_"; - /// the class and field specification of an anonymous named tuple `ty` is an instance /// of, or (for a type form in a value position) the class itself pub(crate) fn anon_named_tuple_class<'db>( @@ -33,14 +28,8 @@ pub(crate) fn anon_named_tuple_class<'db>( let ClassLiteral::DynamicNamedTuple(class) = literal else { return None; }; - if !class.name(db).as_str().starts_with(CLASS_PREFIX) { - return None; - } - match class.anchor(db) { - DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => Some((class, *spec)), - DynamicNamedTupleAnchor::CollectionsDefinition { .. } - | DynamicNamedTupleAnchor::TypingDefinition(_) => None, - } + let spec = class.anchor(db).anonymous_spec()?; + Some((class, spec)) } /// every anonymous named tuple instance `expected` allows, in the order its union diff --git a/crates/ty_python_semantic/src/types/class/named_tuple.rs b/crates/ty_python_semantic/src/types/class/named_tuple.rs index 6045953c08..87c56988d1 100644 --- a/crates/ty_python_semantic/src/types/class/named_tuple.rs +++ b/crates/ty_python_semantic/src/types/class/named_tuple.rs @@ -193,7 +193,8 @@ impl<'db> DynamicNamedTupleLiteral<'db> { match self.anchor(db) { DynamicNamedTupleAnchor::CollectionsDefinition { definition, .. } | DynamicNamedTupleAnchor::TypingDefinition(definition) => Some(*definition), - DynamicNamedTupleAnchor::ScopeOffset { .. } => None, + DynamicNamedTupleAnchor::ScopeOffset { .. } + | DynamicNamedTupleAnchor::Anonymous { .. } => None, } } @@ -202,7 +203,8 @@ impl<'db> DynamicNamedTupleLiteral<'db> { match self.anchor(db) { DynamicNamedTupleAnchor::CollectionsDefinition { definition, .. } | DynamicNamedTupleAnchor::TypingDefinition(definition) => definition.scope(db), - DynamicNamedTupleAnchor::ScopeOffset { scope, .. } => *scope, + DynamicNamedTupleAnchor::ScopeOffset { scope, .. } + | DynamicNamedTupleAnchor::Anonymous { scope, .. } => *scope, } } @@ -221,6 +223,10 @@ impl<'db> DynamicNamedTupleLiteral<'db> { DynamicNamedTupleAnchor::ScopeOffset { offset, .. } => { DynamicClassHeaderAnchor::ScopeOffset(*offset) } + // every form of this shape makes the class, so no one of them is its header + DynamicNamedTupleAnchor::Anonymous { .. } => { + DynamicClassHeaderAnchor::ScopeOffset(DynamicClassScopeOffset::Node(0)) + } }; dynamic_class_header_range(db, self.scope(db), anchor) } @@ -463,7 +469,8 @@ impl<'db> DynamicNamedTupleLiteral<'db> { match self.anchor(db) { DynamicNamedTupleAnchor::CollectionsDefinition { spec, .. } - | DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => *spec, + | DynamicNamedTupleAnchor::ScopeOffset { spec, .. } + | DynamicNamedTupleAnchor::Anonymous { spec, .. } => *spec, DynamicNamedTupleAnchor::TypingDefinition(definition) => deferred_spec(db, *definition), } } @@ -537,6 +544,15 @@ pub enum DynamicNamedTupleAnchor<'db> { offset: DynamicClassScopeOffset, spec: NamedTupleSpec<'db>, }, + + /// basedpython: the class an anonymous named tuple form `(name: T, ...)` or + /// `(name=v, ...)` makes, which no call the program makes can produce. its identity + /// is its shape: every form in a module with the same spec is the same class, anchored + /// at the module's scope + Anonymous { + scope: ScopeId<'db>, + spec: NamedTupleSpec<'db>, + }, } impl<'db> DynamicNamedTupleAnchor<'db> { @@ -562,6 +578,21 @@ impl<'db> DynamicNamedTupleAnchor<'db> { offset: *offset, spec: spec.recursive_type_normalized_impl(db, env, div, nested)?, }), + Self::Anonymous { scope, spec } => Some(Self::Anonymous { + scope: *scope, + spec: spec.recursive_type_normalized_impl(db, env, div, nested)?, + }), + } + } + + /// the field specification an anonymous named tuple form made this class from, or + /// `None` for a named tuple the program makes itself + pub(crate) fn anonymous_spec(&self) -> Option> { + match self { + Self::Anonymous { spec, .. } => Some(*spec), + Self::CollectionsDefinition { .. } + | Self::TypingDefinition(_) + | Self::ScopeOffset { .. } => None, } } } diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 9c1a382bfe..22ce39e910 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -22,7 +22,7 @@ use ty_module_resolver::{KnownModule, Module, file_to_module}; use crate::Db; use crate::place::{DefinedPlace, Place, builtins_symbol, global_symbol}; use crate::types::callable::CallableTypeKind; -use crate::types::class::{ClassLiteral, ClassType, DynamicNamedTupleAnchor, GenericAlias}; +use crate::types::class::{ClassLiteral, ClassType, GenericAlias}; use crate::types::constraints::ConstraintSetBuilder; use crate::types::function::{FunctionType, OverloadLiteral}; use crate::types::generics::{GenericContext, Specialization}; @@ -1371,34 +1371,27 @@ impl<'db> FmtDetailed<'db> for ClassDisplay<'_, 'db> { // class name. Positional fields use the synthetic `arg` name and // are rendered without a label if let ClassLiteral::DynamicNamedTuple(nt) = self.class - && nt.name(self.db).as_str().starts_with("_AnonNamedTuple_") + && let Some(spec) = nt.anchor(self.db).anonymous_spec() { - let spec = match nt.anchor(self.db) { - DynamicNamedTupleAnchor::CollectionsDefinition { spec, .. } - | DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => Some(*spec), - DynamicNamedTupleAnchor::TypingDefinition(_) => None, - }; - if let Some(spec) = spec { - let fields = spec.fields(self.db); - let ty = Type::ClassLiteral(self.class); - let mut f = f.with_type(ty); - f.write_char('(')?; - for (i, field) in fields.iter().enumerate() { - if i > 0 { - f.write_str(", ")?; - } - let synthetic_pos = field.name.as_str() == format!("arg{i}"); - if !synthetic_pos { - write!(f, "{}: ", field.name)?; - } - field - .ty - .display_with(self.db, env, self.settings.clone()) - .fmt_detailed(&mut f)?; + let fields = spec.fields(self.db); + let ty = Type::ClassLiteral(self.class); + let mut f = f.with_type(ty); + f.write_char('(')?; + for (i, field) in fields.iter().enumerate() { + if i > 0 { + f.write_str(", ")?; } - f.write_char(')')?; - return Ok(()); + let synthetic_pos = field.name.as_str() == format!("arg{i}"); + if !synthetic_pos { + write!(f, "{}: ", field.name)?; + } + field + .ty + .display_with(self.db, env, self.settings.clone()) + .fmt_detailed(&mut f)?; } + f.write_char(')')?; + return Ok(()); } let qualification_level = self.settings.qualified.get(&**self.class.name(self.db)); diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index febe55d921..ad11be70e4 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -66,9 +66,9 @@ use crate::types::call::{Argument, Binding, Bindings, CallArguments, CallError, use crate::types::call_type_forms::CallTypeForm; use crate::types::callable::CallableTypeKind; use crate::types::class::{ - ClassLiteral, CodeGeneratorKind, DynamicClassScopeOffset, DynamicNamedTupleAnchor, - DynamicNamedTupleLiteral, DynamicTypedDictAnchor, DynamicTypedDictLiteral, - FrozenDataclassDispatch, MethodDecorator, NamedTupleField, NamedTupleSpec, StaticClassLiteral, + ClassLiteral, CodeGeneratorKind, DynamicNamedTupleAnchor, DynamicNamedTupleLiteral, + DynamicTypedDictAnchor, DynamicTypedDictLiteral, FrozenDataclassDispatch, MethodDecorator, + NamedTupleField, NamedTupleSpec, StaticClassLiteral, }; use crate::types::constraints::{ConstraintSetBuilder, PathBounds, Solutions}; use crate::types::context::InferContext; @@ -8815,6 +8815,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Type::inline_protocol(self.db(), env, members, packs.into_boxed_slice()) } + /// the class of the anonymous named tuple form `tuple`, which the parser flagged as one — + /// `is_anon_named_tuple` for a type, `is_anon_named_tuple_value` for a value. it is the only + /// way the checker makes an anonymous named tuple class, so whether a type is one is read + /// off its [`DynamicNamedTupleAnchor::Anonymous`] anchor fn synthesize_anon_named_tuple_class( &mut self, tuple: &ast::ExprTuple, @@ -8881,14 +8885,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let class_name = ruff_python_ast::name::Name::new(format!("_AnonNamedTuple_{truncated:08x}")); - // Anchor at the module-scope with a constant offset of 0 so structural - // identity (driven entirely by `spec`) determines the synthesized - // class. Different shapes produce different specs; identical shapes - // unify across the file. + // Anchor at the module-scope so structural identity (driven entirely + // by `spec`) determines the synthesized class. Different shapes produce + // different specs; identical shapes unify across the file. let module_scope = global_scope(db, db.program_file(self.file())); - let anchor = DynamicNamedTupleAnchor::ScopeOffset { + let anchor = DynamicNamedTupleAnchor::Anonymous { scope: module_scope, - offset: DynamicClassScopeOffset::Node(0), spec, }; diff --git a/crates/ty_python_semantic/src/types/instance.rs b/crates/ty_python_semantic/src/types/instance.rs index d9cdd1d9bd..5a225458f3 100644 --- a/crates/ty_python_semantic/src/types/instance.rs +++ b/crates/ty_python_semantic/src/types/instance.rs @@ -17,7 +17,7 @@ use super::{ MaterializationKind, SubclassOfType, Type, TypeAliasType, }; use crate::place::PlaceAndQualifiers; -use crate::types::class::{DynamicNamedTupleAnchor, GenericAlias, StaticClassLiteral}; +use crate::types::class::{GenericAlias, StaticClassLiteral}; use crate::types::constraints::{ ConstraintSet, ConstraintSetBuilder, IteratorConstraintsExtension, OwnedConstraintSet, }; @@ -1261,29 +1261,12 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { source: ClassType<'db>, target: ClassType<'db>, ) -> Option> { - let source_nt = match source.class_literal(db) { - ClassLiteral::DynamicNamedTuple(nt) - if nt.name(db).as_str().starts_with("_AnonNamedTuple_") => - { - nt - } - _ => return None, - }; - let target_nt = match target.class_literal(db) { - ClassLiteral::DynamicNamedTuple(nt) - if nt.name(db).as_str().starts_with("_AnonNamedTuple_") => - { - nt - } - _ => return None, - }; - let spec_of = |nt: crate::types::class::DynamicNamedTupleLiteral<'db>| match nt.anchor(db) { - DynamicNamedTupleAnchor::CollectionsDefinition { spec, .. } - | DynamicNamedTupleAnchor::ScopeOffset { spec, .. } => Some(*spec), - DynamicNamedTupleAnchor::TypingDefinition(_) => None, + let anonymous_spec = |class: ClassType<'db>| match class.class_literal(db) { + ClassLiteral::DynamicNamedTuple(nt) => nt.anchor(db).anonymous_spec(), + _ => None, }; - let source_spec = spec_of(source_nt)?; - let target_spec = spec_of(target_nt)?; + let source_spec = anonymous_spec(source)?; + let target_spec = anonymous_spec(target)?; let source_fields = source_spec.fields(db); let target_fields = target_spec.fields(db); if source_fields.len() != target_fields.len() { From a72a6056139f89aca56f111f0bc22f6925aa493c Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 23 Oct 2026 00:53:10 +1000 Subject: [PATCH 45/47] a `.py` file that spells `===` or `!==` gets a syntax error, "identity operator `===` is not valid in .py files", as every other basedpython-only form does. the python-mode parser used to accept both silently, so `buff check` and `by check` reported nothing for a `.py` file python itself rejects, and a `===` left in transpiled output passed phase 3. the parser fixture that pinned their shape in python mode moves from `valid` to `invalid` --- .../invalid/expressions/identity_compare.py | 9 ++ .../valid/expressions/identity_compare.py | 10 --- .../src/parser/expression.rs | 6 ++ crates/ruff_python_parser/src/parser/tests.rs | 22 +++++ ...tax@expressions__identity_compare.py.snap} | 82 ++++++++++++++----- 5 files changed, 100 insertions(+), 29 deletions(-) create mode 100644 crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py delete mode 100644 crates/ruff_python_parser/resources/valid/expressions/identity_compare.py rename crates/ruff_python_parser/tests/snapshots/{valid_syntax@expressions__identity_compare.py.snap => invalid_syntax@expressions__identity_compare.py.snap} (74%) diff --git a/crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py b/crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py new file mode 100644 index 0000000000..280199b84e --- /dev/null +++ b/crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py @@ -0,0 +1,9 @@ +# basedpython identity-comparison operators. python has no such operator, so a +# `.py` file spelling one is a syntax error, while the comparison keeps its shape +x === y +x !== y + +# chained with each other. a chain mixing in the `is` keyword is rejected in +# basedpython mode, and has its own test in `parser::tests` +a === b !== c +a !== b diff --git a/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py b/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py deleted file mode 100644 index e460bb77b6..0000000000 --- a/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py +++ /dev/null @@ -1,10 +0,0 @@ -# basedpython identity-comparison operators. this fixture parses in python -# mode, where the lexer still produces the tokens — a `.py` file spelling them -# was never valid python, and the parser's job here is only to keep the shape -x === y -x !== y - -# chained with each other. a chain mixing in the `is` keyword is rejected in -# basedpython mode, and has its own test in `parser::tests` -a === b !== c -a !== b diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 94e3beecd5..363393511d 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -2315,6 +2315,9 @@ impl<'src> Parser<'src> { CmpOp::Is => { // accept either `is` or `===`; bump whichever the lexer emitted if self.at(TokenKind::EqEqEqual) { + self.error_if_not_basedpython( + "identity operator `===` is not valid in .py files".to_string(), + ); (TokenKind::EqEqEqual, None) } else { (TokenKind::Is, None) @@ -2323,6 +2326,9 @@ impl<'src> Parser<'src> { CmpOp::IsNot => { // accept either `is not` or `!==`; bump whichever the lexer emitted if self.at(TokenKind::BangEqEqual) { + self.error_if_not_basedpython( + "identity operator `!==` is not valid in .py files".to_string(), + ); (TokenKind::BangEqEqual, None) } else { (TokenKind::Is, Some(TokenKind::Not)) diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index aa4679c606..bb8d01a41e 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -72,6 +72,28 @@ fn identity_operators_are_recorded_per_operator() { ); } +/// `===` and `!==` are basedpython's identity operators. python has no such +/// operator, so a `.py` file spelling one is a syntax error there, as every +/// other basedpython-only form is +#[test] +fn identity_operators_are_not_python() { + for source in ["a === b\n", "a !== b\n", "a == b === c\n"] { + let parsed = parse_module(source); + let Err(error) = parsed else { + panic!("expected a syntax error for {source:?}"); + }; + assert!( + matches!(&error.error, ParseErrorType::BasedPythonOnly(message) if message.contains("is not valid in .py files")), + "{source:?}: {error}" + ); + } + assert!( + parse_basedpython_module_with_errors("a === b !== c\n") + .errors() + .is_empty() + ); +} + #[test] fn a_type_test_may_not_be_chained() { // python chains `a is int is str` into `a is int and int is str`, whose diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__identity_compare.py.snap similarity index 74% rename from crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap rename to crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__identity_compare.py.snap index cae10a971b..e64f560302 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__identity_compare.py.snap @@ -1,6 +1,6 @@ --- source: crates/ruff_python_parser/tests/fixtures.rs -input_file: crates/ruff_python_parser/resources/valid/expressions/identity_compare.py +input_file: crates/ruff_python_parser/resources/invalid/expressions/identity_compare.py --- ## AST @@ -8,20 +8,20 @@ input_file: crates/ruff_python_parser/resources/valid/expressions/identity_compa Module( ModModule { node_index: NodeIndex(None), - range: 0..410, + range: 0..337, body: [ Expr( StmtExpr { node_index: NodeIndex(None), - range: 234..241, + range: 161..168, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 234..241, + range: 161..168, left: Name( ExprName { node_index: NodeIndex(None), - range: 234..235, + range: 161..162, id: Name("x"), ctx: Load, }, @@ -33,7 +33,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 240..241, + range: 167..168, id: Name("y"), ctx: Load, }, @@ -53,15 +53,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 242..249, + range: 169..176, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 242..249, + range: 169..176, left: Name( ExprName { node_index: NodeIndex(None), - range: 242..243, + range: 169..170, id: Name("x"), ctx: Load, }, @@ -73,7 +73,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 248..249, + range: 175..176, id: Name("y"), ctx: Load, }, @@ -93,15 +93,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 388..401, + range: 315..328, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 388..401, + range: 315..328, left: Name( ExprName { node_index: NodeIndex(None), - range: 388..389, + range: 315..316, id: Name("a"), ctx: Load, }, @@ -114,7 +114,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 394..395, + range: 321..322, id: Name("b"), ctx: Load, }, @@ -122,7 +122,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 400..401, + range: 327..328, id: Name("c"), ctx: Load, }, @@ -143,15 +143,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 402..409, + range: 329..336, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 402..409, + range: 329..336, left: Name( ExprName { node_index: NodeIndex(None), - range: 402..403, + range: 329..330, id: Name("a"), ctx: Load, }, @@ -163,7 +163,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 408..409, + range: 335..336, id: Name("b"), ctx: Load, }, @@ -184,3 +184,47 @@ Module( }, ) ``` +## Errors + + | +1 | # basedpython identity-comparison operators. python has no such operator, so a +2 | # `.py` file spelling one is a syntax error, while the comparison keeps its shape +3 | x === y + | ^^^ Syntax Error: identity operator `===` is not valid in .py files +4 | x !== y + | + + + | +2 | # `.py` file spelling one is a syntax error, while the comparison keeps its shape +3 | x === y +4 | x !== y + | ^^^ Syntax Error: identity operator `!==` is not valid in .py files +5 | +6 | # chained with each other. a chain mixing in the `is` keyword is rejected in + | + + + | +6 | # chained with each other. a chain mixing in the `is` keyword is rejected in +7 | # basedpython mode, and has its own test in `parser::tests` +8 | a === b !== c + | ^^^ Syntax Error: identity operator `===` is not valid in .py files +9 | a !== b + | + + + | +6 | # chained with each other. a chain mixing in the `is` keyword is rejected in +7 | # basedpython mode, and has its own test in `parser::tests` +8 | a === b !== c + | ^^^ Syntax Error: identity operator `!==` is not valid in .py files +9 | a !== b + | + + + | +7 | # basedpython mode, and has its own test in `parser::tests` +8 | a === b !== c +9 | a !== b + | ^^^ Syntax Error: identity operator `!==` is not valid in .py files From b2a2cf4f65aace72c509b1553160f1920c52b34a Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:48:50 +1000 Subject: [PATCH 46/47] =?UTF-8?q?the=20anonymous-named-tuple=20cleanup=20t?= =?UTF-8?q?hat=20runs=20over=20a=20file's=20transpiled=20output=20parses?= =?UTF-8?q?=20it=20first=20and=20returns=20it=20untouched=20when=20it=20sp?= =?UTF-8?q?ells=20no=20anonymous=20named=20tuple,=20as=20a=20type=20or=20a?= =?UTF-8?q?s=20a=20value,=20and=20no=20inline=20protocol,=20instead=20of?= =?UTF-8?q?=20building=20a=20single-file=20db=20and=20inferring=20the=20wh?= =?UTF-8?q?ole=20output=20to=20ask=20which=20plain=20tuple=20literals=20th?= =?UTF-8?q?e=20checker=20reads=20as=20an=20anonymous=20named=20tuple.=20th?= =?UTF-8?q?ose=20two=20are=20the=20only=20syntax=20the=20cleanup=20lowers,?= =?UTF-8?q?=20and=20the=20checker=20makes=20an=20anonymous=20named=20tuple?= =?UTF-8?q?=20type=20only=20from=20a=20form=20the=20parser=20flags,=20mark?= =?UTF-8?q?ed=20by=20its=20`DynamicNamedTupleAnchor::Anonymous`=20anchor;?= =?UTF-8?q?=20the=20cleanup's=20db=20reads=20nothing=20but=20the=20output?= =?UTF-8?q?=20and=20the=20vendored=20stubs,=20and=20a=20test=20checks=20th?= =?UTF-8?q?at=20no=20vendored=20stub=20spells=20one.=20phase=203=20no=20lo?= =?UTF-8?q?nger=20scans=20its=20parse=20of=20the=20output=20for=20leftover?= =?UTF-8?q?=20basedpython=20syntax:=20a=20parse=20as=20python=20rejects=20?= =?UTF-8?q?every=20basedpython-only=20form,=20so=20that=20parse=20is=20the?= =?UTF-8?q?=20check.=20`VendoredFileSystem::read=5Fdirectory`=20lists=20th?= =?UTF-8?q?e=20archive's=20root.=20on=20neuromorph=20(213=20files,=20ABBA,?= =?UTF-8?q?=204=20rounds)=20`by=20build`=20goes=20from=2034.3=E2=80=9336.7?= =?UTF-8?q?s=20cpu=20/=201.26=E2=80=931.35=20GB=20peak=20to=2016.3?= =?UTF-8?q?=E2=80=9317.4s=20cpu=20/=200.71=E2=80=930.82=20GB,=20with=20byt?= =?UTF-8?q?e-identical=20output,=20and=20a=20differential=20`by=20transpil?= =?UTF-8?q?e`=20of=20the=206821=20distinct=20sandpit=20files=20at=20`requi?= =?UTF-8?q?res-python=20>=3D3.10`,=20with=20and=20without=20the=20early=20?= =?UTF-8?q?return,=20gives=20identical=20output,=20diagnostics=20and=20exi?= =?UTF-8?q?t=20codes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/by_transforms/src/lib.rs | 495 +++++++++++++----- crates/ruff_db/src/vendored.rs | 24 +- .../development/how-transpilation-works.md | 18 +- 3 files changed, 382 insertions(+), 155 deletions(-) diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 2995c7da1b..79d284cc7a 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -30,8 +30,8 @@ use std::collections::{BTreeSet, HashSet}; use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::{DbWithWritableSystem as _, SystemPath, SystemPathBuf}; use ruff_diagnostics::{Edit, Fix, IsolationLevel, SourceMap}; -use ruff_python_ast::Stmt; -use ruff_python_ast::visitor::Visitor; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{Expr, Stmt}; use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; use salsa::Setter as _; @@ -646,101 +646,187 @@ fn run_anon_named_tuple_cleanup( quotes: &WrittenQuotes, config: &Config, ) -> Result { - use ruff_python_ast::visitor::Visitor; - for _ in 0..4 { - let (db, file) = make_in_memory_db(&source); - let source_ref = ruff_db::source::source_text(&db, file); - let src = source_ref.as_str(); - let module = ruff_db::parsed::parsed_module( - &db, - ty_python_semantic::Db::program_file(&db, file).python_file(&db), - ) - .load(&db); - let model = ty_python_semantic::SemanticModel::new( - &db, - ty_python_semantic::Db::program_file(&db, file), - ); + match run_cleanup_round(&source, written, quotes, config)? { + CleanupRound::Skipped | CleanupRound::Unchanged => return Ok(source), + CleanupRound::Lowered(lowered) => source = lowered, + } + } + Ok(source) +} + +/// what one round of [`run_anon_named_tuple_cleanup`] did +#[derive(Debug, PartialEq, Eq)] +enum CleanupRound { + /// the output spells nothing the cleanup lowers, so the round built no db and asked the + /// checker nothing + Skipped, + /// the output was analysed and nothing in it needed lowering + Unchanged, + /// the output with what was left lowered, which the next round reads again + Lowered(String), +} - let mut anon = transforms::anon_named_tuple::AnonNamedTuple::new( - src, - written, - quotes, - &model, - config.clone(), - module.suite(), +/// one round of [`run_anon_named_tuple_cleanup`] over `source` +/// +/// the only syntax the round lowers is an anonymous named tuple form and an inline protocol, +/// and the only other thing it does is construct each plain tuple literal the checker reads as +/// an anonymous named tuple. the checker makes that type only from a form the parser flags +/// (`DynamicNamedTupleAnchor::Anonymous`), and the round's db reads nothing but `source` and +/// the vendored stubs, none of which spells one (`vendored_stubs_spell_no_anonymous_named_tuple` +/// checks that). so a `source` that spells neither has nothing to lower or construct, and the +/// round leaves it before building a db +fn run_cleanup_round( + source: &str, + written: WrittenNames, + quotes: &WrittenQuotes, + config: &Config, +) -> Result { + // read as basedpython, the way the round's db reads it + let parsed = ruff_python_parser::parse_unchecked_source( + source, + ruff_python_ast::PySourceType::BasedPython, + ); + if SurfaceSyntax::find(parsed.suite(), SurfaceSyntax::CLEANED_UP).is_none() { + return Ok(CleanupRound::Skipped); + } + let (db, file) = make_in_memory_db(source); + let source_ref = ruff_db::source::source_text(&db, file); + let src = source_ref.as_str(); + let module = ruff_db::parsed::parsed_module( + &db, + ty_python_semantic::Db::program_file(&db, file).python_file(&db), + ) + .load(&db); + let model = ty_python_semantic::SemanticModel::new( + &db, + ty_python_semantic::Db::program_file(&db, file), + ); + + let mut anon = transforms::anon_named_tuple::AnonNamedTuple::new( + src, + written, + quotes, + &model, + config.clone(), + module.suite(), + ); + for stmt in module.suite() { + anon.visit_stmt(stmt); + } + anon.finish(); + if let Some(err) = anon.errors.first() { + return Err(err.clone()); + } + let protocol = + transforms::protocol_type::cleanup(src, written, quotes, &model, module.suite(), config)?; + + if anon.edits.is_empty() && !anon.needs_import && protocol.is_none() { + return Ok(CleanupRound::Unchanged); + } + let mut preamble = String::new(); + if anon.needs_import { + // an earlier run over the pre-splice source already emitted a class + // (and its imports) for an anon-NT whose edit an AST pass then + // re-rendered away, so only what the spliced output is actually + // missing gets prepended. matched as a run of whole lines, the way + // the inline-protocol cleanup does + let source_lines: Vec<&str> = source.lines().collect(); + let push_missing = |preamble: &mut String, entry: &str| { + let entry_lines: Vec<&str> = entry.lines().collect(); + let present = !entry_lines.is_empty() + && source_lines + .windows(entry_lines.len()) + .any(|window| window == entry_lines.as_slice()); + if !present { + preamble.push_str(entry); + preamble.push('\n'); + } + }; + push_missing( + &mut preamble, + &written.import_from("typing", &["NamedTuple"]), ); - for stmt in module.suite() { - anon.visit_stmt(stmt); + let (imports, helpers) = anon.callable.take_requirements(); + for line in imports + .into_iter() + .chain(runtime_entries(config, &helpers, source)) + { + push_missing(&mut preamble, line.trim_end_matches('\n')); } - anon.finish(); - if let Some(err) = anon.errors.first() { - return Err(err.clone()); + for defs in [anon.callable.class_defs().to_owned(), anon.class_defs()] { + for class_def in defs.split_inclusive("\n\n") { + push_missing(&mut preamble, class_def.trim_end_matches('\n')); + } } - let protocol = transforms::protocol_type::cleanup( - src, - written, - quotes, - &model, - module.suite(), - config, - )?; - - if anon.edits.is_empty() && !anon.needs_import && protocol.is_none() { - return Ok(source); + } + let mut edits = anon.edits; + if let Some((protocol_edits, protocol_preamble)) = protocol { + preamble.push_str(&protocol_preamble); + edits.extend(protocol_edits); + } + + let (body, _) = apply_transforms_once(src, edits); + Ok(CleanupRound::Lowered(if preamble.is_empty() { + body + } else { + splice_preamble(&body, &preamble) + })) +} + +/// basedpython-only syntax [`run_anon_named_tuple_cleanup`] lowers when an earlier pass left it +/// in the output +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SurfaceSyntax { + /// an anonymous named tuple, as a type `(name: T, ...)` or as a value `(name=v, ...)` + AnonymousNamedTuple, + /// `protocol(...)` + InlineProtocol, +} + +impl SurfaceSyntax { + /// what [`run_anon_named_tuple_cleanup`] lowers + const CLEANED_UP: &[Self] = &[Self::AnonymousNamedTuple, Self::InlineProtocol]; + + fn of(expr: &Expr) -> Option { + match expr { + Expr::Tuple(tuple) if tuple.is_anon_named_tuple || tuple.is_anon_named_tuple_value => { + Some(Self::AnonymousNamedTuple) + } + Expr::ProtocolType(_) => Some(Self::InlineProtocol), + _ => None, } - let mut preamble = String::new(); - if anon.needs_import { - // an earlier run over the pre-splice source already emitted a class - // (and its imports) for an anon-NT whose edit an AST pass then - // re-rendered away, so only what the spliced output is actually - // missing gets prepended. matched as a run of whole lines, the way - // the inline-protocol cleanup does - let source_lines: Vec<&str> = source.lines().collect(); - let push_missing = |preamble: &mut String, entry: &str| { - let entry_lines: Vec<&str> = entry.lines().collect(); - let present = !entry_lines.is_empty() - && source_lines - .windows(entry_lines.len()) - .any(|window| window == entry_lines.as_slice()); - if !present { - preamble.push_str(entry); - preamble.push('\n'); + } + + /// the first expression in `stmts`, in source order, that spells one of `kinds` + fn find(stmts: &[Stmt], kinds: &[Self]) -> Option<(Self, TextRange)> { + struct Finder<'a> { + kinds: &'a [SurfaceSyntax], + found: Option<(SurfaceSyntax, TextRange)>, + } + impl<'ast> Visitor<'ast> for Finder<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + if self.found.is_none() { + walk_stmt(self, stmt); } - }; - push_missing( - &mut preamble, - &written.import_from("typing", &["NamedTuple"]), - ); - let (imports, helpers) = anon.callable.take_requirements(); - for line in imports - .into_iter() - .chain(runtime_entries(config, &helpers, &source)) - { - push_missing(&mut preamble, line.trim_end_matches('\n')); } - for defs in [anon.callable.class_defs().to_owned(), anon.class_defs()] { - for class_def in defs.split_inclusive("\n\n") { - push_missing(&mut preamble, class_def.trim_end_matches('\n')); + fn visit_expr(&mut self, expr: &'ast Expr) { + if self.found.is_some() { + return; + } + if let Some(kind) = SurfaceSyntax::of(expr) + && self.kinds.contains(&kind) + { + self.found = Some((kind, expr.range())); + return; } + walk_expr(self, expr); } } - let mut edits = anon.edits; - if let Some((protocol_edits, protocol_preamble)) = protocol { - preamble.push_str(&protocol_preamble); - edits.extend(protocol_edits); - } - - let (body, _) = apply_transforms_once(src, edits); - source = if preamble.is_empty() { - body - } else { - splice_preamble(&body, &preamble) - }; - - let _ = config; + let mut finder = Finder { kinds, found: None }; + finder.visit_body(stmts); + finder.found } - Ok(source) } /// Rewrite stdlib imports to `typing_extensions` where the imported name is @@ -1151,7 +1237,7 @@ impl From for TranspileError { } fn verify_syntax(source: &str, author: &AuthorNames) -> Result<(), TranspileError> { - use ruff_python_ast::{PySourceType, visitor::Visitor}; + use ruff_python_ast::PySourceType; let parsed = ruff_python_parser::parse_unchecked_source(source, PySourceType::Python); let parse_errors = parsed.errors(); @@ -1172,63 +1258,6 @@ fn verify_syntax(source: &str, author: &AuthorNames) -> Result<(), TranspileErro }); } - #[expect(clippy::items_after_statements, reason = "scanner colocated with use")] - struct AnonNamedTupleScanner { - leftover_range: Option, - leftover_typeof: Option, - } - #[expect(clippy::items_after_statements, reason = "scanner colocated with use")] - impl<'ast> Visitor<'ast> for AnonNamedTupleScanner { - fn visit_expr(&mut self, expr: &'ast ruff_python_ast::Expr) { - if self.leftover_range.is_some() || self.leftover_typeof.is_some() { - return; - } - if let ruff_python_ast::Expr::Tuple(t) = expr { - if t.is_anon_named_tuple || t.is_anon_named_tuple_value { - self.leftover_range = Some(::range(t)); - return; - } - } - if let ruff_python_ast::Expr::Subscript(s) = expr { - if s.is_typeof { - self.leftover_typeof = - Some(::range(s)); - return; - } - } - ruff_python_ast::visitor::walk_expr(self, expr); - } - } - - let mut scanner = AnonNamedTupleScanner { - leftover_range: None, - leftover_typeof: None, - }; - for stmt in parsed.suite() { - scanner.visit_stmt(stmt); - if scanner.leftover_range.is_some() || scanner.leftover_typeof.is_some() { - break; - } - } - if let Some(range) = scanner.leftover_range { - let snippet = &source[usize::from(range.start())..usize::from(range.end())]; - return Err(TranspileError { - kind: TranspileErrorKind::Bug, - message: format!("transpiler failed to lower anonymous named tuple syntax `{snippet}`"), - output_range: Some(range), - by_range: None, - }); - } - if let Some(range) = scanner.leftover_typeof { - let snippet = &source[usize::from(range.start())..usize::from(range.end())]; - return Err(TranspileError { - kind: TranspileErrorKind::Bug, - message: format!("transpiler failed to lower `typeof` syntax `{snippet}`"), - output_range: Some(range), - by_range: None, - }); - } - verify_runtime_helpers(parsed.suite(), &author.reads)?; verify_no_helper_name_clash(parsed.suite(), author)?; verify_no_shadowed_helper(parsed.suite(), author) @@ -3851,3 +3880,183 @@ mod synthesized_class_names { } } } + +#[cfg(test)] +mod leftover_surface_syntax { + use ruff_db::files::{File, system_path_to_file, vendored_path_to_file}; + use ruff_db::system::DbWithWritableSystem as _; + use ruff_db::vendored::{FileType, VendoredPathBuf}; + use ruff_python_ast::PySourceType; + + use super::{ + CleanupRound, Config, SurfaceSyntax, WrittenNames, WrittenQuotes, make_in_memory_db, + run_cleanup_round, + }; + + /// what the cleanup finds to lower in `source` + fn spells(source: &str) -> Option { + let parsed = ruff_python_parser::parse_unchecked_source(source, PySourceType::BasedPython); + SurfaceSyntax::find(parsed.suite(), SurfaceSyntax::CLEANED_UP).map(|(kind, _)| kind) + } + + #[test] + fn an_anonymous_named_tuple_type_is_left_over() { + assert_eq!( + spells("def f(p: list[(a: int, b: str)]) -> None: ...\n"), + Some(SurfaceSyntax::AnonymousNamedTuple) + ); + } + + #[test] + fn an_anonymous_named_tuple_value_is_left_over() { + assert_eq!( + spells("def f() -> object:\n return [(a=1, b=2)]\n"), + Some(SurfaceSyntax::AnonymousNamedTuple) + ); + } + + #[test] + fn an_inline_protocol_type_is_left_over() { + assert_eq!( + spells("def f(p: int | protocol(a: int)) -> None: ...\n"), + Some(SurfaceSyntax::InlineProtocol) + ); + } + + #[test] + fn an_inline_protocol_value_is_left_over() { + assert_eq!( + spells("class A:\n P = protocol(a: int)\n"), + Some(SurfaceSyntax::InlineProtocol) + ); + } + + /// phase 3 reads the output as python, which rejects every basedpython-only form, so one + /// an earlier pass left behind fails the transpile rather than reaching python + #[test] + fn phase_3_rejects_a_leftover_basedpython_form() { + for output in [ + "a = b = 1\nprint(a === b)\n", + "a = b = 1\nprint(a !== b)\n", + "x: (a: int)\n", + "x = (a=1)\n", + "P = protocol(a: int)\n", + "y = 1\nx: typeof y = y\n", + ] { + let error = super::verify_syntax(output, &super::AuthorNames::none()) + .expect_err("a leftover basedpython form must not pass phase 3"); + assert_eq!( + error.kind, + super::TranspileErrorKind::InvalidSyntax, + "{output:?}" + ); + assert!( + error.message.contains("is not valid in .py files"), + "{output:?}: {}", + error.message + ); + } + } + + /// python that names its own named tuple and protocol, and builds plain tuples + const PYTHON: &str = concat!( + "from typing import NamedTuple, Protocol\n", + "\n", + "class P(NamedTuple):\n", + " a: int\n", + "\n", + "class Q(Protocol):\n", + " a: int\n", + "\n", + "def f(p: P = P(1), q: tuple[int, int] = (1, 2)) -> P:\n", + " return P(*q[:1])\n", + ); + + #[test] + fn python_spells_none_of_it() { + assert_eq!(spells(PYTHON), None); + } + + fn round(source: &str) -> CleanupRound { + let quotes = WrittenQuotes::default(); + run_cleanup_round( + source, + WrittenNames::new(source), + "es, + &Config::test_default(), + ) + .expect("the round should succeed") + } + + /// the speedup: an output with nothing left to lower is passed over before any db is built + #[test] + fn a_round_over_python_builds_no_db() { + assert_eq!(round(PYTHON), CleanupRound::Skipped); + } + + #[test] + fn a_round_over_a_leftover_lowers_it() { + let CleanupRound::Lowered(lowered) = round("def f(p: (a: int)) -> int:\n return p.a\n") + else { + panic!("the leftover anonymous named tuple was not lowered"); + }; + assert_eq!(spells(&lowered), None, "{lowered}"); + } + + /// whether the db reads `file` as spelling an anonymous named tuple anywhere + fn spells_anonymous_named_tuple(db: &ty_project::TestDb, file: File) -> bool { + let module = ruff_db::parsed::parsed_module( + db, + ty_python_semantic::Db::program_file(db, file).python_file(db), + ) + .load(db); + SurfaceSyntax::find(module.suite(), &[SurfaceSyntax::AnonymousNamedTuple]).is_some() + } + + /// the cleanup passes over an output that spells no anonymous named tuple without asking + /// the checker which of its plain tuples it reads as one, which holds only while nothing + /// else its db reads spells one either + #[test] + fn vendored_stubs_spell_no_anonymous_named_tuple() { + let (mut db, _) = make_in_memory_db(""); + db.write_file("/control.byi", "def f(p: (a: int, b: str)) -> None: ...\n") + .expect("write file failed"); + let control = system_path_to_file(&db, "/control.byi").expect("file not in db"); + assert!( + spells_anonymous_named_tuple(&db, control), + "the scan must see a stub that does spell one" + ); + + let mut scanned = 0; + let mut spelling = Vec::new(); + let mut directories = vec![VendoredPathBuf::from("")]; + while let Some(directory) = directories.pop() { + let entries: Vec<_> = ruff_db::Db::vendored(&db) + .read_directory(&directory) + .collect(); + for entry in entries { + match entry.file_type() { + FileType::Directory => directories.push(entry.into_path()), + FileType::File => { + if !matches!(entry.path().extension(), Some("pyi" | "byi")) { + continue; + } + let file = vendored_path_to_file(&db, entry.path()) + .expect("a listed vendored file exists"); + scanned += 1; + if spells_anonymous_named_tuple(&db, file) { + spelling.push(entry.path().to_string()); + } + } + } + } + } + assert!(scanned > 0, "no vendored stub was scanned"); + assert!( + spelling.is_empty(), + "these vendored stubs spell an anonymous named tuple, so the cleanup in \ + `run_cleanup_round` can no longer pass over an output that spells none without \ + asking the checker: {spelling:?}" + ); + } +} diff --git a/crates/ruff_db/src/vendored.rs b/crates/ruff_db/src/vendored.rs index eafd696acd..bd817e130d 100644 --- a/crates/ruff_db/src/vendored.rs +++ b/crates/ruff_db/src/vendored.rs @@ -135,10 +135,13 @@ impl VendoredFileSystem { &self, dir: impl AsRef, ) -> impl Iterator + '_ { - let directory_prefix = NormalizedVendoredPath::from(dir.as_ref()) - .with_trailing_slash() - .0 - .into_owned(); + let normalized = NormalizedVendoredPath::from(dir.as_ref()); + // the archive's root has no name of its own: every entry is under it + let directory_prefix = if normalized.as_str().is_empty() { + String::new() + } else { + normalized.with_trailing_slash().0.into_owned() + }; self.inner.0.file_names().filter_map(move |name| { // Any entry that doesn't have the `path` (with a @@ -650,6 +653,19 @@ pub(crate) mod tests { "); } + #[test] + fn read_directory_root() { + let mock_typeshed = mock_typeshed(); + + for root in ["", ".", "./"] { + assert_eq!( + readdir_snapshot(&mock_typeshed, root), + "vendored://stdlib/", + "{root:?}" + ); + } + } + #[test] fn read_directory_asyncio() { let mock_typeshed = mock_typeshed(); diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index c41a42dae0..3f880c4d93 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -35,9 +35,13 @@ source (.by) │ when X is not yet stdlib at the configured min Python version │ ├─ phase 2b anon-named-tuple cleanup - │ └─ re-run `anon_named_tuple` on the post-lowering output to catch anon-NT - │ spans copied verbatim by other transforms (e.g. the PEP-695 polyfill). - │ bounded to a few iterations + │ └─ re-run `anon_named_tuple` and the inline-protocol lowering on the + │ post-lowering output to catch spans copied verbatim by other transforms + │ (e.g. the PEP-695 polyfill). bounded to a few iterations. an output that + │ spells no anonymous named tuple and no inline protocol is passed over + │ after a parse, without a db or any type inference: the checker makes an + │ anonymous named tuple type only from such a form, so no plain tuple in + │ it can need constructing as one │ ├─ phase 2c lazy-import marking │ └─ lower imports to the `lazy` keyword (3.15+) or a runtime polyfill @@ -51,11 +55,9 @@ source (.by) │ └─ phase 3 syntax verification ├─ parse the final output as `.py` — any parse error aborts with a - │ source-annotated diagnostic (the span is mapped back to `.by`) - ├─ scan the AST for leftover basedpython-only flags - │ (`is_anon_named_tuple`, `is_anon_named_tuple_value`, `is_typeof`). - │ a leftover flag means a transform failed to lower its construct; the - │ pipeline aborts rather than emit syntactically-valid-but-wrong Python + │ source-annotated diagnostic (the span is mapped back to `.by`). a + │ python parse rejects every basedpython-only form, so this is also + │ what catches one a transform failed to lower ├─ reject a call to one of the transpiler's own runtime helpers that the │ module never got. a transform emits the call and records the need in │ two different places, and forgetting the second half produces python From ba56e91d9d25d4fa1b83276ded531af96f377cff Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:54:20 +1000 Subject: [PATCH 47/47] ask whether a module-level name a lowering writes into code holds there the binding the checker resolved once the file is inferred, never from inside inference. inference plans a type test, a checked cast and a bare reified call or construction without asking, since the question needs an `InferredInCheckOrder` for the file, which only the check-order inference makes, and the methods that ask it live on `LoweringModel`, which inference never builds. it reports what does not depend on the answer, and records the site (`RuntimeReadSite`, by node index, in the diagnostics it already carries). `runtime_read_checks`, a query per scope that `check_types` runs after every scope is inferred, infers the scopes around the site first, asks the question with the same plans the transpiler uses, and reports `erased-type-check` and `unspecialized-reified-generic` for a name that does not hold, with the messages inference used to give. the module-level names a type test or a checked cast is written with reach `unused-import` from that query instead of from inference. asked inside the module's inference, the end-of-module question inferred every module-level condition after a type test, each inside the inference of the next: 37 levels of `analyze_predicate` in `parametric_protocol_checks_run_correctly`, which overflowed a 2 MiB test thread at opt-level 0. a reason that does not depend on the bindings (an unspellable argument, a protocol target) is now the one reported where both apply, and an erroneous test against a class the program does not bind narrows on its negative branch as an otherwise checkable one does --- crates/by_irbuild/src/lib.rs | 4 +- crates/by_irbuild/src/single_file.rs | 8 +- crates/by_transforms/src/lib.rs | 16 +- .../reverse_transforms/forward_references.rs | 2 +- .../src/transforms/ast_driver.rs | 2 +- .../src/transforms/repeated_underscore.rs | 2 +- .../by_transforms/src/transforms/soundness.rs | 2 +- crates/by_transforms/src/type_info.rs | 25 +- crates/ty/docs/rules.md | 408 +++++++++--------- crates/ty/src/by_commands.rs | 2 +- .../basedpython_type_checking_imports.md | 61 +++ ...that_\342\200\246_(18932f76614d8074).snap" | 39 ++ crates/ty_python_semantic/src/lib.rs | 2 +- crates/ty_python_semantic/src/place.rs | 11 +- .../ty_python_semantic/src/semantic_model.rs | 282 +++++++----- crates/ty_python_semantic/src/types.rs | 17 +- .../src/types/check_order.rs | 70 +++ .../ty_python_semantic/src/types/context.rs | 10 + .../src/types/diagnostic.rs | 244 +++++++++++ .../src/types/infer/builder.rs | 393 ++++------------- .../src/types/infer/tests.rs | 31 ++ .../src/types/lowering_reads.rs | 6 +- crates/ty_python_semantic/src/types/narrow.rs | 7 +- .../src/types/reified_infer.rs | 198 +++++---- .../src/types/runtime_reads.rs | 312 ++++++++++++++ .../src/types/runtime_spelling.rs | 56 ++- .../ty_python_semantic/src/types/soundness.rs | 15 +- .../src/types/soundness_sites.rs | 34 +- .../src/types/unused_imports.rs | 6 +- 29 files changed, 1453 insertions(+), 812 deletions(-) create mode 100644 "crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_che\342\200\246_-_basedpython___a_name_\342\200\246_-_A_reified_call_that_\342\200\246_(18932f76614d8074).snap" create mode 100644 crates/ty_python_semantic/src/types/check_order.rs create mode 100644 crates/ty_python_semantic/src/types/runtime_reads.rs diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 9abf134efe..af91d0d8de 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -145,7 +145,7 @@ use ty_python_semantic::ProgramEnvironment; use ty_python_semantic::types::soundness::{CheckKind, CheckTarget}; use ty_python_semantic::types::soundness_sites::SiteCheck; use ty_python_semantic::types::{KnownClass, SpecialFormType, Type, TypeDefinition, TypeQualifier}; -use ty_python_semantic::{HasType, SemanticModel}; +use ty_python_semantic::{HasType, LoweringModel, SemanticModel}; /// lower every module-level function ty can represent natively /// @@ -155,7 +155,7 @@ use ty_python_semantic::{HasType, SemanticModel}; pub fn build_module( db: &dyn ty_python_semantic::Db, env: &ProgramEnvironment<'_>, - model: &SemanticModel<'_>, + model: &LoweringModel<'_>, suite: &[Stmt], module_name: impl Into, options: impl Into, diff --git a/crates/by_irbuild/src/single_file.rs b/crates/by_irbuild/src/single_file.rs index 57885acecc..7474b9c231 100644 --- a/crates/by_irbuild/src/single_file.rs +++ b/crates/by_irbuild/src/single_file.rs @@ -11,7 +11,7 @@ use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; use ruff_python_ast::Stmt; use ty_project::{ProjectMetadata, TestDb}; -use ty_python_semantic::{ProgramEnvironment, SemanticModel}; +use ty_python_semantic::{LoweringModel, ProgramEnvironment}; use crate::Language; @@ -22,7 +22,7 @@ pub(crate) fn with_source( f: impl FnOnce( &dyn ty_python_semantic::Db, &ProgramEnvironment<'_>, - &SemanticModel<'_>, + &LoweringModel<'_>, &[Stmt], ) -> T, ) -> T { @@ -37,14 +37,14 @@ pub(crate) fn with_source_in( f: impl FnOnce( &dyn ty_python_semantic::Db, &ProgramEnvironment<'_>, - &SemanticModel<'_>, + &LoweringModel<'_>, &[Stmt], ) -> T, ) -> T { let (db, file) = make_db(source, language); let program_file = ty_python_semantic::Db::program_file(&db, file); let parsed = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); - let model = SemanticModel::new(&db, program_file); + let model = LoweringModel::new(&db, program_file); let env = ProgramEnvironment::from_file(program_file); f(&db, &env, &model, parsed.suite()) } diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 79d284cc7a..01c6ae8b26 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -132,7 +132,7 @@ fn run_context_sensitive_phase<'a>( if !parsed.errors().is_empty() { return std::borrow::Cow::Borrowed(source); } - let model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); + let model = ty_python_semantic::LoweringModel::new(db, db.program_file(file)); model.infer_in_check_order(); transforms::context_sensitive::qualify(source, parsed.suite(), &model) } @@ -160,7 +160,7 @@ fn run_erased_union_phase<'a>( if !parsed.errors().is_empty() { return std::borrow::Cow::Borrowed(source); } - let model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); + let model = ty_python_semantic::LoweringModel::new(db, db.program_file(file)); model.infer_in_check_order(); transforms::erased_union::reify(source, parsed.suite(), &model, config.min_version) } @@ -287,7 +287,7 @@ fn transpile_with_report( ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); - let model = ty_python_semantic::SemanticModel::new( + let model = ty_python_semantic::LoweringModel::new( &db, ty_python_semantic::Db::program_file(&db, file), ); @@ -511,7 +511,7 @@ pub fn transpile_typed_with_report( }; // which imports must stay eager, computed against the *project* db: a // single-file db cannot resolve the modules that declare the conformances - let eager_model = ty_python_semantic::SemanticModel::new(db, db.program_file(file)); + let eager_model = ty_python_semantic::LoweringModel::new(db, db.program_file(file)); let eager_imports = eager_model.eagerly_imported_modules(); let eager_names = eager_model.eagerly_imported_names(); let (spliced, ast_errors, phase0_map) = transforms::ast_driver::run_against_source( @@ -698,7 +698,7 @@ fn run_cleanup_round( ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); - let model = ty_python_semantic::SemanticModel::new( + let model = ty_python_semantic::LoweringModel::new( &db, ty_python_semantic::Db::program_file(&db, file), ); @@ -904,7 +904,7 @@ fn run_lazy_import_phase( ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); - let model = ty_python_semantic::SemanticModel::new( + let model = ty_python_semantic::LoweringModel::new( &db, ty_python_semantic::Db::program_file(&db, file), ); @@ -1703,7 +1703,7 @@ pub fn optional_marker_edits(source: &str) -> Vec<(TextRange, String)> { ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); - let model = ty_python_semantic::SemanticModel::new( + let model = ty_python_semantic::LoweringModel::new( &db, ty_python_semantic::Db::program_file(&db, file), ); @@ -1734,7 +1734,7 @@ pub fn reverse_transpile(source: &str, config: &Config) -> Result String { if !module.errors().is_empty() { return source.to_owned(); } - let model = ty_python_semantic::SemanticModel::new(&db, program_file); + let model = ty_python_semantic::LoweringModel::new(&db, program_file); let ranges = literal_type_strings(source, module.suite(), &model); let mut strings = Strings { ranges: &ranges, diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 8317b3d074..8a86698bf7 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -1266,7 +1266,7 @@ pub(crate) fn run_against_source<'a>( ruff_db::parsed::parsed_module(sem_db, sem_db.program_file(sem_file).python_file(sem_db)) .load(sem_db); let semantic_model = - ty_python_semantic::SemanticModel::new(sem_db, sem_db.program_file(sem_file)); + ty_python_semantic::LoweringModel::new(sem_db, sem_db.program_file(sem_file)); // identity line table for the no-change early returns: stripping variance // is within-line, so every line still maps to itself diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index 01f094e555..a4e87ee078 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -98,7 +98,7 @@ pub(crate) fn collect(suite: &[Stmt], types: &dyn TypeInfo, slash: bool) -> Unde /// reads the same answer: [`WrittenNames::with_lowerings`] these, and every name and `/` /// it writes is the transpiler's pub fn repeated_underscore_lowerings( - model: &ty_python_semantic::SemanticModel<'_>, + model: &ty_python_semantic::LoweringModel<'_>, suite: &[Stmt], ) -> UnderscoreLowerings { collect(suite, model, true) diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index 86e1aac129..93555ec2c7 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -215,7 +215,7 @@ impl SoundnessSites { /// decide every soundness check `suite` makes under `positions`. `written` is the /// module `suite` is parsed from, as it was written pub fn soundness_sites( - model: &ty_python_semantic::SemanticModel<'_>, + model: &ty_python_semantic::LoweringModel<'_>, written: WrittenNames, suite: &[Stmt], positions: SoundnessPositions, diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 35a76e5e32..658c9989f0 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -21,7 +21,7 @@ use ty_python_semantic::types::{ DisplaySettings, DynamicType, KnownClass, KnownInstanceType, Type, UnpackedKwargs, character, }; use ty_python_semantic::{ - AnonNamedTupleClass, AnonNamedTupleId, Db, HasType, ImplicitReceiverReference, + AnonNamedTupleClass, AnonNamedTupleId, Db, HasType, ImplicitReceiverReference, LoweringModel, ProgramEnvironment, SemanticModel, }; @@ -874,7 +874,7 @@ pub(crate) use ty_python_semantic::types::soundness::CastCheck; /// re-export of the ty-side framework role so transforms name a single type pub(crate) use ty_python_semantic::types::FrameworkRole; -impl TypeInfo for SemanticModel<'_> { +impl TypeInfo for LoweringModel<'_> { fn is_type_fn_application(&self, expr: &Expr) -> bool { let Expr::Subscript(subscript) = expr else { return false; @@ -999,7 +999,7 @@ impl TypeInfo for SemanticModel<'_> { &self, call: &ruff_python_ast::ExprCall, ) -> Option { - SemanticModel::reified_call_specialization(self, call) + LoweringModel::reified_call_specialization(self, call) } fn constructor_specialization( @@ -1022,7 +1022,7 @@ impl TypeInfo for SemanticModel<'_> { lhs: &Expr, rhs: &Expr, ) -> Option { - SemanticModel::parametric_is_plan(self, lhs, rhs) + LoweringModel::parametric_is_plan(self, lhs, rhs) } fn is_plain_value(&self, expr: &Expr) -> bool { @@ -1502,18 +1502,11 @@ impl TypeInfo for SemanticModel<'_> { } fn reads_names_bound_where_written(&self, expr: &Expr) -> bool { - SemanticModel::reads_names_bound_where_written(self, expr) + LoweringModel::reads_names_bound_where_written(self, expr) } fn cast_check_plan(&self, type_expr: &Expr) -> Option { - let ty = type_expr.inferred_type(self)?; - ty_python_semantic::types::soundness::cast_check_plan( - self.db(), - &self.program_environment(), - self.file(), - ty, - Some(type_expr), - ) + LoweringModel::checked_cast_plan(self, type_expr) } fn cast_is_redundant(&self, value: &Expr, target: &Expr) -> bool { @@ -1761,7 +1754,7 @@ fn display_for_python<'db>( /// `class` with each field's type spelled for the emitted python fn anon_named_tuple_construction<'db>( - model: &SemanticModel<'db>, + model: &LoweringModel<'db>, class: AnonNamedTupleClass<'db>, min_version: ruff_python_ast::PythonVersion, ) -> AnonNamedTupleConstruction { @@ -1818,7 +1811,7 @@ fn anon_named_tuple_construction<'db>( fn spell_for_python<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, - model: &SemanticModel<'db>, + model: &LoweringModel<'db>, ty: Type<'db>, min_version: ruff_python_ast::PythonVersion, literal: bool, @@ -1921,7 +1914,7 @@ fn typing_name(name: &str, literal: bool) -> Option<&'static str> { /// signature it annotates. Anything else would reach the output unresolved, so the spelling /// is refused rather than written with a name that is not there. fn resolvable_in_output( - model: &SemanticModel<'_>, + model: &LoweringModel<'_>, min_version: ruff_python_ast::PythonVersion, spelled: &SynthesizedType, bare_classes: &[String], diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index 6730aacd29..f0b9da6c16 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -8,7 +8,7 @@ Default level: error · Added in 0.0.64 · Related issues · -View source +View source @@ -44,7 +44,7 @@ class Base(ABC): Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -90,7 +90,7 @@ class Derived(Base): # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -123,7 +123,7 @@ f(1, b=s1) # ok — explicit Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -164,7 +164,7 @@ report(Celsius()) # error: two conversions apply Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -199,7 +199,7 @@ extension list: Default level: warn · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -263,7 +263,7 @@ class SubProto(BaseProto, Protocol): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -345,7 +345,7 @@ value = unknown # ty: ignore[unresolved-reference] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -388,7 +388,7 @@ a4 = True + 1 # ok — a boolean used as a boolean Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -440,7 +440,7 @@ Foo.method() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -468,7 +468,7 @@ Calling a non-callable object will raise a `TypeError` at runtime. Default level: error · Added in 0.0.7 · Related issues · -View source +View source @@ -503,7 +503,7 @@ def f(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -551,7 +551,7 @@ def main(): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -598,7 +598,7 @@ def Fixed(show: bool): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -631,7 +631,7 @@ a = 1 # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -665,7 +665,7 @@ class C(A, B): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -711,7 +711,7 @@ def App(done: bool) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -745,7 +745,7 @@ class B(A): ... # error Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -790,7 +790,7 @@ type Tree = int | list[Tree] # valid recursive alias Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -826,7 +826,7 @@ class Example: Default level: warn · Added in 0.0.1-alpha.16 · Related issues · -View source +View source @@ -865,7 +865,7 @@ old_func() # error: [deprecated] Default level: ignore · Added in 0.0.78 · Related issues · -View source +View source @@ -1037,7 +1037,7 @@ soundness checks from their type checker, and it may have false positives in som Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1069,7 +1069,7 @@ This rule is currently disabled by default because of the number of false positi Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1100,7 +1100,7 @@ class B(A, A): ... # error Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -1140,7 +1140,7 @@ class A: # error Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -1251,7 +1251,7 @@ Python code. Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -1300,7 +1300,7 @@ def bar() -> str: # error: [empty-body] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -1361,7 +1361,7 @@ def h(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -1463,7 +1463,7 @@ def foo() -> "intt\b": ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1502,7 +1502,7 @@ def f(local fn: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1543,7 +1543,7 @@ for x in [1, 2, 3]: Default level: warn · Added in 0.0.50 · Related issues · -View source +View source @@ -1583,7 +1583,7 @@ def g(value: ~A) -> None: ... # error: [experimental-syntax] Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -1617,7 +1617,7 @@ def my_function() -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.40 · Related issues · -View source +View source @@ -1652,7 +1652,7 @@ let a = 1 Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -1769,7 +1769,7 @@ def test() -> "Literal[5]": Default level: ignore · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -1822,7 +1822,7 @@ unpacking — is not a declaration, and is never reported. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -1897,7 +1897,7 @@ print(Labelled) # warning: prints `` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -1936,7 +1936,7 @@ def audit(account: Account) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1972,7 +1972,7 @@ class C(A, B): ... # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2001,7 +2001,7 @@ t[3] # error Default level: warn · Added in 0.0.1-alpha.33 · Related issues · -View source +View source @@ -2038,7 +2038,7 @@ class MyClass: ... Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2074,7 +2074,7 @@ class Point: Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -2169,7 +2169,7 @@ will produce instances with an atypical memory layout. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2201,7 +2201,7 @@ func("foo") # error: [invalid-argument-type] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2231,7 +2231,7 @@ a: int = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2312,7 +2312,7 @@ box.value = 1 # okay Default level: error · Added in 0.0.33 · Related issues · -View source +View source @@ -2357,7 +2357,7 @@ class Sub(Base): Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -2399,7 +2399,7 @@ asyncio.run(main()) Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2426,7 +2426,7 @@ class A(42): ... # error: [invalid-base] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2464,7 +2464,7 @@ build: # error: `build` is an experimental feature, and is off Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.5 · Related issues · -View source +View source @@ -2501,7 +2501,7 @@ extension str(A): # error: `str` does not answer every member of `A` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2530,7 +2530,7 @@ with 1: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -2563,7 +2563,7 @@ class Fahrenheit: Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -2616,7 +2616,7 @@ See: Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -2652,7 +2652,7 @@ class A: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2684,7 +2684,7 @@ a: str # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.82 · Related issues · -View source +View source @@ -2755,7 +2755,7 @@ decorator def traced(fn: (...) -> object, label: str = "") -> int: Default level: warn · Added in 0.0.20 · Related issues · -View source +View source @@ -2811,7 +2811,7 @@ class Pet(Enum): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2876,7 +2876,7 @@ This rule corresponds to Ruff's Default level: error · Added in 0.0.1-alpha.28 · Related issues · -View source +View source @@ -2930,7 +2930,7 @@ class D(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -2963,7 +2963,7 @@ extension list[T: int]: # error: `list` declares no type parameter `T` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -2992,7 +2992,7 @@ Author.objects.filter(name__startswith=1) # error: lookup wants `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -3028,7 +3028,7 @@ def test_user(user: int) -> None: # error: fixture provides `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -3076,7 +3076,7 @@ f"{'name':>10}" # ok Default level: error · Added in 0.0.1-alpha.35 · Related issues · -View source +View source @@ -3126,7 +3126,7 @@ class NonFrozenChild(FrozenBase): # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3181,7 +3181,7 @@ class E(Generic[V]): Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -3276,7 +3276,7 @@ a = 20 / 0 # type: ignore Default level: error · Added in 0.0.1-alpha.17 · Related issues · -View source +View source @@ -3323,7 +3323,7 @@ carol = Person(name="Carol", aeg=25) # typo! Default level: warn · Added in 0.0.15 · Related issues · -View source +View source @@ -3384,7 +3384,7 @@ def f(x, y, /): # Python 3.8+ syntax Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3424,7 +3424,7 @@ def f(t: TypeVar("U")): ... # ty: ignore[invalid-type-form] Default level: error · Added in 0.0.18 · Related issues · -View source +View source @@ -3475,7 +3475,7 @@ match object(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3509,7 +3509,7 @@ class B(metaclass=42): ... # error Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -3626,7 +3626,7 @@ Correct use of `@override` is enforced by ty's [`invalid-explicit-override`](#in Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -3669,7 +3669,7 @@ implements Backend # error: `Backend` is not a protocol Default level: error · Added in 0.0.72 · Related issues · -View source +View source @@ -3707,7 +3707,7 @@ from module import missing # error Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -3772,7 +3772,7 @@ TypeError: typing.ClassVar[int] is not valid as type argument Default level: warn · Added in 0.0.31 · Related issues · -View source +View source @@ -3818,7 +3818,7 @@ admin[0] # "Alice" Default level: error · Added in 0.0.1-alpha.27 · Related issues · -View source +View source @@ -3856,7 +3856,7 @@ Baz = NewType("Baz", int | str) # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3933,7 +3933,7 @@ def foo(x: object = "a", y: int = 0) -> object: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -3971,7 +3971,7 @@ B().f() # 1, not 2 Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4026,7 +4026,7 @@ def h(a: S = 1) -> S: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4059,7 +4059,7 @@ def test_add(a: int, b: int) -> None: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4095,7 +4095,7 @@ P2 = ParamSpec() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4147,7 +4147,7 @@ Declare the type variable with `TypeVar("T", covariant=True)` instead. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4218,7 +4218,7 @@ def g(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -4253,7 +4253,7 @@ def g[T](value: T) raises T: # error: `T@g` is not always an exception Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4286,7 +4286,7 @@ if m := re.match("(a)(b)", "ab"): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -4318,7 +4318,7 @@ type Alias[reified T] = list[T] # error: an alias's parameters are erased Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.83 · Related issues · -View source +View source @@ -4389,7 +4389,7 @@ class Override(Base): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4535,7 +4535,7 @@ def detail(request, pk: int): ... # ok Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -4574,7 +4574,7 @@ import "data/missing.json" as missing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4682,7 +4682,7 @@ class C: ... Default level: error · Added in 0.0.10 · Related issues · -View source +View source @@ -4733,7 +4733,7 @@ class MyClass: Default level: error · Added in 0.0.1-alpha.6 · Related issues · -View source +View source @@ -4779,7 +4779,7 @@ InvalidAlias = TypeAliasType("InvalidAlias", list[T], type_params=(list[T],)) # Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -4845,7 +4845,7 @@ Bar[int] # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4878,7 +4878,7 @@ TYPE_CHECKING = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4913,7 +4913,7 @@ b: Annotated[int] # error Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -4970,7 +4970,7 @@ class C: Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -5036,7 +5036,7 @@ class Owner[U]: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5093,7 +5093,7 @@ V = TypeVar("V", list[int], int) # valid constrained Type Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -5135,7 +5135,7 @@ U = TypeVar("U", int, str, default=bytes) # error: [invalid-type-variable-defau Default level: error · Added in 0.0.28 · Related issues · -View source +View source @@ -5171,7 +5171,7 @@ class Child(Base): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5212,7 +5212,7 @@ def f(options: dict[str, object]): Default level: error · Added in 0.0.9 · Related issues · -View source +View source @@ -5246,7 +5246,7 @@ class Foo(TypedDict): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5287,7 +5287,7 @@ type Alias[out T] = list[T] # error: `list` is invariant Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -5327,7 +5327,7 @@ class Point: Default level: error · Added in 0.0.25 · Related issues · -View source +View source @@ -5361,7 +5361,7 @@ def gen() -> Iterator[int]: Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5427,7 +5427,7 @@ def h(arg2: type): Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -5476,7 +5476,7 @@ def g(arg: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -5507,7 +5507,7 @@ def f(s: str): Default level: warn · Added in 0.0.30 · Related issues · -View source +View source @@ -5549,7 +5549,7 @@ Movie = TypedDict("Film", {"title": str}) # error: [mismatched-type-name] Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -5617,7 +5617,7 @@ and nothing is reported. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5648,7 +5648,7 @@ func() # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5687,7 +5687,7 @@ def g(): ... Default level: ignore · Preview (since 0.0.76) · Related issues · -View source +View source @@ -5776,7 +5776,7 @@ Add `urllib3` to `project.dependencies` if your code imports it directly. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -5804,7 +5804,7 @@ from django.db import models # warning: install `django-stubs` for precise type Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.81 · Related issues · -View source +View source @@ -5856,7 +5856,7 @@ def ignore(event: str): ... Default level: error · Level under ty-compatible: ignore · Added in 0.0.41 · Related issues · -View source +View source @@ -5917,7 +5917,7 @@ class ExplicitChild(Parent): Default level: error · Added in 0.0.75 · Related issues · -View source +View source @@ -6004,7 +6004,7 @@ class Item: Default level: error · Level under ty-compatible: ignore · Added in 0.0.45 · Related issues · -View source +View source @@ -6042,7 +6042,7 @@ def handle(m: re.Match[str]) -> str: Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -6081,7 +6081,7 @@ alice["age"] # KeyError Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -6128,7 +6128,7 @@ def App(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6161,7 +6161,7 @@ def f(a: int | None): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6199,7 +6199,7 @@ func("string") # error: [no-matching-overload] Default level: error · Added in 0.0.30 · Related issues · -View source +View source @@ -6236,7 +6236,7 @@ class Sub(Super): ... # error: [non-callable-init-subclass] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -6267,7 +6267,7 @@ def f(x: int | str) -> int: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6296,7 +6296,7 @@ def f(a: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6340,7 +6340,7 @@ def g(o: object, shape: Shape): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6369,7 +6369,7 @@ for i in 34: # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6397,7 +6397,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6426,7 +6426,7 @@ def f(once done: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6454,7 +6454,7 @@ def f(once done: () -> None): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6496,7 +6496,7 @@ def f(x: int?): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6551,7 +6551,7 @@ def g(name: str | None): Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -6588,7 +6588,7 @@ class B(A): Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -6625,7 +6625,7 @@ class B(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.38 · Related issues · -View source +View source @@ -6668,7 +6668,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6699,7 +6699,7 @@ f(1, x=2) # error Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6730,7 +6730,7 @@ f(x=1) # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6768,7 +6768,7 @@ A.c # error Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6806,7 +6806,7 @@ A()[0] # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6850,7 +6850,7 @@ from module import a # error Default level: warn · Added in 0.0.23 · Related issues · -View source +View source @@ -6882,7 +6882,7 @@ html.parser # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6918,7 +6918,7 @@ print(x) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -6957,7 +6957,7 @@ Id("x") # error: `Id`'s constructor is private Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -6988,7 +6988,7 @@ __all__ = ["helper"] # error: `helper` is private Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7020,7 +7020,7 @@ from helpers import Key # error: `Key` is private to `helpers` Default level: warn · Added in 0.0.60 · Related issues · -View source +View source @@ -7095,7 +7095,7 @@ def test() -> "int": Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7137,7 +7137,7 @@ def g(a: bool | None): Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7172,7 +7172,7 @@ cast(int, f()) # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7226,7 +7226,7 @@ if sys.version_info >= (3, 12): # ok — artificially constant Default level: warn · Added in 0.0.18 · Related issues · -View source +View source @@ -7264,7 +7264,7 @@ class C: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7319,7 +7319,7 @@ class Sub(Base): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7368,7 +7368,7 @@ def f(value: int | str) -> int: Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -7432,7 +7432,7 @@ def g(values: tuple[int, ...]) -> None: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -7465,7 +7465,7 @@ class C: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -7501,7 +7501,7 @@ class C[T]: Default level: warn · Added in 0.0.71 · Related issues · -View source +View source @@ -7544,7 +7544,7 @@ def build(t: Tag) -> None: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -7588,7 +7588,7 @@ class Outer[T]: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7637,7 +7637,7 @@ def Observed(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7681,7 +7681,7 @@ def Counter(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7715,7 +7715,7 @@ static_assert(int(2.0 * 3.0) == 6) # error Default level: warn · Added in 0.0.39 · Related issues · -View source +View source @@ -7767,7 +7767,7 @@ limitation. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7801,7 +7801,7 @@ class B(A): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7833,7 +7833,7 @@ class Circle(Shape): ... # error: `Shape` is sealed in another workspace Default level: error · Added in 0.0.1-alpha.30 · Related issues · -View source +View source @@ -7942,7 +7942,7 @@ class Book: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7972,7 +7972,7 @@ f("foo") # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8011,7 +8011,7 @@ def find(items: list[int]) -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8058,7 +8058,7 @@ g: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8093,7 +8093,7 @@ f: # error: the block returns `None`, not `str` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8132,7 +8132,7 @@ def _(x: int): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -8163,7 +8163,7 @@ class User(BaseModel): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8222,7 +8222,7 @@ class A: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -8295,7 +8295,7 @@ the project registers with `@register.simple_block_tag`. Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -8361,7 +8361,7 @@ what the projects depending on it read. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8390,7 +8390,7 @@ def f() raises TypeError: Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8419,7 +8419,7 @@ reveal_type(1) # revealed: Literal[1] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8457,7 +8457,7 @@ def check(x: int | None) -> asserts x: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8487,7 +8487,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8518,7 +8518,7 @@ f(x=1, y=2) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -8729,7 +8729,7 @@ page does not render at all. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -8763,7 +8763,7 @@ implements Backend # error: this module does not answer `Backend` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8822,7 +8822,7 @@ def Held(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8855,7 +8855,7 @@ A().foo # error Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -8930,7 +8930,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8958,7 +8958,7 @@ import foo # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8987,7 +8987,7 @@ def check(value: int | None) -> asserts values: # error: `values` is nothing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9105,7 +9105,7 @@ is one whose template set cannot be established. Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -9233,7 +9233,7 @@ Python code. Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -9274,7 +9274,7 @@ def f(a: object, b: int, c: Any): Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9419,7 +9419,7 @@ Python code. Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9566,7 +9566,7 @@ generator boundaries. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -9616,7 +9616,7 @@ A() # error: nothing says which specialization this is Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -9663,7 +9663,7 @@ def Observed(items: StateList[int]): ... # ok: an observable handle Default level: warn · Added in 0.0.1-alpha.7 · Related issues · -View source +View source @@ -9709,7 +9709,7 @@ class D(C): ... # error: [unsupported-base] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9758,7 +9758,7 @@ b1 < b2 < b1 # error Default level: warn · Level under ty-compatible: ignore · Added in 0.0.12 · Related issues · -View source +View source @@ -9803,7 +9803,7 @@ def factory(base: type[Base]) -> type: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9835,7 +9835,7 @@ A() + A() # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -9878,7 +9878,7 @@ reveal_type(project.root) # revealed: "." Default level: warn · Added in 0.0.21 · Related issues · -View source +View source @@ -9957,7 +9957,7 @@ to `false` to prevent this rule from reporting unused `type: ignore` comments. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-beta.2 · Related issues · -View source +View source @@ -10037,7 +10037,7 @@ on. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.71 · Related issues · -View source +View source @@ -10123,7 +10123,7 @@ to `false`. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.81 · Related issues · -View source +View source @@ -10168,7 +10168,7 @@ parse("1") Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -10247,7 +10247,7 @@ def foo(x: int | str) -> int | str: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 9b2fdfb0fa..43d732428a 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1341,7 +1341,7 @@ pub(crate) fn cmd_compile( let program_file = ty_python_semantic::Db::program_file(&*db, *file); let parsed = ruff_db::parsed::parsed_module(&*db, program_file.python_file(&*db)).load(&*db); - let model = ty_python_semantic::SemanticModel::new(&*db, program_file); + let model = ty_python_semantic::LoweringModel::new(&*db, program_file); // a `.py` source needs no transpiling to be its own interpreted fallback let mut options = options.clone(); if path.extension().is_some_and(|x| x == "py") { diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md b/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md index 36b5e33b83..885a516123 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md @@ -302,6 +302,67 @@ def check(value: object) -> Decimal: return value cast! Decimal ``` +## A failed test against an unbound class narrows + +a type test against a class the program does not bind is reported, and the checker reads the rest of +the function as it would if the test could run: the branch where the test fails rules the class out + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +def whole(value: int | Decimal) -> int: + # error: [erased-type-check] "`is Decimal` cannot be checked at runtime: `Decimal` is not bound when it runs" + if value is Decimal: + return 0 + reveal_type(value) # revealed: int + return value +``` + +## A reified call that has a reason of its own + +where a call's specialization cannot be written for a reason of its own, as for a lambda's type, +that reason is the one reported, even when another argument names a class the program does not bind + + + +```by +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from decimal import Decimal + +def pair[reified A, reified B](a: A, b: B) -> str: + return A.__name__ + B.__name__ + +def describe(value: Decimal) -> str: + # error: [unspecialized-reified-generic] + return pair(value, lambda: 1) +``` + +## A type test at the module's own level + +a test written at the module's own level runs as the module reaches it. a class the module unbinds +further down still holds where the test runs, so the test reads the class by the name the source +wrote, and the statements after it, the `assert`s among them, decide nothing about it + +```by +class Shown: ... + +def make() -> object: + return Shown() + +value = make() + +assert value is Shown +assert not (value is int) +assert isinstance(value, Shown) + +del Shown +``` + ## A class the program binds differently from the checker a module may import a class for the checker and bind the name some other way as it runs, in the diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_che\342\200\246_-_basedpython___a_name_\342\200\246_-_A_reified_call_that_\342\200\246_(18932f76614d8074).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_che\342\200\246_-_basedpython___a_name_\342\200\246_-_A_reified_call_that_\342\200\246_(18932f76614d8074).snap" new file mode 100644 index 0000000000..cce848cabd --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_che\342\200\246_-_basedpython___a_name_\342\200\246_-_A_reified_call_that_\342\200\246_(18932f76614d8074).snap" @@ -0,0 +1,39 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_type_checking_imports.md - basedpython: a name that is not bound where the program reads it - A reified call that has a reason of its own +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_type_checking_imports.md +--- + +# Source files + +## mdtest_snippet.by + +``` + 1 | from typing import TYPE_CHECKING + 2 | + 3 | if TYPE_CHECKING: + 4 | from decimal import Decimal + 5 | + 6 | def pair[reified A, reified B](a: A, b: B) -> str: + 7 | return A.__name__ + B.__name__ + 8 | + 9 | def describe(value: Decimal) -> str: +10 | # error: [unspecialized-reified-generic] +11 | return pair(value, lambda: 1) +``` + +# Diagnostics + +``` +error[unspecialized-reified-generic]: Cannot call reified generic function `pair` without explicit specialization + --> src/mdtest_snippet.by:11:12 + | +11 | return pair(value, lambda: 1) + | ^^^^^^^^^^^^^^^^^^^^^^ +info: inferred type `() -> int` for type parameter `B` has no runtime spelling — specialize with `pair[...]` + +``` diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index c03cf51200..f058eb8d17 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -25,7 +25,7 @@ use ruff_db::source::{SourceTextError, source_text}; use rustc_hash::FxHasher; pub use semantic_model::{ Completion, DjangoLookupArgument, ExpectedStringLiteralCompletion, ExtensionOperatorRewrite, - HasDefinition, HasType, ImplicitReceiverReference, MemberDefinition, NameKind, + HasDefinition, HasType, ImplicitReceiverReference, LoweringModel, MemberDefinition, NameKind, PreludeDunderReceiver, SemanticModel, basedpython_augmented_assignment_type, basedpython_class_type, }; diff --git a/crates/ty_python_semantic/src/place.rs b/crates/ty_python_semantic/src/place.rs index f4016ce88b..93e01d5218 100644 --- a/crates/ty_python_semantic/src/place.rs +++ b/crates/ty_python_semantic/src/place.rs @@ -19,6 +19,7 @@ use crate::reachability::{ NarrowingProjector, ReachabilityEvaluationCache, evaluate_reachability, evaluate_reachability_with_cache, }; +use crate::types::check_order::InferredInCheckOrder; use crate::types::{ DynamicType, KnownClass, MemberLookupPolicy, Type, TypeAndQualifiers, TypeQualifiers, UnionBuilder, UnionType, binding_type, inferred_declaration, is_discarded_dict_key_assignment, @@ -612,12 +613,20 @@ pub(crate) enum RuntimeRead<'a> { /// completes, which a module the checker finds never ends, one that stops with `sys.exit()` /// say, needs. a binding made under `if TYPE_CHECKING:` never runs, which is how an import /// cycle is broken +/// +/// the answer reads the bindings the module makes after the read, and the conditions they +/// are made under, so it is asked only once the scopes around the read are inferred +/// ([`InferredInCheckOrder`]), never by inference itself: asked from inside the module's +/// inference, each later condition would be inferred inside the inference of the next. +/// inference records where to ask ([`crate::types::runtime_reads`]) pub(crate) fn runtime_binding<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, - file: ProgramFile<'db>, read: RuntimeRead<'_>, + inferred: InferredInCheckOrder<'db>, ) -> RuntimeBinding { + // a question is asked of the file whose inference the proof is for, and of no other + let file = inferred.file(); let index = semantic_index(db, file); let module = parsed_module(db, file.python_file(db)).load(db); match read { diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 798a7fb3e8..0c74355d2a 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -1,3 +1,5 @@ +use std::sync::OnceLock; + use compact_str::CompactString; use ruff_db::PythonFile; use ruff_db::files::{File, FilePath}; @@ -21,15 +23,17 @@ use crate::place::imported_symbol; use crate::place::{builtins_module_scope, implicit_builtins_symbol_scope}; use crate::types::AnonNamedTupleClass; use crate::types::TypeExpressionFlags; +use crate::types::check_order::{InferredInCheckOrder, infer_file_in_check_order}; use crate::types::ide_support::{ImportAliasResolution, definition_for_name}; use crate::types::implicit_names::implicit_name; use crate::types::list_members::{all_members, all_reachable_members}; +use crate::types::reified_infer::WrittenTarget; use crate::types::runtime_spelling::RuntimeSite; +use crate::types::soundness::CastCheck; use crate::types::{ ClassPatternPositionalResult, ClassPatternPositionalSource, CycleDetector, ProgramEnvironment, SpecialFormType, Type, TypeQualifiers, binding_type, class_pattern_positional_result, - class_pattern_positional_sources, infer_complete_scope_types, infer_scope_types, - inferred_declaration, + class_pattern_positional_sources, infer_complete_scope_types, inferred_declaration, }; use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_core::node_key::NodeKey; @@ -71,21 +75,6 @@ impl<'db> SemanticModel<'db> { self.db } - /// basedpython: infers every scope of the file, each before the scopes inside it, as - /// checking the file does. a reader about to ask for the types of most of the file asks - /// this first: a scope inferred before the scope around it reads that scope's names - /// before that scope has inferred the conditions deciding whether they are bound there, - /// and each of those conditions is then inferred inside the inference of the next - pub fn infer_in_check_order(&self) { - let index = semantic_index(self.db, self.file); - for scope in index.scope_ids() { - // a scope that takes type context is inferred inside the scope around it - if !scope.accepts_type_context(self.db) { - infer_scope_types(self.db, scope, crate::types::TypeContext::default()); - } - } - } - pub fn file(&self) -> File { self.file.file(self.db) } @@ -241,46 +230,6 @@ impl<'db> SemanticModel<'db> { .to_instance_approximation(self.db, &self.program_environment()) } - /// basedpython: the specialization step the transpiler splices in after the - /// callee of a bare reified-generic call (`f(1)` → `"[int]"`). `None` when - /// the call is not a bare reified-generic call or no injectable spelling - /// exists — the checker reports the latter as `unspecialized-reified-generic` - pub fn reified_call_specialization(&self, call: &ast::ExprCall) -> Option { - let env = &self.program_environment(); - let db = self.db; - let callee_ty = call.func.inferred_type(self)?; - let function = match callee_ty { - Type::FunctionLiteral(function) => function, - Type::BoundMethod(method) => method.function(db), - _ => return None, - }; - if function.is_classmethod(db) || !function.is_unspecialized_reified(db) { - return None; - } - let mut positional = Vec::with_capacity(call.arguments.args.len()); - for argument in &call.arguments.args { - if argument.is_starred_expr() { - return None; - } - positional.push(argument.inferred_type(self)?); - } - let mut keywords = Vec::with_capacity(call.arguments.keywords.len()); - for keyword in &call.arguments.keywords { - let name = keyword.arg.as_ref()?; - keywords.push((name.as_str(), keyword.value.inferred_type(self)?)); - } - crate::types::reified_infer::injectable_call_specialization( - db, - env, - self.file(), - callee_ty, - function, - positional, - keywords, - Some(RuntimeSite::at(call)), - ) - } - /// basedpython: the runtime registrations a conformance extension /// (`extension str(A):`) emits — one per interface in its header. /// @@ -782,20 +731,6 @@ impl<'db> SemanticModel<'db> { AnonNamedTupleClass::of(self.db, &self.program_environment(), ty) } - /// basedpython: whether every name `expr` reads holds, where it is written, the binding the - /// checker resolved when the program runs — so writing `expr` into code that runs there - /// evaluates what the checker saw. a class imported only under `if TYPE_CHECKING:`, bound - /// only on some paths or only further on does not - pub fn reads_names_bound_where_written(&self, expr: &ast::Expr) -> bool { - crate::types::reified_infer::written_unbound_read( - self.db, - &self.program_environment(), - self.file(), - expr, - ) - .is_none() - } - /// could any conversion apply at this call at all? /// /// Binding the call a second time to find out is the expensive part, and @@ -1139,30 +1074,6 @@ impl<'db> SemanticModel<'db> { .map(|member| member.qualifier(self.db).to_string()) } - /// basedpython: the bracketed type-argument spelling the transpiler - /// injects at a bare constructor call of a generic class (`A(1)` → - /// `"int"`), read from the inferred type of the constructed instance. - /// `None` when the callee is not a generic class literal or no runtime - /// spelling exists — reification of constructors is best-effort, never - /// an error - pub fn reified_constructor_type_arguments( - &self, - call: &ast::ExprCall, - ) -> Option { - let env = &self.program_environment(); - let callee_ty = call.func.inferred_type(self)?; - let class_literal = callee_ty.as_class_literal()?; - let constructed = call.inferred_type(self)?; - crate::types::reified_infer::constructor_specialization_display( - self.db, - env, - self.file(), - class_literal, - constructed, - Some(RuntimeSite::at(call)), - ) - } - /// basedpython: the keyword the transpiler passes a trailing lambda block /// with — the name of the callee's last declared parameter. `None` means /// the lambda is appended as a positional argument instead (unknown @@ -1244,29 +1155,6 @@ impl<'db> SemanticModel<'db> { } } - /// basedpython: how the type test `lhs is rhs` resolves, from the value's - /// inferred type and the type its right-hand side names. - /// - /// The right-hand side is a type expression, so `rhs` may name its target - /// however a type expression can: directly (`list[int]`), through an - /// implicit alias (`X = list[int]`), or through a PEP 695 `type` alias. - /// `None` only when inference has no type for one of the operands. - pub fn parametric_is_plan( - &self, - lhs: &ast::Expr, - rhs: &ast::Expr, - ) -> Option { - let env = &self.program_environment(); - Some(crate::types::reified_infer::type_test_plan( - self.db, - env, - self.file(), - lhs.inferred_type(self)?, - rhs.inferred_type(self)?, - Some(rhs), - )) - } - /// basedpython: when `annotation` is a type expression denoting a union of /// specializations of one erased origin (`list[int] | list[str]`), how the /// arms differ. the transpiler uses this to give the parameter a reified @@ -2109,6 +1997,164 @@ impl<'db> SemanticModel<'db> { } } +/// basedpython: a [`SemanticModel`] for code that writes the checked program out — the +/// transpiler, the native backend, and what the checker reads once a file is checked. it can +/// also ask what a name holds when the program runs, which reads the inference of every scope +/// around the name, those after it included, so it infers the file in the order checking does +/// before the first such question. inference builds plain [`SemanticModel`]s, which cannot ask +/// it: asked from inside inference, the scopes it reads would be inferred inside the scope +/// being inferred +pub struct LoweringModel<'db> { + model: SemanticModel<'db>, + in_check_order: OnceLock>, +} + +impl<'db> std::ops::Deref for LoweringModel<'db> { + type Target = SemanticModel<'db>; + + fn deref(&self) -> &Self::Target { + &self.model + } +} + +impl<'db> LoweringModel<'db> { + pub fn new(db: &'db dyn Db, file: ProgramFile<'db>) -> Self { + Self { + model: SemanticModel::new(db, file), + in_check_order: OnceLock::new(), + } + } + + /// infers every scope of the file, each before the scopes inside it, as checking the file + /// does. a reader about to ask for the types of most of the file asks this first: a scope + /// inferred before the scope around it reads that scope's names before that scope has + /// inferred the conditions deciding whether they are bound there, and each of those + /// conditions is then inferred inside the inference of the next + pub fn infer_in_check_order(&self) { + self.inferred_in_check_order(); + } + + /// [`Self::infer_in_check_order`], run once, with the proof it hands out + pub(crate) fn inferred_in_check_order(&self) -> InferredInCheckOrder<'db> { + *self + .in_check_order + .get_or_init(|| infer_file_in_check_order(self.db, self.file, |_, _| {})) + } + + /// basedpython: the specialization step the transpiler splices in after the + /// callee of a bare reified-generic call (`f(1)` → `"[int]"`). `None` when + /// the call is not a bare reified-generic call or no injectable spelling + /// exists — the checker reports the latter as `unspecialized-reified-generic` + pub fn reified_call_specialization(&self, call: &ast::ExprCall) -> Option { + let env = &self.program_environment(); + let db = self.db; + let callee_ty = call.func.inferred_type(self)?; + let function = match callee_ty { + Type::FunctionLiteral(function) => function, + Type::BoundMethod(method) => method.function(db), + _ => return None, + }; + if function.is_classmethod(db) || !function.is_unspecialized_reified(db) { + return None; + } + let mut positional = Vec::with_capacity(call.arguments.args.len()); + for argument in &call.arguments.args { + if argument.is_starred_expr() { + return None; + } + positional.push(argument.inferred_type(self)?); + } + let mut keywords = Vec::with_capacity(call.arguments.keywords.len()); + for keyword in &call.arguments.keywords { + let name = keyword.arg.as_ref()?; + keywords.push((name.as_str(), keyword.value.inferred_type(self)?)); + } + crate::types::reified_infer::injectable_call_specialization( + db, + env, + self.file(), + callee_ty, + function, + positional, + keywords, + Some(RuntimeSite::at(call, self.inferred_in_check_order())), + ) + } + + /// basedpython: whether every name `expr` reads holds, where it is written, the binding the + /// checker resolved when the program runs — so writing `expr` into code that runs there + /// evaluates what the checker saw. a class imported only under `if TYPE_CHECKING:`, bound + /// only on some paths or only further on does not + pub fn reads_names_bound_where_written(&self, expr: &ast::Expr) -> bool { + crate::types::reified_infer::written_unbound_read( + self.db, + &self.program_environment(), + expr, + self.inferred_in_check_order(), + ) + .is_none() + } + + /// basedpython: the runtime check a checked cast to `type_expr` applies where it is + /// written, or `None` when the cast cannot check its value there + pub fn checked_cast_plan(&self, type_expr: &ast::Expr) -> Option { + crate::types::soundness::cast_check_plan( + self.db, + &self.program_environment(), + self.file(), + type_expr.inferred_type(self)?, + Some(RuntimeSite::at(type_expr, self.inferred_in_check_order())), + ) + } + + /// basedpython: the bracketed type-argument spelling the transpiler + /// injects at a bare constructor call of a generic class (`A(1)` → + /// `"int"`), read from the inferred type of the constructed instance. + /// `None` when the callee is not a generic class literal or no runtime + /// spelling exists — reification of constructors is best-effort, never + /// an error + pub fn reified_constructor_type_arguments( + &self, + call: &ast::ExprCall, + ) -> Option { + let env = &self.program_environment(); + let callee_ty = call.func.inferred_type(self)?; + let class_literal = callee_ty.as_class_literal()?; + let constructed = call.inferred_type(self)?; + crate::types::reified_infer::constructor_specialization_display( + self.db, + env, + self.file(), + class_literal, + constructed, + Some(RuntimeSite::at(call, self.inferred_in_check_order())), + ) + } + + /// basedpython: how the type test `lhs is rhs` resolves, from the value's + /// inferred type and the type its right-hand side names. + /// + /// The right-hand side is a type expression, so `rhs` may name its target + /// however a type expression can: directly (`list[int]`), through an + /// implicit alias (`X = list[int]`), or through a PEP 695 `type` alias. + /// `None` only when inference has no type for one of the operands. + pub fn parametric_is_plan( + &self, + lhs: &ast::Expr, + rhs: &ast::Expr, + ) -> Option { + let env = &self.program_environment(); + Some(crate::types::reified_infer::type_test_plan( + self.db, + env, + self.file(), + lhs.inferred_type(self)?, + rhs.inferred_type(self)?, + Some(WrittenTarget::checked(rhs, self.inferred_in_check_order())), + )) + } +} + /// The type and definition of a symbol. #[derive(Clone, Debug)] pub struct MemberDefinition<'db> { diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 40271a0afa..1e1bd4dc78 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -181,6 +181,7 @@ mod call; pub mod call_type_forms; mod callable; pub mod character; +pub(crate) mod check_order; mod class; mod class_base; pub mod classinfo_spelling; @@ -239,6 +240,7 @@ pub(crate) mod relation; mod relation_error; pub mod repeated_underscore; mod restricted; +pub(crate) mod runtime_reads; pub(crate) mod runtime_spelling; mod safe_variance; mod set_theoretic; @@ -298,18 +300,15 @@ pub(crate) fn check_types_with( let index = semantic_index(db, file); let mut diagnostics = TypeCheckDiagnostics::default(); - for scope_id in index.scope_ids() { - // Scopes that may require type context are inferred during the inference of - // their outer scope. - if scope_id.accepts_type_context(db) { - continue; - } - - let result = infer_scope_types(db, scope_id, TypeContext::default()); - + check_order::infer_file_in_check_order(db, file, |_, result| { if let Some(scope_diagnostics) = result.diagnostics() { diagnostics.extend(scope_diagnostics); } + }); + // basedpython: whether a name a lowering writes into code holds there the binding the + // checker resolved is asked once the file is inferred + for checks in runtime_reads::file_runtime_read_checks(db, file) { + diagnostics.extend(&checks.diagnostics); } // basedpython: an import may be used only through an extension it makes applicable, diff --git a/crates/ty_python_semantic/src/types/check_order.rs b/crates/ty_python_semantic/src/types/check_order.rs new file mode 100644 index 0000000000..11455959f0 --- /dev/null +++ b/crates/ty_python_semantic/src/types/check_order.rs @@ -0,0 +1,70 @@ +//! basedpython: inferring a file's scopes in the order checking it does, each before the scopes +//! inside it +//! +//! a scope inferred before the scope around it reads that scope's names before that scope has +//! inferred the conditions deciding whether they are bound there, and each of those conditions +//! is then inferred inside the inference of the next. inferred in order, every condition a +//! question reads is already inferred when it is asked + +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::{ProgramFile, semantic_index}; + +use crate::Db; +use crate::types::TypeContext; +use crate::types::infer::{ScopeInference, infer_complete_scope_types, infer_scope_types}; + +/// that the scopes a question about the running program reads are inferred, each before the +/// scopes inside it: the bindings a name has when the program runs read every binding the +/// module makes and every condition deciding whether each is made, those after the read +/// included ([`crate::place::runtime_binding`]). +/// +/// only [`infer_file_in_check_order`] and [`infer_scope_in_check_order`] make one, so +/// inference, which those run, cannot ask such a question of the scope it is inferring. one +/// is for one file, and a question asked with it is asked of that file +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct InferredInCheckOrder<'db> { + /// the file whose scopes are inferred, and so the only one a question may be asked of + file: ProgramFile<'db>, +} + +impl<'db> InferredInCheckOrder<'db> { + pub(crate) fn file(self) -> ProgramFile<'db> { + self.file + } +} + +/// infers every scope of `file` in the order checking it does, handing each scope's inference to +/// `each` +pub(crate) fn infer_file_in_check_order<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + mut each: impl FnMut(ScopeId<'db>, &'db ScopeInference<'db>), +) -> InferredInCheckOrder<'db> { + for scope in semantic_index(db, file).scope_ids() { + // a scope that takes type context is inferred inside the scope around it + if !scope.accepts_type_context(db) { + each(scope, infer_scope_types(db, scope, TypeContext::default())); + } + } + InferredInCheckOrder { file } +} + +/// infers every scope around `scope`, outermost first, and then `scope` +pub(crate) fn infer_scope_in_check_order<'db>( + db: &'db dyn Db, + scope: ScopeId<'db>, +) -> (&'db ScopeInference<'db>, InferredInCheckOrder<'db>) { + let file = scope.program_file(db); + let enclosing: Vec = semantic_index(db, file) + .ancestor_scopes(scope.file_scope_id(db)) + .skip(1) + .map(|(ancestor, _)| ancestor) + .collect(); + for ancestor in enclosing.into_iter().rev() { + infer_complete_scope_types(db, ancestor.to_scope_id(db, file)); + } + ( + infer_complete_scope_types(db, scope), + InferredInCheckOrder { file }, + ) +} diff --git a/crates/ty_python_semantic/src/types/context.rs b/crates/ty_python_semantic/src/types/context.rs index 42981c3b16..efc73d3065 100644 --- a/crates/ty_python_semantic/src/types/context.rs +++ b/crates/ty_python_semantic/src/types/context.rs @@ -21,6 +21,7 @@ use crate::types::class::StaticClassLiteral; use crate::types::diagnostic::{INVALID_TYPE_FORM, UNBOUND_TYPE_VARIABLE}; use crate::types::function::FunctionDecorators; use crate::types::infer::InferenceFlags; +use crate::types::runtime_reads::{RuntimeReadCheck, RuntimeReadSite}; use crate::types::runtime_spelling::RuntimeSpelling; use crate::{ Db, Program, @@ -275,6 +276,15 @@ impl<'db, 'ast> InferContext<'db, 'ast> { } } + /// basedpython: records a site whose module-level names are checked once the file is + /// inferred, reported there only if this inference may report diagnostics + pub(crate) fn record_runtime_read_check(&self, site: RuntimeReadSite) { + let reports = !self.diagnostics_suppressed && !self.is_in_no_type_check(); + self.diagnostics + .borrow_mut() + .record_runtime_read_check(RuntimeReadCheck { site, reports }); + } + /// basedpython: [`Self::record_extension_use`], for a use known only by the module that /// declares the extension pub(crate) fn record_extension_module_use(&self, module: File) { diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index b0271ecf79..014b260512 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -23,7 +23,9 @@ use crate::types::function::{FunctionDecorators, FunctionType, KnownFunction, Ov use crate::types::infer::UnsupportedComparisonError; use crate::types::overrides::MethodKind; use crate::types::protocol_class::ProtocolMember; +use crate::types::reified_infer::{ErasedTargetReason, ReifiedInferenceError}; use crate::types::repeated_underscore::UnderscoreRefusal; +use crate::types::runtime_reads::RuntimeReadCheck; use crate::types::special_form::TypeQualifier; use crate::types::string_annotation::{ ESCAPE_CHARACTER_IN_FORWARD_ANNOTATION, IMPLICIT_CONCATENATED_STRING_TYPE_ANNOTATION, @@ -4220,6 +4222,233 @@ struct InferenceUses { /// module-level name a runtime spelling the transpiled program writes reads. each is a /// symbol of the scope that owns it scope_reads: FxHashSet<(FileScopeId, ScopedSymbolId)>, + /// the sites that write module-level names into code that runs, whose names are checked + /// once the file is inferred ([`crate::types::runtime_reads`]) + runtime_read_checks: Vec, +} + +/// reports a bare call of the reified generic function `name` whose specialization cannot be +/// written into the call, saying why — `None` when it would have to pass through unpacked +/// arguments +pub(crate) fn report_unspecialized_reified_call<'db>( + context: &InferContext<'db, '_>, + call: TextRange, + name: &str, + failure: Option>, +) { + let Some(builder) = context.report_lint(&UNSPECIALIZED_REIFIED_GENERIC, call) else { + return; + }; + let (db, env) = (context.db(), context.program_environment()); + let mut diagnostic = builder.into_diagnostic(format_args!( + "Cannot call reified generic function `{name}` without explicit specialization" + )); + match failure { + None => diagnostic.info(format_args!( + "the inferred specialization cannot be injected through unpacked arguments — \ + specialize with `{name}[...]`" + )), + Some(ReifiedInferenceError::Unsolved(parameter)) => diagnostic.info(format_args!( + "reified type parameter `{parameter}` cannot be inferred from the arguments — \ + specialize with `{name}[...]`" + )), + Some(ReifiedInferenceError::Unspellable(parameter, ty)) => { + diagnostic.info(format_args!( + "inferred type `{}` for type parameter `{parameter}` has no runtime spelling — \ + specialize with `{name}[...]`", + ty.display(db, env), + )); + } + Some(ReifiedInferenceError::NotAtRuntime(read, binding)) => { + diagnostic.info(binding.describe(&read)); + } + Some(ReifiedInferenceError::NoBinding) => diagnostic.info(format_args!( + "the specialization cannot be inferred from this call — specialize with \ + `{name}[...]`" + )), + } +} + +/// reports a bare construction of the reified generic class `name` whose specialization +/// cannot be written into the call, saying why +pub(crate) fn report_unspecialized_reified_construction<'db>( + context: &InferContext<'db, '_>, + call: TextRange, + name: &str, + failure: ReifiedInferenceError<'db>, +) { + let Some(builder) = context.report_lint(&UNSPECIALIZED_REIFIED_GENERIC, call) else { + return; + }; + let (db, env) = (context.db(), context.program_environment()); + let mut diagnostic = builder.into_diagnostic(format_args!( + "Cannot construct reified generic class `{name}` without a specialization" + )); + match failure { + ReifiedInferenceError::Unsolved(parameter) => diagnostic.info(format_args!( + "reified type parameter `{parameter}` cannot be inferred here — construct a \ + specialization, as in `{name}[int](...)`" + )), + ReifiedInferenceError::Unspellable(parameter, ty) => diagnostic.info(format_args!( + "inferred type `{}` for type parameter `{parameter}` has no runtime spelling — \ + construct a specialization, as in `{name}[int](...)`", + ty.display(db, env), + )), + ReifiedInferenceError::NotAtRuntime(read, binding) => { + diagnostic.info(binding.describe(&read)); + } + ReifiedInferenceError::NoBinding => diagnostic.info(format_args!( + "the specialization cannot be inferred from this call — construct one, as in \ + `{name}[int](...)`" + )), + } +} + +pub(crate) fn report_erased_type_check( + context: &InferContext<'_, '_>, + primary: TextRange, + target: &str, + reason: ErasedTargetReason, +) { + let Some(builder) = context.report_lint(&ERASED_TYPE_CHECK, primary) else { + return; + }; + match reason { + ErasedTargetReason::NotAtRuntime(name, binding) => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: {}", + binding.why_not_checkable(&name) + )); + diagnostic.info(binding.describe(&name)); + } + ErasedTargetReason::Protocol => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a protocol's instances don't \ + record which specialization they satisfy" + )); + diagnostic.info(format_args!( + "an instance's `__orig_class__` names its concrete class, never the protocol, \ + and a structural `isinstance` check can't see type arguments" + )); + diagnostic.info(format_args!( + "reify the type parameter (`def f[T](x: T)`), or test against a concrete class \ + that records the specialization on its instances or across its mro" + )); + } + ErasedTargetReason::NotSubscriptable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: the target class cannot be \ + subscripted" + )); + diagnostic.info(format_args!( + "the class has no `__class_getitem__` on this python version, so the test's \ + runtime check would raise `TypeError` evaluating `{target}`" + )); + diagnostic.info(format_args!( + "drop the type arguments and test against the bare class, which is all the \ + runtime records anyway" + )); + } + ErasedTargetReason::BuiltinCollection => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a builtin collection's instances \ + don't record their type arguments" + )); + diagnostic.info(format_args!( + "test against a subclass that fixes the arguments (`class B(list[int])`), \ + whose `__orig_bases__` the probe can unwind" + )); + } + ErasedTargetReason::Dynamic => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` admits every value, \ + so there is nothing for the test to look for" + )); + diagnostic.info(format_args!( + "a test that can only answer `True` narrows nothing; drop it, or name the \ + type the value is expected to have" + )); + } + ErasedTargetReason::Callable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a callable's parameter and \ + return types are not recorded on the value" + )); + diagnostic.info(format_args!( + "the runtime can see that a value is callable and nothing more, so the \ + signature would be assumed rather than checked" + )); + diagnostic.info(format_args!( + "test against a bare `Callable`, which asks only what the runtime records, \ + or take the value's type from the call site" + )); + } + ErasedTargetReason::TypedDict => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a `TypedDict`'s instances are \ + plain dicts" + )); + diagnostic.info(format_args!( + "the test could only ask whether the value is a `dict`, assuming every key \ + and value type `{target}` declares" + )); + diagnostic.info(format_args!( + "`isinstance` refuses a `TypedDict` for the same reason" + )); + } + ErasedTargetReason::NonRuntimeCheckableProtocol => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` has a member with no \ + runtime spelling" + )); + diagnostic.info(format_args!( + "a member whose type the emitted python cannot name leaves nothing to check \ + the value's own annotation against" + )); + diagnostic.info(format_args!( + "decorate `{target}` with `@typing.runtime_checkable` for the check python \ + itself performs, which asks only that the members are present" + )); + } + ErasedTargetReason::Intersection => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: an intersection has no single \ + runtime form" + )); + diagnostic.info(format_args!( + "test each part separately and join the tests with `and`" + )); + } + ErasedTargetReason::Subclass => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `issubclass` cannot take what \ + `{target}` is parameterized by" + )); + diagnostic.info(format_args!( + "the test can see that the value is a class and nothing more; name the class \ + it must be a subclass of" + )); + } + ErasedTargetReason::UncomparableLiteral => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: equality does not decide \ + membership of `{target}`" + )); + diagnostic.info(format_args!( + "`0.0 == -0.0` and `nan != nan`, so a `float` or `complex` literal names a \ + type the runtime cannot be asked about; `LiteralString` is a property of how \ + a value was written rather than of the value" + )); + } + ErasedTargetReason::Unspellable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` has no runtime form" + )); + diagnostic.info(format_args!( + "the emitted python has no expression to evaluate for this target" + )); + } + } } pub(crate) fn report_mismatched_type_name<'db>( @@ -4261,6 +4490,8 @@ impl TypeCheckDiagnostics { let uses = self.uses_mut(); uses.extension_modules.extend(&other_uses.extension_modules); uses.scope_reads.extend(&other_uses.scope_reads); + uses.runtime_read_checks + .extend_from_slice(&other_uses.runtime_read_checks); } } @@ -4303,6 +4534,18 @@ impl TypeCheckDiagnostics { .flat_map(|uses| uses.scope_reads.iter().copied()) } + /// basedpython: records a site whose module-level names are checked once the file is + /// inferred + pub(crate) fn record_runtime_read_check(&mut self, check: RuntimeReadCheck) { + self.uses_mut().runtime_read_checks.push(check); + } + + pub(crate) fn runtime_read_checks(&self) -> &[RuntimeReadCheck] { + self.uses + .as_ref() + .map_or(&[], |uses| &uses.runtime_read_checks) + } + fn uses_mut(&mut self) -> &mut InferenceUses { self.uses.get_or_insert_with(Box::default) } @@ -4312,6 +4555,7 @@ impl TypeCheckDiagnostics { if let Some(uses) = &mut self.uses { uses.extension_modules.shrink_to_fit(); uses.scope_reads.shrink_to_fit(); + uses.runtime_read_checks.shrink_to_fit(); } self.diagnostics.shrink_to_fit(); } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index ad11be70e4..c11b6f40c7 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -79,19 +79,18 @@ use crate::types::definition_resolution::AttributeRead; use crate::types::diagnostic::{ self, AMBIGUOUS_CONVERSION, AMBIGUOUS_EXTENSION_MEMBER, CALL_NON_CALLABLE, CONFLICTING_DECLARATIONS, CYCLIC_TYPE_ALIAS_DEFINITION, DYNAMIC_FUNCTION_DECORATOR_RETURN, - ERASED_CAST_ARGUMENT, ERASED_TYPE_CHECK, FINAL_ON_VARIABLE, GeneratorMismatchKind, - IMPLICIT_DECLARATION, INACCESSIBLE_MEMBER, INEFFECTIVE_FINAL, INVALID_ARGUMENT_TYPE, - INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_DECLARATION, - INVALID_ENUM_MEMBER_ANNOTATION, INVALID_FIELD_LOOKUP, INVALID_LEGACY_TYPE_VARIABLE, - INVALID_NEWTYPE, INVALID_PARAMSPEC, INVALID_REGEX, INVALID_REIFIED_TYPE_PARAM, - INVALID_TYPE_ALIAS_TYPE, INVALID_TYPE_FORM, INVALID_TYPE_VARIABLE_CONSTRAINTS, - INVALID_TYPE_VARIABLE_DEFAULT, INVALID_VARIANCE_DECLARATION, NARROWING_GUARD_AS_VALUE, - NON_EXHAUSTIVE_STATEMENT_EXPRESSION, NON_OVERLAPPING_CAST, NON_OVERLAPPING_TYPE_TEST, - OPTIONAL_OBJECT_CONVERSION, POSSIBLY_MISSING_IMPLICIT_CALL, POSSIBLY_MISSING_SUBMODULE, - PRIVATE_CONSTRUCTOR, REFUTABLE_DESTRUCTURING, REFUTABLE_UNPACKING, TRAILING_LAMBDA_PARAMETERS, - TypeCheckDiagnostics, UNANNOTATED_MODEL_FIELD, UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, - UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, UNRESOLVED_REFERENCE, - UNSOUND_ASSIGNMENT, UNSOUND_CAST, UNSOUND_YIELD, UNSPECIALIZED_REIFIED_GENERIC, + ERASED_CAST_ARGUMENT, FINAL_ON_VARIABLE, GeneratorMismatchKind, IMPLICIT_DECLARATION, + INACCESSIBLE_MEMBER, INEFFECTIVE_FINAL, INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, + INVALID_ATTRIBUTE_ACCESS, INVALID_DECLARATION, INVALID_ENUM_MEMBER_ANNOTATION, + INVALID_FIELD_LOOKUP, INVALID_LEGACY_TYPE_VARIABLE, INVALID_NEWTYPE, INVALID_PARAMSPEC, + INVALID_REGEX, INVALID_REIFIED_TYPE_PARAM, INVALID_TYPE_ALIAS_TYPE, INVALID_TYPE_FORM, + INVALID_TYPE_VARIABLE_CONSTRAINTS, INVALID_TYPE_VARIABLE_DEFAULT, INVALID_VARIANCE_DECLARATION, + NARROWING_GUARD_AS_VALUE, NON_EXHAUSTIVE_STATEMENT_EXPRESSION, NON_OVERLAPPING_CAST, + NON_OVERLAPPING_TYPE_TEST, OPTIONAL_OBJECT_CONVERSION, POSSIBLY_MISSING_IMPLICIT_CALL, + POSSIBLY_MISSING_SUBMODULE, PRIVATE_CONSTRUCTOR, REFUTABLE_DESTRUCTURING, REFUTABLE_UNPACKING, + TRAILING_LAMBDA_PARAMETERS, TypeCheckDiagnostics, UNANNOTATED_MODEL_FIELD, + UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, + UNRESOLVED_GLOBAL, UNRESOLVED_REFERENCE, UNSOUND_ASSIGNMENT, UNSOUND_CAST, UNSOUND_YIELD, UNSUPPORTED_OPERATOR, UNUSED_AWAITABLE, USED_UNDERSCORE_NAME, YieldKind, autofix_with_notimplementederror, display_required_elements, hint_if_stdlib_attribute_exists_on_other_versions, refutable_unpacking_applies, @@ -99,7 +98,7 @@ use crate::types::diagnostic::{ report_bad_dunder_delete_call, report_bool_as_int, report_bool_as_int_assignment, report_call_to_abstract_method, report_cannot_pop_required_field_on_typed_dict, report_capturing_case_name, report_capturing_case_name_alternative, - report_dynamic_function_decorator_return, report_invalid_assignment, + report_dynamic_function_decorator_return, report_erased_type_check, report_invalid_assignment, report_invalid_class_match_pattern, report_invalid_exception_caught, report_invalid_exception_cause, report_invalid_exception_raised, report_invalid_exception_tuple_caught, report_invalid_generator_yield_type, @@ -110,7 +109,8 @@ use crate::types::diagnostic::{ report_possibly_missing_attribute, report_possibly_unresolved_reference, report_restricted_constructor, report_too_many_positional_patterns_for_class_pattern, report_unfilled_decoration_context_parameters, report_unplaceable_starred_class_pattern, - report_unsound_assignment, report_unsound_yield, report_unsupported_augmented_assignment, + report_unsound_assignment, report_unsound_yield, report_unspecialized_reified_call, + report_unspecialized_reified_construction, report_unsupported_augmented_assignment, report_unsupported_comparison, }; use crate::types::enums::{enum_ignored_names, is_enum_class_by_inheritance}; @@ -142,11 +142,11 @@ use crate::types::narrow::{ use crate::types::newtype::NewType; use crate::types::receivers; use crate::types::regex; -use crate::types::reified_infer::{self, ErasedTargetReason, ReifiedInferenceError}; +use crate::types::reified_infer; use crate::types::repeated_underscore::{ LoweredParameters, WrittenNames, lower_repeated_underscores, parameter_slots, }; -use crate::types::runtime_spelling::RuntimeSite; +use crate::types::runtime_reads::{ReifiedGeneric, RuntimeReadSite}; use crate::types::set_theoretic::RecursivelyDefined; use crate::types::signatures::{CallableSignature, ReturnCallableTypeVarScope}; use crate::types::soundness::{ @@ -11243,7 +11243,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.file(), value_ty, alias, - Some(type_arg), + Some(crate::types::reified_infer::WrittenTarget::unchecked( + type_arg + )), ), crate::types::reified_infer::ParametricIsPlan::TokenEq(_) | crate::types::reified_infer::ParametricIsPlan::Fold(_) @@ -11288,47 +11290,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } - /// a checked cast with no runtime check the checker can spell tests its value against - /// the target as written, so a target reading a name that does not hold there the binding - /// the checker resolved — a class imported only under `if TYPE_CHECKING:` — would raise - /// `NameError`, or test against another class. a cast that needs no check (a widening) is - /// a plain `typing.cast`, whose target is written as a string, and is left alone - fn report_cast_target_unbound_at_runtime( - &mut self, - type_arg: &ast::Expr, - value_ty: Type<'db>, - target: Type<'db>, - ) { - let env = self.program_environment(); - let db = self.db(); - if cast_is_redundant(db, env, value_ty, target) - || crate::types::soundness::cast_check_plan( - db, - env, - self.file(), - target, - Some(type_arg), - ) - .is_some() - { - return; - } - let Some((name, binding)) = - crate::types::reified_infer::written_unbound_read(db, env, self.file(), type_arg) - else { - return; - }; - let Some(builder) = self.context.report_lint(&ERASED_TYPE_CHECK, type_arg) else { - return; - }; - let mut diagnostic = builder.into_diagnostic(format_args!( - "`{}` cannot be checked at runtime: {}", - target.display(db, env), - binding.why_not_checkable(&name) - )); - diagnostic.info(binding.describe(&name)); - } - /// Report a plain `cast` that is not a widening, and say whether it was one. /// /// The unsuffixed `cast` reinterprets a value without looking at it, so the @@ -11568,8 +11529,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // claim; the plain `cast` never reaches a check in the first place if cast_kind.verifies_at_runtime() { self.report_erased_cast_argument(type_arg, value_ty, target); - self.report_cast_target_unbound_at_runtime(type_arg, value_ty, target); - self.record_checked_cast_spellings(value_ty, target, type_arg); + self.context + .record_runtime_read_check(RuntimeReadSite::CheckedCast { + value: value_arg.node_index().load(), + target: type_arg.node_index().load(), + }); } // an unsound plain `cast` is already the sharper report — a // non-overlapping one would only repeat that the value is not the @@ -13064,8 +13028,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .keywords .iter() .any(|keyword| keyword.arg.is_none()); - let inference_failure = if has_unpacked_arguments { - Some(None) + let inference = if has_unpacked_arguments { + Err(None) } else { reified_infer::inferred_call_type_arguments( self.db(), @@ -13074,49 +13038,30 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { callable_type, function, &call_arguments, - Some(RuntimeSite::at(call_expression)), + None, ) - .err() - .map(Some) + .map_err(Some) }; - if let Some(failure) = inference_failure - && let Some(builder) = self - .context - .report_lint(&UNSPECIALIZED_REIFIED_GENERIC, call_expression) - { - let name = function.name(self.db()); - let mut diagnostic = builder.into_diagnostic(format_args!( - "Cannot call reified generic function `{name}` \ - without explicit specialization" - )); - match failure { - None => diagnostic.info(format_args!( - "the inferred specialization cannot be injected through \ - unpacked arguments — specialize with `{name}[...]`" - )), - Some(ReifiedInferenceError::Unsolved(parameter)) => { - diagnostic.info(format_args!( - "reified type parameter `{parameter}` cannot be inferred \ - from the arguments — specialize with `{name}[...]`" - )); - } - Some(ReifiedInferenceError::Unspellable(parameter, ty)) => { - diagnostic.info(format_args!( - "inferred type `{}` for type parameter `{parameter}` has \ - no runtime spelling — specialize with `{name}[...]`", - ty.display(self.db(), env), - )); - } - Some(ReifiedInferenceError::NotAtRuntime(read, binding)) => { - diagnostic.info(binding.describe(&read)); - } - Some(ReifiedInferenceError::NoBinding) => { - diagnostic.info(format_args!( - "the specialization cannot be inferred from this call — \ - specialize with `{name}[...]`" - )); - } + let name = function.name(self.db()); + match inference { + Ok(arguments) => { + self.context + .record_runtime_read_check(RuntimeReadSite::Reified { + call: call_expression.node_index().load(), + generic: ReifiedGeneric::Function(name.clone()), + reads: arguments + .iter() + .flat_map(reified_infer::TypeArgument::global_reads) + .cloned() + .collect(), + }); } + Err(failure) => report_unspecialized_reified_call( + &self.context, + call_expression.range(), + name, + failure, + ), } } @@ -13276,43 +13221,28 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return; } let env = self.program_environment(); - let Some(failure) = reified_infer::reified_construction_error( + match reified_infer::constructor_specialization( db, env, self.file(), class_literal, return_ty, - Some(RuntimeSite::at(call_expression)), - ) else { - return; - }; - let Some(builder) = self - .context - .report_lint(&UNSPECIALIZED_REIFIED_GENERIC, call_expression) - else { - return; - }; - let name = class.name(db); - let mut diagnostic = builder.into_diagnostic(format_args!( - "Cannot construct reified generic class `{name}` without a specialization" - )); - match failure { - ReifiedInferenceError::Unsolved(parameter) => diagnostic.info(format_args!( - "reified type parameter `{parameter}` cannot be inferred here — \ - construct a specialization, as in `{name}[int](...)`" - )), - ReifiedInferenceError::Unspellable(parameter, ty) => diagnostic.info(format_args!( - "inferred type `{}` for type parameter `{parameter}` has no runtime \ - spelling — construct a specialization, as in `{name}[int](...)`", - ty.display(db, env), - )), - ReifiedInferenceError::NotAtRuntime(read, binding) => { - diagnostic.info(binding.describe(&read)); + None, + ) { + Ok(spelling) => { + self.context + .record_runtime_read_check(RuntimeReadSite::Reified { + call: call_expression.node_index().load(), + generic: ReifiedGeneric::Class(class.name(db).clone()), + reads: spelling.global_reads().cloned().collect(), + }); } - ReifiedInferenceError::NoBinding => diagnostic.info(format_args!( - "the specialization cannot be inferred from this call — construct \ - one, as in `{name}[int](...)`" - )), + Err(failure) => report_unspecialized_reified_construction( + &self.context, + call_expression.range(), + class.name(db), + failure, + ), } } @@ -15976,39 +15906,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Some(returned) } - /// basedpython: records the bindings a checked cast's runtime check reads. the lowering - /// builds its predicate from the type test the cast shares with `is`, and falls back to - /// the shallow target the soundness checks use, so both are spelled here as it spells them - fn record_checked_cast_spellings( - &self, - value_ty: Type<'db>, - target: Type<'db>, - type_arg: &ast::Expr, - ) { - if !self.is_basedpython_file() { - return; - } - let db = self.db(); - let env = self.program_environment(); - let plan = crate::types::reified_infer::type_test_plan( - db, - env, - self.file(), - value_ty, - target, - Some(type_arg), - ); - for spelling in plan.spellings() { - self.context.record_spelling(spelling); - } - if let Some(spelling) = - crate::types::soundness::cast_check_plan(db, env, self.file(), target, Some(type_arg)) - .and_then(|check| check.spelling()) - { - self.context.record_spelling(&spelling); - } - } - /// basedpython: records the binding a context-sensitive name's lowering reads its /// qualifier through — `Red` is written `Color.Red`, which reads `Color` although the /// source never spells it @@ -16842,22 +16739,27 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.file(), left_ty, target, - Some(right), + Some(reified_infer::WrittenTarget::unchecked(right)), ); for module in plan.conformance_modules() { self.context.record_extension_module_use(module); } - for spelling in plan.spellings() { - self.context.record_spelling(spelling); - } if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = &plan { let source = ruff_db::source::source_text(db, self.file()); - self.report_erased_type_check( + report_erased_type_check( + &self.context, TextRange::new(left.start(), right.end()), &source[right.range()], reason.clone(), ); } else { + // whether the names the test is written with hold where it runs is asked once + // the module is inferred + self.context + .record_runtime_read_check(RuntimeReadSite::TypeTest { + value: left.node_index().load(), + target: right.node_index().load(), + }); // a target with no runtime test has already been rejected, and // saying it can also never hold would pile a second report on one // mistake @@ -16918,153 +16820,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { )); } - fn report_erased_type_check( - &self, - primary: TextRange, - target: &str, - reason: ErasedTargetReason, - ) { - let Some(builder) = self.context.report_lint(&ERASED_TYPE_CHECK, primary) else { - return; - }; - match reason { - ErasedTargetReason::NotAtRuntime(name, binding) => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: {}", - binding.why_not_checkable(&name) - )); - diagnostic.info(binding.describe(&name)); - } - ErasedTargetReason::Protocol => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: a protocol's instances don't \ - record which specialization they satisfy" - )); - diagnostic.info(format_args!( - "an instance's `__orig_class__` names its concrete class, never the protocol, \ - and a structural `isinstance` check can't see type arguments" - )); - diagnostic.info(format_args!( - "reify the type parameter (`def f[T](x: T)`), or test against a concrete class \ - that records the specialization on its instances or across its mro" - )); - } - ErasedTargetReason::NotSubscriptable => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: the target class cannot be \ - subscripted" - )); - diagnostic.info(format_args!( - "the class has no `__class_getitem__` on this python version, so the test's \ - runtime check would raise `TypeError` evaluating `{target}`" - )); - diagnostic.info(format_args!( - "drop the type arguments and test against the bare class, which is all the \ - runtime records anyway" - )); - } - ErasedTargetReason::BuiltinCollection => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: a builtin collection's instances \ - don't record their type arguments" - )); - diagnostic.info(format_args!( - "test against a subclass that fixes the arguments (`class B(list[int])`), \ - whose `__orig_bases__` the probe can unwind" - )); - } - ErasedTargetReason::Dynamic => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: `{target}` admits every value, \ - so there is nothing for the test to look for" - )); - diagnostic.info(format_args!( - "a test that can only answer `True` narrows nothing; drop it, or name the \ - type the value is expected to have" - )); - } - ErasedTargetReason::Callable => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: a callable's parameter and \ - return types are not recorded on the value" - )); - diagnostic.info(format_args!( - "the runtime can see that a value is callable and nothing more, so the \ - signature would be assumed rather than checked" - )); - diagnostic.info(format_args!( - "test against a bare `Callable`, which asks only what the runtime records, \ - or take the value's type from the call site" - )); - } - ErasedTargetReason::TypedDict => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: a `TypedDict`'s instances are \ - plain dicts" - )); - diagnostic.info(format_args!( - "the test could only ask whether the value is a `dict`, assuming every key \ - and value type `{target}` declares" - )); - diagnostic.info(format_args!( - "`isinstance` refuses a `TypedDict` for the same reason" - )); - } - ErasedTargetReason::NonRuntimeCheckableProtocol => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: `{target}` has a member with no \ - runtime spelling" - )); - diagnostic.info(format_args!( - "a member whose type the emitted python cannot name leaves nothing to check \ - the value's own annotation against" - )); - diagnostic.info(format_args!( - "decorate `{target}` with `@typing.runtime_checkable` for the check python \ - itself performs, which asks only that the members are present" - )); - } - ErasedTargetReason::Intersection => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: an intersection has no single \ - runtime form" - )); - diagnostic.info(format_args!( - "test each part separately and join the tests with `and`" - )); - } - ErasedTargetReason::Subclass => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: `issubclass` cannot take what \ - `{target}` is parameterized by" - )); - diagnostic.info(format_args!( - "the test can see that the value is a class and nothing more; name the class \ - it must be a subclass of" - )); - } - ErasedTargetReason::UncomparableLiteral => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: equality does not decide \ - membership of `{target}`" - )); - diagnostic.info(format_args!( - "`0.0 == -0.0` and `nan != nan`, so a `float` or `complex` literal names a \ - type the runtime cannot be asked about; `LiteralString` is a property of how \ - a value was written rather than of the value" - )); - } - ErasedTargetReason::Unspellable => { - let mut diagnostic = builder.into_diagnostic(format_args!( - "`is {target}` cannot be checked at runtime: `{target}` has no runtime form" - )); - diagnostic.info(format_args!( - "the emitted python has no expression to evaluate for this target" - )); - } - } - } - /// `reification` names the owner of this list, which decides whether a /// basedpython `reified` parameter — or a declared variance — can take /// effect; one that cannot is reported here rather than silently dropped. diff --git a/crates/ty_python_semantic/src/types/infer/tests.rs b/crates/ty_python_semantic/src/types/infer/tests.rs index bca5a4e2ab..3169128c6f 100644 --- a/crates/ty_python_semantic/src/types/infer/tests.rs +++ b/crates/ty_python_semantic/src/types/infer/tests.rs @@ -854,6 +854,37 @@ class Form(Outer): handle.join().expect("regression test thread panicked") } +/// basedpython: a type test at a module's own level is checked against what the class it names +/// holds when the program runs, which reads every condition the module goes on to make, each +/// `assert` below the test among them. asked during the module's own inference, each of those +/// would be inferred inside the inference of the next, and the stack would grow with them +#[test] +fn module_level_type_test_before_many_asserts_does_not_overflow_stack() -> anyhow::Result<()> { + const ASSERTS: usize = 2_000; + let handle = std::thread::Builder::new() + .name("module-level-type-test-stack-test".into()) + // far less than ty's worker threads have, so a stack that grows with the asserts + // overflows at any optimization level, while one that does not fits with room to spare + .stack_size(2 * 1024 * 1024) + .spawn(|| -> anyhow::Result<()> { + let mut db = setup_db(); + let mut source = String::from( + "class Shown: ...\n\ndef make() -> object:\n return Shown()\n\n\ + value = make()\nflag = value is Shown\n", + ); + for _ in 0..ASSERTS { + source.push_str("assert make()\n"); + } + db.write_file("src/a.by", source)?; + + assert_file_diagnostics(&db, "src/a.by", &[]); + + Ok(()) + })?; + + handle.join().expect("regression test thread panicked") +} + #[test] fn implicit_attribute_preserves_terminal_narrowing_after_many_calls() -> anyhow::Result<()> { let mut db = setup_db(); diff --git a/crates/ty_python_semantic/src/types/lowering_reads.rs b/crates/ty_python_semantic/src/types/lowering_reads.rs index 69a38d68b8..c5a568ed67 100644 --- a/crates/ty_python_semantic/src/types/lowering_reads.rs +++ b/crates/ty_python_semantic/src/types/lowering_reads.rs @@ -18,14 +18,14 @@ use ruff_python_ast::name::Name; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{self as ast, Expr, Stmt}; -use crate::semantic_model::SemanticModel; +use crate::semantic_model::LoweringModel; use crate::types::runtime_spelling::RuntimeSpelling; use crate::types::soundness_sites::{SoundnessPositions, soundness_sites}; /// every module-level name a spelling the transpiled program writes for `suite` reads. every /// soundness position counts, because which ones a build enables is the build's choice and /// the answer has to hold for any build of the file -pub(crate) fn spelled_module_reads(model: &SemanticModel<'_>, suite: &[Stmt]) -> Vec { +pub(crate) fn spelled_module_reads(model: &LoweringModel<'_>, suite: &[Stmt]) -> Vec { let mut spellings = Spellings { model, reads: Vec::new(), @@ -40,7 +40,7 @@ pub(crate) fn spelled_module_reads(model: &SemanticModel<'_>, suite: &[Stmt]) -> } struct Spellings<'a, 'db> { - model: &'a SemanticModel<'db>, + model: &'a LoweringModel<'db>, reads: Vec, } diff --git a/crates/ty_python_semantic/src/types/narrow.rs b/crates/ty_python_semantic/src/types/narrow.rs index 170910c1b8..eda0fd730b 100644 --- a/crates/ty_python_semantic/src/types/narrow.rs +++ b/crates/ty_python_semantic/src/types/narrow.rs @@ -4667,7 +4667,12 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { expression.file(self.db), lhs_ty, rhs_ty, - Some(right), + // planned as inference plans the test, without asking what the + // names its target is written with hold where it runs. a target + // whose names do not hold the class the checker resolved is + // reported once the file is inferred, and narrows as the test it + // would be: the program does not run with that error in it + Some(crate::types::reified_infer::WrittenTarget::unchecked(right)), ) .narrows_negatively() .then_some(rhs_ty) diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 9eabcec10d..05bfe6da0c 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -29,6 +29,7 @@ use crate::Db; use crate::place::{RuntimeBinding, RuntimeRead, builtins_symbol, global_symbol, runtime_binding}; use crate::types::ProgramEnvironment; use crate::types::call::{Argument, CallArguments}; +use crate::types::check_order::InferredInCheckOrder; use crate::types::class::{ClassLiteral, ClassType, GenericAlias}; use crate::types::class_base::ClassBase; use crate::types::function::FunctionType; @@ -68,7 +69,7 @@ pub(crate) fn inferred_call_type_arguments<'db>( callee: Type<'db>, function: FunctionType<'db>, arguments: &CallArguments<'_, 'db>, - site: Option, + site: Option>, ) -> Result, ReifiedInferenceError<'db>> { let bindings = callee .try_call(db, env, arguments) @@ -98,7 +99,7 @@ pub(crate) fn injectable_call_specialization<'db>( function: FunctionType<'db>, positional: Vec>, keywords: Vec<(&str, Type<'db>)>, - site: Option, + site: Option>, ) -> Option { let arguments: CallArguments<'_, 'db> = positional .into_iter() @@ -152,7 +153,7 @@ fn rendered_type_arguments<'db>( file: File, function: FunctionType<'db>, specialization: Option>, - site: Option, + site: Option>, ) -> Result, ReifiedInferenceError<'db>> { let signature = function.signature(db); let generic_context = signature @@ -218,7 +219,7 @@ fn rendered_type_arguments<'db>( ReifiedInferenceError::Unspellable(parameter.name.clone(), promoted) })?; if let Some(site) = site - && let Some((name, binding)) = text.unbound_read(db, env, file, site) + && let Some((name, binding)) = text.unbound_read(db, env, site) { return Err(ReifiedInferenceError::NotAtRuntime(name, binding)); } @@ -242,6 +243,13 @@ pub(crate) struct TypeArgument { keyword: bool, } +impl TypeArgument { + /// every binding of the module's global scope the argument's spelling reads + pub(crate) fn global_reads(&self) -> impl Iterator { + self.text.global_reads() + } +} + /// a type parameter paired with the value the call solved it to struct ResolvedParameter<'db, 'name> { name: &'name Name, @@ -618,15 +626,20 @@ fn spell_specialization_arguments<'db>( /// instance, so the answer always matches the checker's solved specialization — /// including PEP 696 defaults and any usage-based widening. This is the single /// decision behind both directions: [`constructor_specialization_display`] is -/// the transpiler's injection and [`reified_construction_error`] the checker's -/// diagnostic, so the two cannot disagree about which constructions are legal. -fn constructor_specialization<'db>( +/// the transpiler's injection and the checker reports a reified class's +/// construction this rejects, so the two cannot disagree about which +/// constructions are legal. +/// +/// at a `site`, every name the spelling reads must hold there the binding the +/// checker resolved. inference asks without one, and the names are checked once +/// the module is inferred ([`crate::types::runtime_reads`]) +pub(crate) fn constructor_specialization<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, - site: Option, + site: Option>, ) -> Result> { let Type::NominalInstance(instance) = constructed.promote_in(db, env, file) else { return Err(ReifiedInferenceError::NoBinding); @@ -640,7 +653,7 @@ fn constructor_specialization<'db>( } let spelling = spell_specialization_arguments(db, env, file, origin, alias.specialization(db))?; if let Some(site) = site - && let Some((name, binding)) = spelling.unbound_read(db, env, file, site) + && let Some((name, binding)) = spelling.unbound_read(db, env, site) { return Err(ReifiedInferenceError::NotAtRuntime(name, binding)); } @@ -652,39 +665,19 @@ fn constructor_specialization<'db>( /// /// `None` when the construction has no writable specialization; unlike a bare /// reified-generic call this is never an error on its own — the call simply -/// stays bare, and [`reified_construction_error`] reports it only when the class -/// reifies something. +/// stays bare, and the checker reports it only when the class reifies +/// something. pub(crate) fn constructor_specialization_display<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, class_literal: ClassLiteral<'db>, constructed: Type<'db>, - site: Option, + site: Option>, ) -> Option { constructor_specialization(db, env, file, class_literal, constructed, site).ok() } -/// Why a construction of the reified generic class `class_literal` cannot say -/// which specialization it builds, or `None` when it can. -/// -/// A reified class records its type arguments on the specialization its -/// instances are built from, so a construction has to name one. This is -/// [`constructor_specialization_display`]'s decision with the reason kept: the -/// transpiler injects the solved specialization wherever one can be written, and -/// where it cannot the instance would have no answer for a type parameter its -/// own methods read. -pub(crate) fn reified_construction_error<'db>( - db: &'db dyn Db, - env: &ProgramEnvironment<'db>, - file: File, - class_literal: ClassLiteral<'db>, - constructed: Type<'db>, - site: Option, -) -> Option> { - constructor_specialization(db, env, file, class_literal, constructed, site).err() -} - /// the class's bare name, provided that name resolves — in the module's /// globals, else builtins — to this very class, so the injected expression /// evaluates to the intended type object. a module-level `private` class is @@ -824,6 +817,11 @@ impl ParametricIsPlan { /// and a value that records none answers `False` even where the static /// types say it is a match — narrowing on that would remove a type the /// value really has. + /// + /// narrowing asks this of the plan inference makes, without asking what the names the + /// target is written with hold where the test runs ([`WrittenTarget::unchecked`]): a test + /// whose target the program does not bind there is reported once the file is inferred, + /// and narrows as the test it would be pub(crate) fn narrows_negatively(&self) -> bool { match self { Self::Fold(_) @@ -1219,10 +1217,10 @@ pub(crate) fn classify_parametric_is<'db>( file: File, lhs_ty: Type<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, - rhs_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { let target_origin = ClassLiteral::Static(rhs_alias.origin(db)); - let target_args_ast: Vec<&ast::Expr> = match rhs_node { + let target_args_ast: Vec<&ast::Expr> = match written.map(WrittenTarget::node) { Some(ast::Expr::Subscript(subscript)) => match subscript.slice.as_ref() { ast::Expr::Tuple(tuple) => tuple.elts.iter().collect(), single => vec![single], @@ -1236,7 +1234,7 @@ pub(crate) fn classify_parametric_is<'db>( lhs_ty.promote(db, env), rhs_alias, &target_args_ast, - rhs_node, + written, ); // the runtime probe unwinds the value's `__orig_class__` and its class's // generic bases across the mro, so a builtin-collection target is checkable @@ -1284,8 +1282,11 @@ pub(crate) fn classify_parametric_is<'db>( /// rules out a `TypedDict` (its instances are plain dicts), a callable type /// (the runtime sees only that a value is callable), and `Any`. /// -/// `target_node` is the source the target was written as, which the -/// specialization plans use to spell type arguments back out. +/// `written` is the source the target was written as, which the specialization plans use to +/// spell type arguments back out. checked where the test runs +/// ([`WrittenTarget::checked`]), every name a spelling reads must hold there the binding the +/// checker resolved ([`runtime_binding`]), and a target whose names do not is +/// [`ErasedTargetReason::NotAtRuntime`]. inference plans with an unchecked target /// /// `value_ty` is taken exactly as the value has it — a construction's `final A` /// is disjoint from an unrelated class in a way a plain `A` is not, and that is @@ -1297,24 +1298,16 @@ pub(crate) fn type_test_plan<'db>( file: File, value_ty: Type<'db>, target: Type<'db>, - target_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { - let plan = type_test_plan_seen( - db, - env, - file, - value_ty, - target, - target_node, - &mut Vec::new(), - ); + let plan = type_test_plan_seen(db, env, file, value_ty, target, written, &mut Vec::new()); // each spelling the plan rebuilds is written where the test is: one for a union's arm // has no node of its own to have been asked about - if let Some(site) = target_node.map(RuntimeSite::at) + if let Some(site) = written.and_then(WrittenTarget::site) && let Some((name, binding)) = plan .spellings() .into_iter() - .find_map(|spelling| spelling.unbound_read(db, env, file, site)) + .find_map(|spelling| spelling.unbound_read(db, env, site)) { return ParametricIsPlan::ErasedTarget(ErasedTargetReason::NotAtRuntime(name, binding)); } @@ -1332,14 +1325,14 @@ fn type_test_plan_seen<'db>( file: File, value_ty: Type<'db>, target: Type<'db>, - target_node: Option<&ast::Expr>, + written: Option>, open: &mut Vec>, ) -> ParametricIsPlan { // an `Unknown` target is not evidence of anything: every value is a subtype // of it, and folding on that would answer `True` for a test the source got // wrong somewhere else if target.is_dynamic() { - return runtime_test_plan(db, env, file, value_ty, target, target_node, open); + return runtime_test_plan(db, env, file, value_ty, target, written, open); } // a target that resolves statically needs no runtime residue at all, and // answering it here keeps every unspellable-but-decidable target working: @@ -1350,7 +1343,7 @@ fn type_test_plan_seen<'db>( if value_ty.is_disjoint_from(db, env, target) { return ParametricIsPlan::Fold(false); } - runtime_test_plan(db, env, file, value_ty, target, target_node, open) + runtime_test_plan(db, env, file, value_ty, target, written, open) } /// The runtime residue of a type test whose answer the static types do not @@ -1362,7 +1355,7 @@ fn runtime_test_plan<'db>( file: File, value_ty: Type<'db>, target: Type<'db>, - target_node: Option<&ast::Expr>, + written: Option>, open: &mut Vec>, ) -> ParametricIsPlan { if target.is_none(db) { @@ -1425,7 +1418,7 @@ fn runtime_test_plan<'db>( // a literal type holds exactly the values equal to it, which is what // the runtime compares. an enum member is a literal too, and its // equality is identity - Type::LiteralValue(literal) => literal_target_plan(db, env, file, literal, target_node), + Type::LiteralValue(literal) => literal_target_plan(db, env, file, literal, written), // `type[C]`: the runtime can check this one in full — the value must be // a class, and a subclass of `C` @@ -1442,7 +1435,7 @@ fn runtime_test_plan<'db>( }, Type::NominalInstance(_) | Type::ProtocolInstance(_) => { - class_target_plan(db, env, file, value_ty, target, target_node) + class_target_plan(db, env, file, value_ty, target, written) } // the value's type is carried by a reified type parameter on the @@ -1484,16 +1477,17 @@ fn literal_target_plan<'db>( env: &ProgramEnvironment<'db>, file: File, literal: crate::types::LiteralValueType<'db>, - target_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { let unspellable = ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); // the guard names a builtin, and the value is the source's own literal where it wrote // one - let written = target_node - .is_some_and(|node| node.is_literal_expr() || matches!(node, ast::Expr::UnaryOp(_))); + let literal_written = written.is_some_and(|written| { + written.node.is_literal_expr() || matches!(written.node, ast::Expr::UnaryOp(_)) + }); let equality = |class: &str, value: String| ParametricIsPlan::Equality { class: class.to_owned(), - value: if written { + value: if literal_written { TargetSpelling::Written } else { TargetSpelling::Rebuilt(RuntimeSpelling::text(value)) @@ -1517,7 +1511,7 @@ fn literal_target_plan<'db>( // an enum member is a singleton, so identity is exact — and it is the // comparison `Enum.__eq__` performs anyway LiteralValueTypeKind::Enum(member) => { - match written_or_spelled(db, env, file, target_node, false, || { + match written_or_spelled(db, env, written, false, || { spell_class_literal(db, env, file, member.enum_class(db)).map(|mut class| { class.push_str(&format!(".{}", member.name(db))); class @@ -1539,6 +1533,43 @@ fn literal_target_plan<'db>( } } +/// a type test's target as the source wrote it, and whether what the names it is written with +/// hold where the test runs is asked +#[derive(Clone, Copy)] +pub(crate) struct WrittenTarget<'a, 'db> { + node: &'a ast::Expr, + /// present once the scopes around the test are inferred, which the question needs + inferred: Option>, +} + +impl<'a, 'db> WrittenTarget<'a, 'db> { + /// a target whose names are not asked about, as inference plans a test: what they hold + /// where it runs is asked once the file is inferred ([`crate::types::runtime_reads`]) + pub(crate) fn unchecked(node: &'a ast::Expr) -> Self { + Self { + node, + inferred: None, + } + } + + /// a target whose names must hold, where the test runs, the bindings the checker resolved + pub(crate) fn checked(node: &'a ast::Expr, inferred: InferredInCheckOrder<'db>) -> Self { + Self { + node, + inferred: Some(inferred), + } + } + + fn node(self) -> &'a ast::Expr { + self.node + } + + fn site(self) -> Option> { + self.inferred + .map(|inferred| RuntimeSite::at(self.node, inferred)) + } +} + /// how a target is written into the emitted python: the spelling rebuilt from /// the type, or the source's own text where that is what the runtime evaluates. /// @@ -1554,26 +1585,28 @@ fn literal_target_plan<'db>( /// whose value is the specialization it unwinds. /// /// the written node is evaluated where it stands, and a rebuilt spelling where the test is -/// written, so each has to read names that hold there the bindings the checker resolved -/// ([`runtime_binding`]): a class imported under `if TYPE_CHECKING:` is one the checker -/// resolves and the program does not have +/// written, so at a `site` each has to read names that hold there the bindings the checker +/// resolved ([`runtime_binding`]): a class imported under `if TYPE_CHECKING:` is one the +/// checker resolves and the program does not have. without a site neither is asked, and the +/// rebuilt spelling is taken wherever there is one fn written_or_spelled<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, - file: File, - target_node: Option<&ast::Expr>, + written: Option>, subscript_evaluates: bool, spell: impl FnOnce() -> Option, ) -> Result { - let site = target_node.map(RuntimeSite::at); + let site = written.and_then(WrittenTarget::site); let mut unbound = None; if let Some(spelling) = spell() { - match site.and_then(|site| spelling.unbound_read(db, env, file, site)) { + match site.and_then(|site| spelling.unbound_read(db, env, site)) { None => return Ok(TargetSpelling::Rebuilt(spelling)), Some(read) => unbound = Some(read), } } - let Some(node) = target_node.filter(|node| evaluates_to_its_target(node, subscript_evaluates)) + let Some(node) = written + .map(WrittenTarget::node) + .filter(|node| evaluates_to_its_target(node, subscript_evaluates)) else { return Err( unbound.map_or(ErasedTargetReason::Unspellable, |(name, binding)| { @@ -1581,7 +1614,10 @@ fn written_or_spelled<'db>( }), ); }; - match written_unbound_read(db, env, file, node) { + let Some(site) = site else { + return Ok(TargetSpelling::Written); + }; + match written_unbound_read(db, env, node, site.inferred()) { None => Ok(TargetSpelling::Written), Some((name, binding)) => Err(ErasedTargetReason::NotAtRuntime(name, binding)), } @@ -1594,13 +1630,13 @@ fn written_or_spelled<'db>( pub(crate) fn written_unbound_read<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, - file: File, node: &ast::Expr, + inferred: InferredInCheckOrder<'db>, ) -> Option<(Name, RuntimeBinding)> { struct Names<'a, 'db> { db: &'db dyn Db, env: &'a ProgramEnvironment<'db>, - file: File, + inferred: InferredInCheckOrder<'db>, found: Option<(Name, RuntimeBinding)>, } impl<'ast> ast::visitor::Visitor<'ast> for Names<'_, '_> { @@ -1612,8 +1648,8 @@ pub(crate) fn written_unbound_read<'db>( let binding = runtime_binding( self.db, self.env, - self.db.program_file(self.file), RuntimeRead::Written(ast::ExprRef::from(expr)), + self.inferred, ); if binding != RuntimeBinding::Bound { self.found = Some((name.id.clone(), binding)); @@ -1626,7 +1662,7 @@ pub(crate) fn written_unbound_read<'db>( let mut names = Names { db, env, - file, + inferred, found: None, }; ast::visitor::Visitor::visit_expr(&mut names, node); @@ -1681,7 +1717,7 @@ fn class_target_plan<'db>( file: File, value_ty: Type<'db>, target: Type<'db>, - target_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { let Some(class) = target_class(db, env, target) else { return ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); @@ -1705,7 +1741,7 @@ fn class_target_plan<'db>( .iter() .all(Type::is_dynamic) => { - match written_or_spelled(db, env, file, target_node, false, || { + match written_or_spelled(db, env, written, false, || { spell_class_literal(db, env, file, ClassLiteral::Static(alias.origin(db))) }) { Ok(spelling) => ParametricIsPlan::Isinstance(spelling), @@ -1715,14 +1751,14 @@ fn class_target_plan<'db>( // a specialization keeps the engine it always had: reified cells, // an `__orig_class__` probe, or a structural protocol check ClassType::Generic(alias) => { - classify_parametric_is(db, env, file, value_ty, alias, target_node) + classify_parametric_is(db, env, file, value_ty, alias, written) } ClassType::NonGeneric(_) => { // an interface something visibly conforms to is answered by the // registry rather than by the class hierarchy, so it is checkable // even though a conforming type is not a subclass if let Some((members, declared_in)) = conformance_members(db, file, class) { - return match written_or_spelled(db, env, file, target_node, false, || { + return match written_or_spelled(db, env, written, false, || { spell_class(db, env, file, class) }) { Ok(target) => ParametricIsPlan::Conformance { @@ -1752,7 +1788,7 @@ fn class_target_plan<'db>( }; } } - match written_or_spelled(db, env, file, target_node, false, || { + match written_or_spelled(db, env, written, false, || { spell_class(db, env, file, class) }) { Ok(spelling) => ParametricIsPlan::Isinstance(spelling), @@ -2023,7 +2059,7 @@ fn classify_value<'db>( value_ty: Type<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, target_args_ast: &[&ast::Expr], - rhs_node: Option<&ast::Expr>, + written: Option>, ) -> ParametricIsPlan { let target_origin = ClassLiteral::Static(rhs_alias.origin(db)); // when the value's type is carried by a reified type parameter, the answer @@ -2037,7 +2073,7 @@ fn classify_value<'db>( target_origin, rhs_alias, target_args_ast, - rhs_node, + written.map(WrittenTarget::node), ) { return plan; @@ -2055,7 +2091,7 @@ fn classify_value<'db>( // runtime probe (user generic) or an erased-target error (builtin) // the source's own spelling is preferred: a name it wrote is in scope, // and the runtime probe unwraps a `TypeAliasType` for itself - let target = match written_or_spelled(db, env, file, rhs_node, true, || { + let target = match written_or_spelled(db, env, written, true, || { spell_class(db, env, file, ClassType::Generic(rhs_alias)) }) { Ok(target) => target, diff --git a/crates/ty_python_semantic/src/types/runtime_reads.rs b/crates/ty_python_semantic/src/types/runtime_reads.rs new file mode 100644 index 0000000000..ddf2485152 --- /dev/null +++ b/crates/ty_python_semantic/src/types/runtime_reads.rs @@ -0,0 +1,312 @@ +//! basedpython: whether each module-level name a lowering writes into code that runs holds +//! there the binding the checker resolved +//! +//! a type test, a checked cast and a bare call or construction of a reified generic are +//! lowered to code that names classes — the test's target, the specialization the call is +//! given — and the program has to find there the class the checker resolved. a name bound +//! only under `if TYPE_CHECKING:`, only on some paths, or only further on does not hold it +//! ([`runtime_binding`](crate::place::runtime_binding)). which one it is depends on every +//! binding the module makes of the name and on every condition deciding whether each is +//! made, the conditions after the site included, so it is asked once the scope holding the +//! site, and every scope around it, is inferred ([`InferredInCheckOrder`]) +//! +//! inference only records where to ask ([`RuntimeReadCheck`]). asked from inside the +//! module's own inference, the question would infer every condition below the site out of +//! order, each one inside the inference of the next + +use ruff_db::files::File; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_python_ast::name::Name; +use ruff_python_ast::{self as ast, AnyRootNodeRef, NodeIndex}; +use ruff_text_size::{Ranged, TextRange}; +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::{ProgramFile, semantic_index}; + +use crate::semantic_model::{HasType, SemanticModel}; +use crate::types::check_order::{InferredInCheckOrder, infer_scope_in_check_order}; +use crate::types::context::InferContext; +use crate::types::diagnostic::{ + ERASED_TYPE_CHECK, TypeCheckDiagnostics, report_erased_type_check, + report_unspecialized_reified_call, report_unspecialized_reified_construction, +}; +use crate::types::infer::infer_scope_types; +use crate::types::reified_infer::{ + ParametricIsPlan, ReifiedInferenceError, WrittenTarget, type_test_plan, written_unbound_read, +}; +use crate::types::runtime_spelling::{RuntimeSite, first_unbound_read}; +use crate::types::soundness::{CastCheck, cast_check_plan, cast_is_redundant}; +use crate::types::{ProgramEnvironment, TypeContext}; +use crate::{Db, FxIndexMap, FxIndexSet}; + +/// a site inference found that writes module-level names into code that runs, and whether +/// the inference that found it reports diagnostics there +#[derive(Debug, Clone, PartialEq, Eq, get_size2::GetSize)] +pub(crate) struct RuntimeReadCheck { + pub(crate) site: RuntimeReadSite, + pub(crate) reports: bool, +} + +/// what writes the names, by the nodes it is lowered from +#[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] +pub(crate) enum RuntimeReadSite { + /// `value is target`, a test inference found checkable wherever the names hold + TypeTest { value: NodeIndex, target: NodeIndex }, + /// `value cast! target` or `value cast? target` + CheckedCast { value: NodeIndex, target: NodeIndex }, + /// a bare call or construction of a reified generic, with the module-level names the + /// specialization the lowering injects reads + Reified { + call: NodeIndex, + generic: ReifiedGeneric, + reads: Box<[Name]>, + }, +} + +/// the reified generic a bare call specializes +#[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] +pub(crate) enum ReifiedGeneric { + /// the function called, by name + Function(Name), + /// the class constructed, by name + Class(Name), +} + +/// what [`runtime_read_checks`] found for one scope +#[derive(Debug, Default, PartialEq, Eq, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) struct RuntimeReadChecks { + /// a site whose names do not hold the bindings the checker resolved + pub(crate) diagnostics: TypeCheckDiagnostics, + /// the module-level names the type tests and casts are written with, which the source + /// itself need not read + pub(crate) module_reads: Box<[Name]>, +} + +/// [`runtime_read_checks`] of every scope of a checked `file` whose inference recorded a site, +/// which most scopes have none of +pub(crate) fn file_runtime_read_checks<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, +) -> impl Iterator { + semantic_index(db, file) + .scope_ids() + .filter(move |scope| { + !scope.accepts_type_context(db) + && infer_scope_types(db, *scope, TypeContext::default()) + .diagnostics() + .is_some_and(|inferred| !inferred.runtime_read_checks().is_empty()) + }) + .map(move |scope| runtime_read_checks(db, scope)) +} + +/// checks every site the inference of `scope` recorded ([`RuntimeReadCheck`]), once `scope` +/// and every scope around it are inferred +#[salsa::tracked(returns(ref), heap_size = ruff_memory_usage::heap_size)] +pub(crate) fn runtime_read_checks<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> RuntimeReadChecks { + let (inferred, in_check_order) = infer_scope_in_check_order(db, scope); + let Some(inferred) = inferred.diagnostics() else { + return RuntimeReadChecks::default(); + }; + // a site inferred more than once is checked once, and reported if any inference of it + // would have reported there + let mut sites: FxIndexMap<&RuntimeReadSite, bool> = FxIndexMap::default(); + for check in inferred.runtime_read_checks() { + *sites.entry(&check.site).or_default() |= check.reports; + } + if sites.is_empty() { + return RuntimeReadChecks::default(); + } + + let file = scope.program_file(db); + let parsed = parsed_module(db, file.python_file(db)).load(db); + let env = ProgramEnvironment::from_file(file); + let model = SemanticModel::new(db, file); + let mut checker = RuntimeReadChecker { + db, + env: &env, + file: file.file(db), + parsed: &parsed, + model: &model, + in_check_order, + contexts: FxIndexMap::default(), + module_reads: FxIndexSet::default(), + }; + for (site, reports) in sites { + checker.check(site, reports); + } + let mut diagnostics = TypeCheckDiagnostics::default(); + for context in checker.contexts.into_values() { + diagnostics.extend(&context.finish()); + } + diagnostics.shrink_to_fit(); + RuntimeReadChecks { + diagnostics, + module_reads: checker.module_reads.into_iter().collect(), + } +} + +/// checks the sites one scope's inference recorded +struct RuntimeReadChecker<'a, 'db> { + db: &'db dyn Db, + env: &'a ProgramEnvironment<'db>, + file: File, + parsed: &'a ParsedModuleRef, + model: &'a SemanticModel<'db>, + in_check_order: InferredInCheckOrder<'db>, + /// a context per scope a site is in, which decides whether a diagnostic there is reported + contexts: FxIndexMap>, + module_reads: FxIndexSet, +} + +impl<'a, 'db> RuntimeReadChecker<'a, 'db> { + fn expression(&self, index: NodeIndex) -> Option<&'a ast::Expr> { + match self.parsed.get_by_index(index) { + AnyRootNodeRef::Expr(expr) => Some(expr), + _ => None, + } + } + + fn context(&mut self, node: &ast::Expr) -> Option<&InferContext<'db, 'a>> { + let file = self.model.program_file(); + let scope = semantic_index(self.db, file).try_expression_scope_id(node)?; + let (db, env, source_file, parsed) = (self.db, self.env, self.file, self.parsed); + Some(self.contexts.entry(scope).or_insert_with(|| { + InferContext::new( + db, + env, + scope.to_scope_id(db, file), + source_file, + file, + parsed, + ) + })) + } + + fn site(&self, node: &ast::Expr) -> RuntimeSite<'db> { + RuntimeSite::at(node, self.in_check_order) + } + + fn check(&mut self, site: &RuntimeReadSite, reports: bool) { + match site { + RuntimeReadSite::TypeTest { value, target } => { + if let (Some(value), Some(target)) = + (self.expression(*value), self.expression(*target)) + { + self.type_test(value, target, reports); + } + } + RuntimeReadSite::CheckedCast { value, target } => { + if let (Some(value), Some(target)) = + (self.expression(*value), self.expression(*target)) + { + self.checked_cast(value, target, reports); + } + } + RuntimeReadSite::Reified { + call, + generic, + reads, + } => { + if reports && let Some(call) = self.expression(*call) { + self.reified(call, generic, reads); + } + } + } + } + + /// a bare reified call or construction whose specialization reads a name that does not + /// hold, where the call runs, the binding the checker resolved + fn reified(&mut self, call: &ast::Expr, generic: &ReifiedGeneric, reads: &[Name]) { + let Some((name, binding)) = first_unbound_read(self.db, self.env, reads, self.site(call)) + else { + return; + }; + let failure = ReifiedInferenceError::NotAtRuntime(name, binding); + let Some(context) = self.context(call) else { + return; + }; + match generic { + ReifiedGeneric::Function(callee) => { + report_unspecialized_reified_call(context, call.range(), callee, Some(failure)); + } + ReifiedGeneric::Class(class) => { + report_unspecialized_reified_construction(context, call.range(), class, failure); + } + } + } + + fn type_test(&mut self, value: &ast::Expr, target: &'a ast::Expr, reports: bool) { + let (Some(value_ty), Some(target_ty)) = ( + value.inferred_type(self.model), + target.inferred_type(self.model), + ) else { + return; + }; + let written = WrittenTarget::checked(target, self.in_check_order); + let plan = type_test_plan( + self.db, + self.env, + self.file, + value_ty, + target_ty, + Some(written), + ); + self.read_spellings(&plan); + if reports && let ParametricIsPlan::ErasedTarget(reason) = plan { + let source = source_text(self.db, self.file); + let primary = TextRange::new(value.start(), target.end()); + if let Some(context) = self.context(target) { + report_erased_type_check(context, primary, &source[target.range()], reason); + } + } + } + + /// a checked cast's runtime check is the type test the cast shares with `is`, falling + /// back to the shallow target the soundness checks use, so both are read as spelled + fn checked_cast(&mut self, value: &ast::Expr, target: &'a ast::Expr, reports: bool) { + let (Some(value_ty), Some(target_ty)) = ( + value.inferred_type(self.model), + target.inferred_type(self.model), + ) else { + return; + }; + let (db, env, file) = (self.db, self.env, self.file); + let written = WrittenTarget::checked(target, self.in_check_order); + let plan = type_test_plan(db, env, file, value_ty, target_ty, Some(written)); + self.read_spellings(&plan); + let check = cast_check_plan(db, env, file, target_ty, Some(self.site(target))); + if let Some(spelling) = check.as_ref().and_then(CastCheck::spelling) { + self.module_reads.extend(spelling.global_reads().cloned()); + } + // a cast with no runtime check the checker can spell tests its value against the + // target as written, so a target reading a name that does not hold there the binding + // the checker resolved — a class imported only under `if TYPE_CHECKING:` — would + // raise `NameError`, or test against another class. a cast that needs no check (a + // widening) is a plain `typing.cast`, whose target is written as a string + if !reports || check.is_some() || cast_is_redundant(db, env, value_ty, target_ty) { + return; + } + let Some((name, binding)) = written_unbound_read(db, env, target, self.in_check_order) + else { + return; + }; + let Some(builder) = self + .context(target) + .and_then(|context| context.report_lint(&ERASED_TYPE_CHECK, target)) + else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{}` cannot be checked at runtime: {}", + target_ty.display(db, env), + binding.why_not_checkable(&name) + )); + diagnostic.info(binding.describe(&name)); + } + + fn read_spellings(&mut self, plan: &ParametricIsPlan) { + for spelling in plan.spellings() { + self.module_reads.extend(spelling.global_reads().cloned()); + } + } +} diff --git a/crates/ty_python_semantic/src/types/runtime_spelling.rs b/crates/ty_python_semantic/src/types/runtime_spelling.rs index fdf8378f47..3829cf4660 100644 --- a/crates/ty_python_semantic/src/types/runtime_spelling.rs +++ b/crates/ty_python_semantic/src/types/runtime_spelling.rs @@ -20,26 +20,34 @@ use ruff_text_size::{Ranged, TextRange}; use crate::Db; use crate::place::{RuntimeBinding, RuntimeRead, runtime_binding}; use crate::types::ProgramEnvironment; +use crate::types::check_order::InferredInCheckOrder; use crate::types::visibility::private_symbols; /// where a lowering writes a spelling into code that runs, which decides what each name it -/// reads holds there +/// reads holds there. what a name holds there is asked only once the scopes around the code +/// are inferred, so a site is made only with the proof that they are #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct RuntimeSite { +pub(crate) struct RuntimeSite<'db> { range: TextRange, + inferred: InferredInCheckOrder<'db>, } -impl RuntimeSite { - /// the code `node` is written as - pub(crate) fn at(node: &impl Ranged) -> Self { +impl<'db> RuntimeSite<'db> { + /// the code `node` is written as, in the file `inferred` is for + pub(crate) fn at(node: &impl Ranged, inferred: InferredInCheckOrder<'db>) -> Self { Self { range: node.range(), + inferred, } } pub(crate) fn range(self) -> TextRange { self.range } + + pub(crate) fn inferred(self) -> InferredInCheckOrder<'db> { + self.inferred + } } /// a python expression the checker spells for the transpiled program, which evaluates in @@ -117,21 +125,9 @@ impl RuntimeSpelling { &self, db: &'db dyn Db, env: &ProgramEnvironment<'db>, - file: File, - site: RuntimeSite, + site: RuntimeSite<'db>, ) -> Option<(Name, RuntimeBinding)> { - self.global_reads().find_map(|name| { - let binding = runtime_binding( - db, - env, - db.program_file(file), - RuntimeRead::Global { - name, - range: site.range, - }, - ); - (binding != RuntimeBinding::Bound).then(|| (name.clone(), binding)) - }) + first_unbound_read(db, env, self.global_reads(), site) } pub(crate) fn push_str(&mut self, text: &str) { @@ -192,3 +188,25 @@ impl RuntimeSpelling { self.spelled(ToOwned::to_owned) } } + +/// the first of the module-level names `reads` that does not hold, where `site` runs, the +/// binding the checker resolved — with what it holds instead +pub(crate) fn first_unbound_read<'a, 'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + reads: impl IntoIterator, + site: RuntimeSite<'db>, +) -> Option<(Name, RuntimeBinding)> { + reads.into_iter().find_map(|name| { + let binding = runtime_binding( + db, + env, + RuntimeRead::Global { + name, + range: site.range, + }, + site.inferred, + ); + (binding != RuntimeBinding::Bound).then(|| (name.clone(), binding)) + }) +} diff --git a/crates/ty_python_semantic/src/types/soundness.rs b/crates/ty_python_semantic/src/types/soundness.rs index 2dc18b0b3a..f170fd282a 100644 --- a/crates/ty_python_semantic/src/types/soundness.rs +++ b/crates/ty_python_semantic/src/types/soundness.rs @@ -12,7 +12,6 @@ //! what second argument does that check take? (`runtime_check_target`) use ruff_db::files::File; -use ruff_python_ast as ast; use crate::Db; use crate::place::{Place, explicit_global_symbol}; @@ -205,20 +204,20 @@ fn python_string_literal(value: &str) -> String { /// keeps the emitted code from ever running `isinstance` against a subscripted /// or non-`@runtime_checkable` protocol, which raises at runtime. /// -/// `written_at` is the cast's written target, where the check runs: a check whose target -/// reads a name that does not hold there the binding the checker resolved has no plan -pub fn cast_check_plan<'db>( +/// `site` is the cast's written target, where the check runs: a check whose target reads a +/// name that does not hold there the binding the checker resolved has no plan +pub(crate) fn cast_check_plan<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, ty: Type<'db>, - written_at: Option<&ast::Expr>, + site: Option>, ) -> Option { let plan = cast_check_plan_anywhere(db, env, file, ty)?; - if let Some(site) = written_at.map(RuntimeSite::at) + if let Some(site) = site && plan .spelling() - .is_some_and(|spelling| spelling.unbound_read(db, env, file, site).is_some()) + .is_some_and(|spelling| spelling.unbound_read(db, env, site).is_some()) { return None; } @@ -249,7 +248,7 @@ fn cast_check_plan_anywhere<'db>( /// whether a checked cast to `target` has no faithful runtime residue and must /// degrade to an unchecked `typing.cast`: a protocol target that -/// [`cast_check_plan`] cannot validate structurally. this is what stops the +/// `cast_check_plan` cannot validate structurally. this is what stops the /// transpiler emitting `isinstance(value, )`, which raises at runtime pub fn cast_target_is_unverifiable_protocol<'db>( db: &'db dyn Db, diff --git a/crates/ty_python_semantic/src/types/soundness_sites.rs b/crates/ty_python_semantic/src/types/soundness_sites.rs index 3954350d8f..bf42f4f961 100644 --- a/crates/ty_python_semantic/src/types/soundness_sites.rs +++ b/crates/ty_python_semantic/src/types/soundness_sites.rs @@ -14,7 +14,8 @@ use ty_python_core::semantic_index; use crate::Db; use crate::place::{RuntimeBinding, RuntimeRead, binds_one_class_wherever_bound, runtime_binding}; -use crate::semantic_model::{HasType, SemanticModel}; +use crate::semantic_model::{HasType, LoweringModel}; +use crate::types::check_order::InferredInCheckOrder; use crate::types::runtime_spelling::{RuntimeSite, RuntimeSpelling}; use crate::types::soundness::{ ArgSelector, CheckKind, CheckTarget, call_result_is_typevar_derived, @@ -26,7 +27,7 @@ use crate::types::{DynamicType, KnownInstanceType, ProgramEnvironment, Type}; /// every runtime soundness check `suite` makes under `positions`, in the order the /// checks are found pub fn soundness_sites<'ast>( - model: &SemanticModel<'_>, + model: &LoweringModel<'_>, suite: &'ast [Stmt], positions: SoundnessPositions, ) -> Vec> { @@ -175,7 +176,9 @@ pub struct SiteCheck { /// where the checks go, decided once over a syntax tree, and what is not a place /// for one struct Soundness<'a, 'db, 'ast> { - model: &'a SemanticModel<'db>, + model: &'a LoweringModel<'db>, + /// what a name the checks are written with holds is asked once the file is inferred + inferred: InferredInCheckOrder<'db>, positions: SoundnessPositions, sites: Vec>, /// gated expressions already covered by a wrap on an enclosing `!` @@ -188,9 +191,10 @@ struct Soundness<'a, 'db, 'ast> { } impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { - fn new(model: &'a SemanticModel<'db>, positions: SoundnessPositions) -> Self { + fn new(model: &'a LoweringModel<'db>, positions: SoundnessPositions) -> Self { Self { model, + inferred: model.inferred_in_check_order(), positions, sites: Vec::new(), consumed: Vec::new(), @@ -301,7 +305,7 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { fn site_check( &self, plan: CheckKind, - site: RuntimeSite, + site: RuntimeSite<'db>, runs_later: bool, ) -> Option { let db = self.db(); @@ -312,11 +316,11 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { match runtime_binding( db, &self.env(), - program_file, RuntimeRead::Global { name, range: site.range(), }, + site.inferred(), ) { RuntimeBinding::Bound => {} RuntimeBinding::Unbound | RuntimeBinding::NotTheCheckers @@ -343,8 +347,11 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { /// check the value `expr` produces against `plan` fn check_value(&mut self, expr: &'ast Expr, plan: CheckKind) { - let Some(check) = self.site_check(plan, RuntimeSite::at(expr), self.runs_later(expr)) - else { + let Some(check) = self.site_check( + plan, + RuntimeSite::at(expr, self.inferred), + self.runs_later(expr), + ) else { return; }; self.sites.push(SoundnessSite::Value { expr, check }); @@ -360,9 +367,11 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { let Some(plan) = self.check_plan(target_expr) else { return; }; - let Some(check) = - self.site_check(plan, RuntimeSite::at(iterable), self.runs_later(iterable)) - else { + let Some(check) = self.site_check( + plan, + RuntimeSite::at(iterable, self.inferred), + self.runs_later(iterable), + ) else { return; }; self.sites.push(SoundnessSite::Elements { @@ -440,7 +449,8 @@ impl<'a, 'db, 'ast> Soundness<'a, 'db, 'ast> { let parameter = &pwd.parameter; // the guards run where the function's body begins, when it is called if let Some(plan) = self.parameter_plan(parameter) - && let Some(check) = self.site_check(plan, RuntimeSite::at(func), true) + && let Some(check) = + self.site_check(plan, RuntimeSite::at(func, self.inferred), true) { guards.push((parameter, check)); } diff --git a/crates/ty_python_semantic/src/types/unused_imports.rs b/crates/ty_python_semantic/src/types/unused_imports.rs index ffc1b7f956..42f80fa04b 100644 --- a/crates/ty_python_semantic/src/types/unused_imports.rs +++ b/crates/ty_python_semantic/src/types/unused_imports.rs @@ -51,6 +51,7 @@ use crate::types::definition_resolution::user_visible_definitions; use crate::types::diagnostic::{TypeCheckDiagnostics, UNUSED_IMPORT}; use crate::types::extensions::{extension_granting_imports, extensions_in_module}; use crate::types::lowering_reads::spelled_module_reads; +use crate::types::runtime_reads::file_runtime_read_checks; use crate::types::{ProgramEnvironment, Type}; use crate::{Db, FxIndexMap}; @@ -224,8 +225,11 @@ impl<'db> Uses<'db> { .map(|extension| extension.name(db).clone()), ); // a type the transpiled program writes out names its classes as the module binds them - let model = crate::SemanticModel::new(db, file); + let model = crate::LoweringModel::new(db, file); module_names.extend(spelled_module_reads(&model, parsed.suite())); + for checks in file_runtime_read_checks(db, file) { + module_names.extend(checks.module_reads.iter().cloned()); + } Self { definitions,