From eec7b6151ef3e17a8c80357fd6218ad6c9cf0ef7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:51:48 +1000 Subject: [PATCH 01/15] load a staged module with python's own source loader rather than a wrapper around it, so a traceback raised while it is imported shows the line that imported and the line that failed, without python's import machinery or the runner between them --- crates/ty/src/by_commands.rs | 70 +++++++++++++++------- crates/ty/tests/by_e2e.rs | 60 +++++++++++++++++++ docs/basedpython/development/sourcemaps.md | 11 ++++ 3 files changed, 119 insertions(+), 22 deletions(-) diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 43d732428a..e4b91853c7 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -2131,11 +2131,18 @@ class _ByLoader: when the run ends. The `.by` the module was written as is the answer to every question those paths are asked, so it is what the module reports. - A module python's own loader would compile is compiled by `_by_code` instead, - under the `.by` path and lines. One any other loader stands behind — a test - runner that rewrites assertions as it compiles — is left to it, and only its + This is the loader of a module some other loader stands behind — a test runner + that rewrites assertions as it compiles. It is left to that loader, and only its `__file__` moves; its frames are rewritten to `.by` lines when they reach the - excepthook. + excepthook. A module python's own loader would compile gets `_BySourceLoader`. + + `__file__` is set as the module is created, and `exec_module` is not wrapped: + the inner loader's own is the one python calls. A wrapper's frame between + python's import machinery and the module's code is what kept python from + removing that machinery's frames from a traceback raised while importing — a + module that failed as it was imported showed `` + three times and this file once, above the line that failed. The cost is a + reload, which sets `__file__` back to the staged path and is not seen here. """ def __init__(self, inner, by_path): @@ -2143,30 +2150,47 @@ class _ByLoader: self._by_path = by_path def create_module(self, spec): - return self._by_inner.create_module(spec) - - def get_code(self, fullname): - if type(self._by_inner) is importlib.machinery.SourceFileLoader: - code = _by_code(self._by_inner.get_filename(fullname)) - if code is not None: - return code - return self._by_inner.get_code(fullname) - - def exec_module(self, module): + # python sets `__file__` on the module only when it has none + module = self._by_inner.create_module(spec) + if module is None: + module = types.ModuleType(spec.name) module.__file__ = self._by_path - if type(self._by_inner) is importlib.machinery.SourceFileLoader: - code = _by_code(self._by_inner.get_filename(module.__name__)) - if code is not None: - exec(code, module.__dict__) - return - self._by_inner.exec_module(module) + return module def __getattr__(self, name): return getattr(self._by_inner, name) +class _BySourceLoader(importlib.machinery.SourceFileLoader): + """The loader of a staged module python's own loader would load. + + It compiles the module with `_by_code`, under the `.by` path and lines, and + names the `.by` as `__file__`, like `_ByLoader`. It is python's own source + loader rather than a wrapper around one, so that executing the module is + python's own `exec_module`, whose frames python removes from a traceback raised + while importing: what a failed import shows is the line that imported and the + line that failed, as it is for a program python runs itself. + """ + + def __init__(self, fullname, path, by_path): + super().__init__(fullname, path) + self._by_path = by_path + + def get_code(self, fullname): + # python's `exec_module` asks for the code of the module it is about to + # run, which is in `sys.modules` by then — on an import and on a reload, + # which sets `__file__` back to the staged path just before + module = sys.modules.get(fullname) + if module is not None and getattr(module, "__loader__", None) is self: + module.__file__ = self._by_path + code = _by_code(self.get_filename(fullname)) + if code is not None: + return code + return super().get_code(fullname) + + class _ByFinder: - """Puts a `_ByLoader` on every staged module, whoever ends up finding it.""" + """Puts a loader of its own on every staged module, whoever ends up finding it.""" def find_spec(self, fullname, path=None, target=None): for finder in sys.meta_path: @@ -2179,7 +2203,9 @@ class _ByFinder: if spec is None: continue by_path = _by_source_of(getattr(spec, "origin", None)) - if by_path is not None and spec.loader is not None: + if by_path is not None and type(spec.loader) is importlib.machinery.SourceFileLoader: + spec.loader = _BySourceLoader(spec.loader.name, spec.loader.path, by_path) + elif by_path is not None and spec.loader is not None: spec.loader = _ByLoader(spec.loader, by_path) return spec return None diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 7a24d1e7c4..28a2d9b390 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -2529,6 +2529,66 @@ main() ); } +#[test] +fn run_traceback_of_a_failed_import_shows_no_import_machinery() { + // python removes its own import machinery from a traceback raised while a + // module is imported, so that what shows is the line that imported and the + // line that failed. a module `by run` imports has to come out the same: the + // traceback is formatted by `traceback.print_exc`, which no excepthook sees + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("broken.by"), + "\ +table = {'a': 1} +print(table['missing']) +", + ) + .unwrap(); + fs::write( + dir.path().join("main.by"), + "\ +import traceback + +try: + import broken +except KeyError: + traceback.print_exc() +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + let stderr = String::from_utf8_lossy(&output.stderr); + let frames: Vec<&str> = stderr + .lines() + .filter(|line| line.trim_start().starts_with("File ")) + .collect(); + assert_eq!( + frames.len(), + 2, + "the importing line and the failing one:\n{stderr}" + ); + assert!( + frames[0].contains("main.by") && frames[0].contains("line 4"), + "{stderr}" + ); + assert!( + frames[1].contains("broken.by") && frames[1].contains("line 2"), + "{stderr}" + ); +} + #[test] fn run_sourcemap_digests_match_the_files_they_describe() { // `SOURCEMAP` describes a pair of files, and a consumer that reports a `.by` diff --git a/docs/basedpython/development/sourcemaps.md b/docs/basedpython/development/sourcemaps.md index 7b4b2f98b9..708375058e 100644 --- a/docs/basedpython/development/sourcemaps.md +++ b/docs/basedpython/development/sourcemaps.md @@ -107,6 +107,17 @@ the module to python's loader. a `.by` saved over during the run is caught when names the one generated line that `.by` line was written as, or no line when there was more than one +the loader that does this is python's own source loader with that one method +replaced, not a wrapper around it. python removes its import machinery from a +traceback raised while a module is imported, but only when nothing stands between +that machinery and the module's code, and a wrapper's `exec_module` did: a module +that failed as it was imported showed `` three times +and the shim once above the line that failed, in every traceback no excepthook +rewrites. it now shows what python shows for a program it runs itself, the line +that imported and the line that failed. a module some other loader stands behind +— a test runner that rewrites assertions — keeps that loader, which python calls +directly; only its `__file__` is set, as the module is created + both tables are keyed by the generated path exactly as the map spells it. the shim resolves symlinks before matching a frame's filename against it — a temporary tree under `/tmp` on macOS is reached by a different path than the one From f76dec4312f931006c9e7b9edaf34e9a9d8659c7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:01:06 +1000 Subject: [PATCH 02/15] by build prints its diagnostics in the format --output-format or TY_OUTPUT_FORMAT asks for, as by check does, so an editor or a CI job can read them out of the build's output --- crates/ty/docs/cli.md | 11 +++++- crates/ty/src/args.rs | 7 ++++ crates/ty/src/by_commands.rs | 30 +++++++++++++-- crates/ty/src/lib.rs | 2 + crates/ty/tests/by_e2e.rs | 58 +++++++++++++++++++++++++++++ docs/basedpython/features/editor.md | 7 ++++ 6 files changed, 111 insertions(+), 4 deletions(-) diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index 91f296c1f4..1552beb93e 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -313,7 +313,16 @@ by build [OPTIONS]
--min-version version

minimum Python version the output must run on [default: the project's configured python version]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

--out, -o dir

Where to write the output [default: build in the project root, or dist with --wheels]

-
--print-manifest

Report what the build read and produced, as <kind> <value> lines.

+
--output-format output-format

The format to print the build's diagnostics in, as by check takes it.

+

They go to stderr whatever the format, among the lines saying what the build wrote, so a structured format is what lets a CI job or an editor read them out of the rest.

+

May also be set with the TY_OUTPUT_FORMAT environment variable.

Possible values:

+
    +
  • full: Print diagnostics verbosely, with context and helpful hints (default)
    • +
  • concise: Print diagnostics concisely, one per line
    • +
  • gitlab: Print diagnostics in the JSON format expected by GitLab Code Quality reports
    • +
  • github: Print diagnostics in the format used by GitHub Actions workflow error annotations
    • +
  • junit: Print diagnostics as a JUnit-style XML report
    • +
--print-manifest

Report what the build read and produced, as <kind> <value> lines.

input <path> for every file the project is made of — what a source distribution has to carry to rebuild into the same thing — and package <name> for every top-level package that came out.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index b5de2bc9d5..e95589e12c 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -177,6 +177,13 @@ pub(crate) enum Command { /// `package ` for every top-level package that came out. #[arg(long)] print_manifest: bool, + /// The format to print the build's diagnostics in, as `by check` takes it. + /// + /// They go to stderr whatever the format, among the lines saying what + /// the build wrote, so a structured format is what lets a CI job or an + /// editor read them out of the rest. + #[arg(long, env = EnvVars::TY_OUTPUT_FORMAT)] + output_format: Option, #[command(flatten)] lowering: LoweringArgs, }, diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index e4b91853c7..ca38825437 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -14,6 +14,7 @@ use ruff_db::diagnostic::{ use ruff_db::files::system_path_to_file; use ruff_db::system::{OsSystem, SystemPath}; use ruff_text_size::TextRange; +use ty_project::metadata::options::OutputFormat; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; use ty_site_packages::{PythonEnvironment, SysPrefixPathOrigin}; use ty_static::EnvVars; @@ -241,6 +242,7 @@ pub(crate) fn cmd_run( let status = render_check_and_transpile( &db, &handles, + OutputFormat::Full, &mut Emit { config: &config, gate: CheckGate::AllErrors, @@ -936,6 +938,7 @@ pub(crate) fn cmd_build( lowering: &LoweringArgs, out: Option<&Path>, print_manifest: bool, + output_format: OutputFormat, ) -> anyhow::Result { let cwd = std::env::current_dir().context("failed to get current directory")?; let mut config = version_config(min_version, &cwd)?; @@ -963,6 +966,7 @@ pub(crate) fn cmd_build( let status = render_check_and_transpile( &db, &handles, + output_format, &mut Emit { config: &config, gate: CheckGate::ParseErrorsOnly, @@ -1272,6 +1276,7 @@ pub(crate) fn cmd_compile( let transpiled = render_check_and_transpile( &db, &transpilable, + OutputFormat::Full, &mut Emit { config: &tree_config, gate: CheckGate::ParseErrorsOnly, @@ -1724,6 +1729,7 @@ fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { render_check_and_transpile( &db, &handles, + OutputFormat::Full, &mut Emit { config, gate: CheckGate::ParseErrorsOnly, @@ -2312,12 +2318,13 @@ fn compilable_files(root: &Path) -> Vec { fn render_check_and_transpile( db: &ProjectDatabase, handles: &[(PathBuf, ruff_db::files::File)], + format: OutputFormat, emit: &mut Emit<'_>, consume: impl FnMut(&Transpiled<'_>) -> anyhow::Result<()>, ) -> anyhow::Result { let emitted = by_stage::emit::check_and_transpile(db, handles, emit, consume)?; if !emitted.diagnostics.is_empty() { - render_diagnostics(db, &emitted.diagnostics)?; + render_diagnostics_as(format, db, &emitted.diagnostics)?; } Ok(if emitted.ok { ExitStatus::Success @@ -2328,14 +2335,28 @@ fn render_check_and_transpile( }) } -/// Render diagnostics to stderr in the same format as `by check`. The +/// [`render_diagnostics_as`] in the full format, which every command but +/// `build` prints its diagnostics in. +fn render_diagnostics(db: &ProjectDatabase, diagnostics: &[Diagnostic]) -> anyhow::Result<()> { + render_diagnostics_as(OutputFormat::Full, db, diagnostics) +} + +/// Render diagnostics to stderr in `format`, as `by check` renders it. The /// transpiled output goes to stdout, so diagnostics belong on stderr to keep /// the two streams separable. +/// +/// The count is a sentence for a person, so a format made for a program goes +/// without it, as `by check`'s does. #[allow(clippy::print_stderr)] -fn render_diagnostics(db: &ProjectDatabase, diagnostics: &[Diagnostic]) -> anyhow::Result<()> { +fn render_diagnostics_as( + format: OutputFormat, + db: &ProjectDatabase, + diagnostics: &[Diagnostic], +) -> anyhow::Result<()> { use std::io::Write as _; let display_config = DisplayDiagnosticConfig::new("ty") + .format(format.into()) .color(colored::control::SHOULD_COLORIZE.should_colorize()) .context(0); let mut stderr = std::io::stderr().lock(); @@ -2344,6 +2365,9 @@ fn render_diagnostics(db: &ProjectDatabase, diagnostics: &[Diagnostic]) -> anyho "{}", DisplayDiagnostics::new(db, &display_config, diagnostics) )?; + if !format.is_human_readable() { + return Ok(()); + } let n = diagnostics.len(); writeln!( stderr, diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index 8a58a3b049..4b4ca946a6 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -161,6 +161,7 @@ fn run_command(command: Command) -> anyhow::Result { wheels, out, print_manifest, + output_format, lowering, } => { if wheels { @@ -171,6 +172,7 @@ fn run_command(command: Command) -> anyhow::Result { &lowering, out.as_deref(), print_manifest, + output_format.map(Into::into).unwrap_or_default(), ) } } diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 28a2d9b390..6c92b60b1d 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -3017,6 +3017,64 @@ fn build_renders_parse_error_and_aborts() { ); } +/// `--output-format` puts each diagnostic on a line of its own, apart from the lines saying what +/// the build wrote, and drops the count, which is a sentence for a person +#[test] +fn build_prints_its_diagnostics_in_the_format_asked_for() { + let dir = Scratch::new("build_output_format"); + fs::create_dir_all(&*dir).unwrap(); + fs::write(dir.join("main.by"), "x: int = \"nope\"\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["build", "--output-format", "github"]) + .current_dir(&*dir) + .output() + .expect("failed to spawn by"); + + assert!( + !output.status.success(), + "a diagnostic still fails the build" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + let diagnostics: Vec<&str> = stderr + .lines() + .filter(|line| line.starts_with("::")) + .collect(); + assert_eq!(diagnostics.len(), 1, "one line per diagnostic:\n{stderr}"); + assert!( + diagnostics[0].starts_with("::error title=ty (invalid-assignment),file=") + && diagnostics[0].contains("main.by,line=1,col=10,"), + "{stderr}" + ); + assert!(stderr.contains("main.by -> "), "{stderr}"); + assert!(!stderr.contains("Found 1 diagnostic"), "{stderr}"); + // the output is written whatever the format + assert!(dir.join("build").join("main.py").exists()); +} + +/// the format is read from `TY_OUTPUT_FORMAT` too, as `by check` reads it +#[test] +fn build_reads_the_output_format_from_the_environment() { + let dir = Scratch::new("build_output_format_env"); + fs::create_dir_all(&*dir).unwrap(); + fs::write(dir.join("main.by"), "x: int = \"nope\"\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .env(EnvVars::TY_OUTPUT_FORMAT, "concise") + .arg("build") + .current_dir(&*dir) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("main.by:1:10: error[invalid-assignment] "), + "{stderr}" + ); +} + #[test] fn run_refuses_to_execute_on_check_errors() { // a program that fails `by check` must never execute — here `T` cannot be diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index dce6b83e40..ad6b56d7d2 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -491,6 +491,13 @@ the build follows the *module* tree, not the directory tree, so a src-layout project's `src/pkg/main.by` is `build/pkg/main.py` — which is why this is the build's answer rather than a path an editor can assemble +an editor that runs the build itself, for a *build project* action, takes the +build's diagnostics from its output rather than reading them as prose: +`by build --output-format github`, or `TY_OUTPUT_FORMAT=github` in the build's +environment as `by check` reads it, prints each diagnostic on a line of its own, +with its file, place and severity as named fields, among the lines saying what +the build wrote + `by/syntaxOutline` answers one document whole, on every revision the client wants to serve features from: it is what an editor needs to type in an indentation-delimited language — which line opens a suite, where a compound From 01b4ff6cac66b6dc975c0aa094b03fee22ff0957 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:26:57 +1000 Subject: [PATCH 03/15] add `by run --coverage`, which measures the program with coverage.py and writes an LCOV report against the .by files and lines it was written in --- crates/ty/docs/cli.md | 2 + crates/ty/src/args.rs | 12 + crates/ty/src/by_commands.rs | 321 ++++++++++++++++++-- crates/ty/src/by_coverage.rs | 200 ++++++++++++ crates/ty/src/cli-reference.md | 17 ++ crates/ty/src/lib.rs | 3 + crates/ty/tests/by_e2e.rs | 485 ++++++++++++++++++++++++++++++ docs/basedpython/cli-reference.md | 28 ++ 8 files changed, 1048 insertions(+), 20 deletions(-) create mode 100644 crates/ty/src/by_coverage.rs diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index 1552beb93e..209cac3f98 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -255,6 +255,8 @@ by run [OPTIONS] [MODULE] [ARGS]...
--compiled

Compile every imported module to a native extension first.

A function the compiler declines still runs — from the interpreted source embedded in the extension — so this changes speed, not behaviour. Needs a C toolchain and python development headers.

The entry module itself stays interpreted: running something as __main__ needs a code object, and an extension module has none.

+
--coverage path

Measure which lines run, with coverage.py, and write them to PATH as an LCOV report.

+

A transpiled module is reported against its .by file and lines: a .by line is covered when any statement written for it ran. A .py the project carries over is reported against the file in the project rather than the staged copy that ran. The project's coverage.py configuration applies to the project's files as they are in the project, with --in-build too, and coverage.py has to be installed in the interpreter the program runs on.

--help, -h

Print help (see a summary with '-h')

--in-build

Run in the tree this staged instead of in the current directory.

For a program whose arguments are the project's own files — a test runner, a coverage run, a type checker over the output. Those files are python only in the staged tree, so a path naming one resolves only from inside it. Everything else wants the default: a program that reads or writes beside the project has to see the directory it was started in.

diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index e95589e12c..6501251569 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -120,6 +120,18 @@ pub(crate) enum Command { /// was started in. #[arg(long)] in_build: bool, + /// Measure which lines run, with coverage.py, and write them to PATH as + /// an LCOV report. + /// + /// A transpiled module is reported against its `.by` file and lines: a + /// `.by` line is covered when any statement written for it ran. A `.py` + /// the project carries over is reported against the file in the project + /// rather than the staged copy that ran. The project's coverage.py + /// configuration applies to the project's files as they are in the + /// project, with `--in-build` too, and coverage.py has to be installed in + /// the interpreter the program runs on. + #[arg(long, value_name = "PATH", conflicts_with = "compiled")] + coverage: Option, }, /// Start a new project. diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index ca38825437..6198450da1 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::ffi::OsStr; use std::fs; use std::io::{self, Read}; @@ -128,9 +128,9 @@ impl LoweringArgs { /// How `by run` was invoked, beyond what it is running. /// -/// The four choices a caller makes about the *process*: which interpreter, what -/// stands in front of it, whether its modules are compiled first, and which -/// directory it runs in. [`CompileFlags`] groups `by compile`'s for the same +/// The choices a caller makes about the *process*: which interpreter, what +/// stands in front of it, whether its modules are compiled first, which +/// directory it runs in, and whether what it runs is measured. [`CompileFlags`] groups `by compile`'s for the same /// reason. #[derive(Debug, Clone, Copy, Default)] pub(crate) struct RunFlags<'a> { @@ -142,6 +142,8 @@ pub(crate) struct RunFlags<'a> { pub(crate) python: Option<&'a Path>, /// `--launcher`: a program to start the interpreter through. pub(crate) launcher: Option<&'a Path>, + /// `--coverage`: where to write an LCOV report of what ran. + pub(crate) coverage: Option<&'a Path>, } #[allow(clippy::exit, clippy::print_stderr)] @@ -157,6 +159,7 @@ pub(crate) fn cmd_run( in_build, python: python_flag, launcher, + coverage, } = flags; let cwd = std::env::current_dir().context("failed to get current directory")?; // one resolution of the project, for the environment and the target version @@ -278,6 +281,17 @@ pub(crate) fn cmd_run( &mut staging, &BuildRecord::new(&root, &roots, Some(module.clone()), compiled, &config), )?; + // the project file behind each copy in the tree, for the runner to run it as + // and a coverage report to name: the copy is deleted with the tree when the + // run ends + let carried: BTreeMap = staging + .entries() + .filter_map(|(staged, source)| { + let source = source.filter(|source| source.extension() != Some(OsStr::new("by")))?; + Some((staged.to_path_buf(), source.to_path_buf())) + }) + .collect(); + write_project_module(&mut staging, &root, &carried)?; staging.finish()?; if compiled { @@ -367,8 +381,18 @@ pub(crate) fn cmd_run( // the whole project is inside it, configuration and data included, so what // the program makes of the tree is what it would have made of the project let workdir = if in_build { tmp.path() } else { cwd.as_path() }; + command.arg(tmp.path().join(BY_RUNNER_FILENAME)); + // what the runner measured, written outside the tree the program can see + let measured = match coverage { + Some(_) => { + let dir = tempfile::TempDir::new().context("failed to create temp directory")?; + let file = dir.path().join("measured.json"); + command.arg("--coverage").arg(&file); + Some((dir, file)) + } + None => None, + }; let status = command - .arg(tmp.path().join(BY_RUNNER_FILENAME)) .arg(&module) .args(args) .current_dir(workdir) @@ -385,7 +409,28 @@ pub(crate) fn cmd_run( ), })?; - let code = status.code().unwrap_or(1); + let mut code = status.code().unwrap_or(1); + if let (Some(report), Some((_, file))) = (coverage, &measured) { + if file.is_file() { + let report = cwd.join(report); + if let Err(error) = + crate::by_coverage::write_report(file, &report, tmp.path(), &carried) + { + eprintln!("error: no coverage report was written: {error:#}"); + if code == 0 { + code = 1; + } + } + } else if code != 2 { + // 2 is the runner refusing to start without coverage.py, which it has + // said already + eprintln!( + "note: no coverage report was written: the program ended without returning \ + to `by run`, as `os._exit` does" + ); + } + } + drop(measured); // drop the temp dir explicitly: `process::exit` skips destructors, so // exiting while it's still in scope would leak the directory drop(tmp); @@ -1785,14 +1830,80 @@ fn write_traceback_runtime( // through the staging, so that a project of its own with this name is // reported as the collision it is rather than silently overwritten by a shim // it knows nothing about - staging.write(Path::new(BY_RUNNER_FILENAME), None, BY_RUNNER_SRC) + staging.write(Path::new(BY_RUNNER_FILENAME), None, BY_RUNNER_SRC)?; + staging.write( + Path::new(BY_COVERAGE_PLUGIN_FILENAME), + None, + BY_COVERAGE_PLUGIN_SRC, + ) +} + +/// the coverage.py plugin `by run --coverage` measures with, which the runner +/// names by its module name +const BY_COVERAGE_PLUGIN_FILENAME: &str = "_by_coverage_plugin.py"; + +const BY_COVERAGE_PLUGIN_SRC: &str = r#"# generated by `by run` — the `.by` modules a run can leave out, for coverage.py +import os + +import coverage + +from _by_sourcemap import SOURCEMAP + + +class _ByModules(coverage.CoveragePlugin): + """The `.by` files transpiled into the tree, as files coverage.py looks for + under its `source` to report the ones that never ran, as it does a `.py`. + + It traces nothing: a `.by` module that runs is compiled under its `.by` path + and lines, and measured like any python file.""" + + def file_tracer(self, filename): + return None + + def find_executable_files(self, src_dir): + within = os.path.join(os.path.realpath(src_dir), "") + for by_path, _ in SOURCEMAP.values(): + if os.path.realpath(by_path).startswith(within): + yield by_path + + +def coverage_init(reg, options): + reg.add_file_tracer(_ByModules()) +"#; + +/// the runner's record of the project the tree was staged from +const BY_PROJECT_FILENAME: &str = "_by_project.py"; + +/// writes `_by_project.py`: the project's root, and the project file each +/// hand-written module in the tree is a copy of, which the runner runs it as +fn write_project_module( + staging: &mut Staging, + root: &Path, + carried: &BTreeMap, +) -> anyhow::Result<()> { + // a json string is a python string literal, escapes included + let modules: BTreeMap = carried + .iter() + .filter(|(_, source)| source.extension() == Some(OsStr::new("py"))) + .map(|(staged, source)| (staged.display().to_string(), source.display().to_string())) + .collect(); + let text = format!( + "# generated by `by run` — the project this tree was staged from\n\ + ROOT = {}\n\ + CARRIED = {}\n", + serde_json::to_string(&root.display().to_string())?, + serde_json::to_string(&modules)?, + ); + staging.write(Path::new(BY_PROJECT_FILENAME), None, &text) } const BY_RUNNER_SRC: &str = r#"# generated by `by run` — runs the target module with .by-aware tracebacks +import atexit import dis import hashlib import importlib.machinery import importlib.util +import json import linecache import os import sys @@ -1800,6 +1911,7 @@ import traceback import types import warnings +from _by_project import CARRIED, ROOT from _by_sourcemap import DIGESTS, SOURCEMAP # index the sourcemap by realpath so symlinked temp dirs (e.g. /tmp on macOS) @@ -1812,6 +1924,13 @@ _BY_SOURCES = {os.path.realpath(info[0]): py for py, info in SOURCEMAP.items()} # this file, whose frames are never the program's _RUNNER = os.path.realpath(__file__) +# the tree the project was staged into, which this file is at the top of +_TREE = os.path.dirname(_RUNNER) +# the hand-written `.py` files carried over into the tree, each with the project +# file it is a copy of +_CARRIED = { + os.path.realpath(os.path.join(_TREE, staged)): source for staged, source in CARRIED.items() +} # entries already checked against their digests, so each file is read at most # once no matter how many frames it contributes @@ -1960,13 +2079,34 @@ def _excepthook(etype, evalue, tb): sys.stderr.write("".join(traceback.format_exception_only(etype, evalue))) -def _by_source_of(origin): - # the .by file a staged .py was transpiled from, or None for anything else — - # a hand-written .py carried over verbatim, or a module from site-packages +def _project_file_of(origin): + # the project file a staged module stands for: the .by a staged .py was + # transpiled from, or the hand-written .py it is a copy of. None for anything + # else, such as a module from site-packages if not origin: return None - entry = _BY_MAP.get(os.path.realpath(origin)) - return None if entry is None else entry[1][0] + real = os.path.realpath(origin) + entry = _BY_MAP.get(real) + return _CARRIED.get(real) if entry is None else entry[1][0] + + +def _project_code(origin): + """The staged module at `origin` compiled as the project file it stands for, or + `None` when it stands for none, which leaves it to python's own loader. + + A hand-written `.py` is a copy, so it is compiled from its text under the + project file's path: what python derives from the code object — a traceback, + `inspect`, what coverage.py measures — names the file the author wrote, as it + does for a `.by`.""" + code = _by_code(origin) + if code is not None: + return code + project_file = _CARRIED.get(os.path.realpath(origin)) if origin else None + if project_file is None: + return None + with open(origin, "rb") as handle: + source = handle.read() + return compile(source, project_file, "exec", dont_inherit=True) def _varint(out, value): @@ -2129,13 +2269,14 @@ def _by_code(origin): class _ByLoader: - """Wraps a staged module's loader to name its `.by` source as `__file__`. + """Wraps a staged module's loader to name its project file as `__file__`. The program is run out of a directory of transpiled copies, so every path python derives from a module's origin — `__file__`, and anything a tool walks up from it to find — names a temporary file in a directory that is deleted - when the run ends. The `.by` the module was written as is the answer to every - question those paths are asked, so it is what the module reports. + when the run ends. The file the module was written as, a `.by` or a + hand-written `.py`, is the answer to every question those paths are asked, so + it is what the module reports. This is the loader of a module some other loader stands behind — a test runner that rewrites assertions as it compiles. It is left to that loader, and only its @@ -2170,8 +2311,9 @@ class _ByLoader: class _BySourceLoader(importlib.machinery.SourceFileLoader): """The loader of a staged module python's own loader would load. - It compiles the module with `_by_code`, under the `.by` path and lines, and - names the `.by` as `__file__`, like `_ByLoader`. It is python's own source + It compiles the module with `_project_code`, under its project file's path — + and a `.by`'s lines — and names that file as `__file__`, like `_ByLoader`. It + is python's own source loader rather than a wrapper around one, so that executing the module is python's own `exec_module`, whose frames python removes from a traceback raised while importing: what a failed import shows is the line that imported and the @@ -2189,7 +2331,7 @@ class _BySourceLoader(importlib.machinery.SourceFileLoader): module = sys.modules.get(fullname) if module is not None and getattr(module, "__loader__", None) is self: module.__file__ = self._by_path - code = _by_code(self.get_filename(fullname)) + code = _project_code(self.get_filename(fullname)) if code is not None: return code return super().get_code(fullname) @@ -2208,7 +2350,7 @@ class _ByFinder: spec = find(fullname, path, target) if spec is None: continue - by_path = _by_source_of(getattr(spec, "origin", None)) + by_path = _project_file_of(getattr(spec, "origin", None)) if by_path is not None and type(spec.loader) is importlib.machinery.SourceFileLoader: spec.loader = _BySourceLoader(spec.loader.name, spec.loader.path, by_path) elif by_path is not None and spec.loader is not None: @@ -2257,7 +2399,7 @@ def _run_as_main(module): code = spec.loader.get_code(module) if code is None: raise ImportError("module " + repr(module) + " has no code to run", name=module) - by_path = _by_source_of(spec.origin) or spec.origin + by_path = _project_file_of(spec.origin) or spec.origin sys.argv[0] = by_path main_module = types.ModuleType("__main__") main_module.__file__ = by_path @@ -2274,11 +2416,150 @@ def _run_as_main(module): exec(code, main_module.__dict__) +def _start_coverage(report): + """Measure the program with coverage.py, and write what ran to `report` once the + program has ended: after its other threads and its own `atexit` handlers, which + python runs before the ones registered ahead of them.""" + try: + import coverage + from coverage.exceptions import CoverageWarning + except ImportError: + sys.stderr.write( + "error: `by run --coverage` measures with coverage.py, which " + f"{sys.executable} cannot import\n" + " install it into that environment: `uv add --dev coverage`, or " + f"`{sys.executable} -m pip install coverage`\n" + ) + sys.exit(2) + # the project's coverage configuration names the project's files, which is what + # every module runs as, so it is read, and its relative paths resolved, where + # the project is. a program run in the tree (`--in-build`) is in the project's + # directory that the tree's directory is a copy of + here = os.getcwd() + within = os.path.relpath(os.path.realpath(here), _TREE) + project_dir = here if within.split(os.sep)[0] == os.pardir else os.path.join(ROOT, within) + os.chdir(project_dir) + try: + # no data file: what was measured is read back below, and a `.coverage` left + # in the directory the program ran in would be a second, unmapped report of it + measurement = coverage.Coverage(data_file=None) + measurement.set_option( + "run:plugins", [*measurement.get_option("run:plugins"), "_by_coverage_plugin"] + ) + with warnings.catch_warnings(): + # a core that runs no plugin's tracer says so, and the plugin traces + # nothing: it only lists the `.by` files, which coverage.py asks it for + # whatever the core + warnings.filterwarnings( + "ignore", + message=r"Plugin file tracers \(_by_coverage_plugin\._ByModules\) aren't supported", + category=CoverageWarning, + ) + measurement.start() + finally: + os.chdir(here) + atexit.register(_write_coverage, measurement, report) + + +def _analysed(measurement, filename): + """The statements of `filename` and those of them that ran, as coverage.py counts + them, or `None` for a file coverage.py cannot read as python.""" + try: + _, statements, _, missing, _ = measurement.analysis2(filename) + except Exception: + # a file python ran that has no python source to read: code compiled from a + # string under a made-up filename, which coverage.py's own report skips too + return None + return statements, sorted(set(statements) - set(missing)) + + +def _generated_analysis(measurement, filename, lines): + """The statements of the generated `.py` at `filename` and those of them `lines` + ran, as coverage.py's report reads them: each line that ran is put on the + statement it is part of. `None` when coverage.py cannot read the file.""" + from coverage.exceptions import NoSource + from coverage.python import PythonFileReporter + + try: + reporter = PythonFileReporter(filename, measurement) + statements = reporter.lines() + except NoSource: + return None + return statements, reporter.translate_lines(lines) & statements + + +def _write_coverage(measurement, report): + """What ran, with every line of a transpiled module moved to its `.by` line. + + coverage.py records a module by the filename and lines of its code objects. one + `_by_code` compiled is already the `.by` with its lines, and one another loader + compiled — a test runner rewriting assertions — is the staged `.py` with its own. + either way the statements are the generated python's, so each module's lines are + put back on its `.py`, analysed there, and only then moved through the sourcemap + to the `.by`. a `.by` line is covered when any statement written for it ran. + + every other file is reported as coverage.py measured it, staged copies included: + `by run` puts those back where they came from once the run ends. + """ + measurement.stop() + data = measurement.get_data() + ran = {} + files = {} + for measured in data.measured_files(): + real = os.path.realpath(measured) + lines = data.lines(measured) or [] + if real in _BY_SOURCES: + key = _BY_SOURCES[real] + by_lines = set(lines) + generated = SOURCEMAP[key][1] + ran.setdefault(key, set()).update( + index + 1 + for index, line in enumerate(generated) + if line is not None and line + 1 in by_lines + ) + elif real in _BY_MAP: + ran.setdefault(_BY_MAP[real][0], set()).update(lines) + else: + analysed = _analysed(measurement, measured) + if analysed is not None: + files[measured] = {"statements": analysed[0], "executed": analysed[1]} + for key, lines in ran.items(): + if key in _STALE: + # its `.by` was saved over during the run, so no line of it is the line + # this ran as + continue + by_path, generated = SOURCEMAP[key] + analysed = _generated_analysis(measurement, key, lines) + if analysed is None: + continue + statements, executed = ( + { + generated[number - 1] + 1 + for number in numbers + if 0 < number <= len(generated) and generated[number - 1] is not None + } + for numbers in analysed + ) + entry = files.setdefault(by_path, {"statements": [], "executed": []}) + entry["statements"] = sorted(statements.union(entry["statements"])) + entry["executed"] = sorted(executed.union(entry["executed"])) + with open(report, "w", encoding="utf-8") as handle: + json.dump({"files": files}, handle) + + def main(): sys.excepthook = _excepthook sys.meta_path.insert(0, _ByFinder()) + # `by run --coverage` names where to write what ran in front of the module, + # where no module name can be + report = None + if sys.argv[1] == "--coverage": + report = sys.argv[2] + del sys.argv[1:3] module = sys.argv[1] sys.argv = sys.argv[1:] + if report is not None: + _start_coverage(report) try: _run_as_main(module) except SystemExit: diff --git a/crates/ty/src/by_coverage.rs b/crates/ty/src/by_coverage.rs new file mode 100644 index 0000000000..aa1b49383e --- /dev/null +++ b/crates/ty/src/by_coverage.rs @@ -0,0 +1,200 @@ +//! `by run --coverage`: what a run executed, as an LCOV report against the files +//! the project is written in. +//! +//! The measuring happens in the program's own process, where coverage.py runs: +//! the runner `by run` stages measures the program, moves each transpiled +//! module's lines to its `.by` through the sourcemap it already holds, and writes +//! what it found as JSON (see `_write_coverage` in the runner). A `.py` the +//! project carries over runs as its project file, but one a test runner compiled +//! itself runs as the copy it compiled, in a temporary directory that is deleted +//! when the run ends. The staging knows what each copy is a copy of, so the +//! report is finished here: every staged copy is put back on the file it was +//! copied from, and the runner's own files, which belong to no project, are left +//! out. +//! +//! LCOV because it is the format the tools that read coverage share: an editor's +//! gutter, `genhtml`, a CI service. Each line is reported as run once or not at +//! all, since coverage.py records whether a line ran rather than how often. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::Context; +use serde::Deserialize; + +/// What the runner measured, keyed by the path it ran as. +#[derive(Debug, Deserialize)] +struct Measured { + files: BTreeMap, +} + +#[derive(Debug, Default, Deserialize)] +struct Lines { + /// every line that holds the start of a statement + statements: Vec, + /// the statement lines that ran + executed: Vec, +} + +/// One file's lines in the report. +#[derive(Debug, Default, PartialEq, Eq)] +struct Covered { + statements: BTreeSet, + executed: BTreeSet, +} + +/// Turn the runner's measurement at `measured` into an LCOV report at `report`. +/// +/// `tree` is the staged directory the program ran in, and `carried` maps a path +/// within it to the project file it is a copy of. +pub(crate) fn write_report( + measured: &Path, + report: &Path, + tree: &Path, + carried: &BTreeMap, +) -> anyhow::Result<()> { + let text = fs::read_to_string(measured) + .with_context(|| format!("could not read {}", measured.display()))?; + let measured: Measured = serde_json::from_str(&text) + .with_context(|| format!("could not parse {}", measured.display()))?; + // python reports a file under its real path, and a temporary directory is + // reached through a symlink on macos, so the tree is recognised either way + let trees: Vec = std::iter::once(tree.to_path_buf()) + .chain(fs::canonicalize(tree).ok()) + .collect(); + let covered = project_files(measured, &trees, carried); + if let Some(parent) = report + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent) + .with_context(|| format!("could not create {}", parent.display()))?; + } + fs::write(report, render_lcov(&covered)) + .with_context(|| format!("could not write {}", report.display())) +} + +/// The measurement with every staged copy moved back to its project file, and +/// everything else in the staged tree dropped. +fn project_files( + measured: Measured, + trees: &[PathBuf], + carried: &BTreeMap, +) -> BTreeMap { + let mut covered: BTreeMap = BTreeMap::new(); + for (path, lines) in measured.files { + let path = PathBuf::from(path); + let staged = trees.iter().find_map(|tree| path.strip_prefix(tree).ok()); + let file = match staged { + // the runner, the sourcemap and the runtime helpers are staged from + // nothing, and a transpiled module's lines were moved to its `.by` + // already, so only a copy is put back + Some(relative) => match carried.get(relative) { + Some(source) => source.clone(), + None => continue, + }, + None => path, + }; + let entry = covered.entry(file).or_default(); + entry.statements.extend(&lines.statements); + entry.statements.extend(&lines.executed); + entry.executed.extend(lines.executed); + } + covered +} + +/// `covered` as LCOV tracefile records, one per file. +fn render_lcov(covered: &BTreeMap) -> String { + let mut out = String::new(); + for (file, lines) in covered { + let _ = writeln!(out, "TN:"); + let _ = writeln!(out, "SF:{}", file.display()); + for line in &lines.statements { + let hits = u32::from(lines.executed.contains(line)); + let _ = writeln!(out, "DA:{line},{hits}"); + } + let _ = writeln!(out, "LF:{}", lines.statements.len()); + let _ = writeln!(out, "LH:{}", lines.executed.len()); + let _ = writeln!(out, "end_of_record"); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn measured(json: &str) -> Measured { + serde_json::from_str(json).expect("valid measurement") + } + + #[test] + fn a_file_outside_the_tree_is_reported_where_it_ran() { + let covered = project_files( + measured(r#"{"files": {"/p/main.by": {"statements": [1, 2, 4], "executed": [1, 4]}}}"#), + &[PathBuf::from("/tmp/run")], + &BTreeMap::new(), + ); + assert_eq!( + render_lcov(&covered), + "TN:\nSF:/p/main.by\nDA:1,1\nDA:2,0\nDA:4,1\nLF:3\nLH:2\nend_of_record\n" + ); + } + + /// the copy ran out of a directory that is gone by the time anyone reads the + /// report, so it is reported against the file it was copied from + #[test] + fn a_staged_copy_is_reported_against_its_project_file() { + let carried = BTreeMap::from([( + PathBuf::from("pkg/helper.py"), + PathBuf::from("/p/pkg/helper.py"), + )]); + let covered = project_files( + measured( + r#"{"files": {"/private/tmp/run/pkg/helper.py": {"statements": [1], "executed": [1]}}}"#, + ), + &[PathBuf::from("/tmp/run"), PathBuf::from("/private/tmp/run")], + &carried, + ); + assert_eq!( + covered.keys().collect::>(), + [Path::new("/p/pkg/helper.py")] + ); + } + + /// the runner and the sourcemap are `by run`'s, not the project's + #[test] + fn a_file_staged_from_nothing_is_left_out() { + let covered = project_files( + measured( + r#"{"files": {"/tmp/run/_by_runner.py": {"statements": [1], "executed": [1]}}}"#, + ), + &[PathBuf::from("/tmp/run")], + &BTreeMap::new(), + ); + assert!(covered.is_empty(), "{covered:?}"); + } + + /// a project's `.py` can run both from where it was written and as the copy + /// staged beside the transpiled modules, and a line either of them ran is covered + #[test] + fn two_measurements_of_one_file_are_merged() { + let carried = BTreeMap::from([(PathBuf::from("helper.py"), PathBuf::from("/p/helper.py"))]); + let covered = project_files( + measured( + r#"{"files": { + "/p/helper.py": {"statements": [1, 2], "executed": [1]}, + "/tmp/run/helper.py": {"statements": [1, 2], "executed": [2]} + }}"#, + ), + &[PathBuf::from("/tmp/run")], + &carried, + ); + assert_eq!( + render_lcov(&covered), + "TN:\nSF:/p/helper.py\nDA:1,1\nDA:2,1\nLF:2\nLH:2\nend_of_record\n" + ); + } +} diff --git a/crates/ty/src/cli-reference.md b/crates/ty/src/cli-reference.md index b0db245e93..a8e0c3df6f 100644 --- a/crates/ty/src/cli-reference.md +++ b/crates/ty/src/cli-reference.md @@ -38,6 +38,23 @@ there, beside every file the build carries over unchanged, `pyproject.toml` included — so a path naming one of its modules resolves, and a tool reports paths relative to a tree laid out like the project. +### `--coverage` + +measure which lines the program runs and write them as an LCOV report: + +```sh +by run --coverage coverage.lcov main +by run --coverage coverage.lcov --in-build pytest +``` + +the report names the files you wrote: a transpiled module by its `.by` and the +`.by` lines its statements came from, and a `.py` the build carries over by the +file in the project rather than the staged copy that ran. measuring is +coverage.py's, so it has to be installed in the interpreter the program runs on, +and the project's coverage.py configuration applies: its `source`, `include` and +`omit` name the project's files as they are in the project, with `--in-build` +too + ## `by build` transpile every `.by` file in the project to `build/`, mirroring the module diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index 4b4ca946a6..288916e382 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -1,5 +1,6 @@ mod args; mod by_commands; +mod by_coverage; mod by_init; mod by_lowering; mod by_project_server; @@ -124,6 +125,7 @@ fn run_command(command: Command) -> anyhow::Result { lowering, compiled, in_build, + coverage, } => by_commands::cmd_run( module.as_deref(), &args, @@ -134,6 +136,7 @@ fn run_command(command: Command) -> anyhow::Result { in_build, python: python.as_deref(), launcher: launcher.as_deref(), + coverage: coverage.as_deref(), }, ), Command::Init { diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 6c92b60b1d..923e65ba2b 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1042,6 +1042,491 @@ fn run_in_build_runs_in_the_tree_it_staged() { ); } +/// The interpreter the coverage tests run on, and whether it can import +/// coverage.py: `$PYTHON` when it is set, as for every python-executing test, +/// and `python3` otherwise. +fn coverage_interpreter() -> (String, bool) { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let importable = Command::new(&python) + .args(["-c", "import coverage"]) + .status() + .is_ok_and(|status| status.success()); + (python, importable) +} + +/// A project with a transpiled module that runs only partly, a hand-written +/// `.py` the build carries over, and an entry point calling into both. +fn project_with_partly_covered_modules(dir: &Path) { + let package = dir.join("pkg"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.py"), "").unwrap(); + fs::write( + package.join("calc.by"), + "def add(a: int, b: int) -> int:\n if a > 100:\n return -1\n return a + b\n", + ) + .unwrap(); + fs::write( + package.join("plain.py"), + "def helper() -> int:\n return 3\n\n\ndef never() -> int:\n return 4\n", + ) + .unwrap(); + fs::write( + dir.join("main.by"), + "from pkg.calc import add\nfrom pkg.plain import helper\n\nprint(add(1, helper()))\n", + ) + .unwrap(); +} + +/// The `DA:` lines of `file`'s record in an LCOV `report`, or `None` when it has +/// no record. +fn lcov_lines<'a>(report: &'a str, file: &Path) -> Option> { + let header = format!("SF:{}", file.display()); + let record = report + .split("end_of_record") + .find(|record| record.lines().any(|line| line == header))?; + Some( + record + .lines() + .filter(|line| line.starts_with("DA:")) + .collect(), + ) +} + +/// coverage.py measures the generated python, and what reaches the report is +/// the `.by` it was written from — the file and lines an editor shows — and the +/// project's own `.py` rather than the staged copy of it that ran, which is gone +/// once the run ends +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped leg must say why it skipped, or it reads as a pass" +)] +fn run_coverage_reports_by_lines_and_project_files() { + let (python, importable) = coverage_interpreter(); + let dir = tempfile::tempdir().expect("tempdir"); + project_with_partly_covered_modules(dir.path()); + let project = dir.path().canonicalize().unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args([ + "run", + "--python", + &python, + "--coverage", + "out/report.lcov", + "main", + ]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr); + + if !importable { + eprintln!("only checking the refusal: {python} cannot import coverage"); + assert!( + !output.status.success(), + "ran without coverage.py:\n{stderr}" + ); + assert!( + stderr.contains("cannot import") && stderr.contains("pip install coverage"), + "the refusal says neither what is missing nor how to get it:\n{stderr}" + ); + assert!(!dir.path().join("out/report.lcov").exists()); + return; + } + assert!( + output.status.success(), + "by run --coverage failed:\n{stderr}" + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "4"); + let report = fs::read_to_string(dir.path().join("out/report.lcov")).expect("a report"); + + assert_eq!( + lcov_lines(&report, &project.join("pkg/calc.by")), + Some(vec!["DA:1,1", "DA:2,1", "DA:3,0", "DA:4,1"]), + "calc.by is reported on its own lines:\n{report}" + ); + assert_eq!( + lcov_lines(&report, &project.join("main.by")), + Some(vec!["DA:1,1", "DA:2,1", "DA:4,1"]), + "{report}" + ); + assert_eq!( + lcov_lines(&report, &project.join("pkg/plain.py")), + Some(vec!["DA:1,1", "DA:2,1", "DA:5,1", "DA:6,0"]), + "the staged copy is reported as the project file:\n{report}" + ); + let named: Vec<&str> = report + .lines() + .filter_map(|line| line.strip_prefix("SF:")) + .collect(); + assert!( + named + .iter() + .all(|file| Path::new(file).starts_with(&project)), + "a file outside the project — the runner, a staged copy — was reported: {named:?}" + ); + assert!( + !dir.path().join(".coverage").exists(), + "coverage.py left a data file of its own beside the project" + ); +} + +/// The report of a `by run --coverage main` of `dir` with `python`, which can +/// import coverage.py, after checking the run succeeded and printed `stdout`. +fn coverage_report(dir: &Path, python: &str, stdout: &str) -> String { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args([ + "run", + "--python", + python, + "--coverage", + "report.lcov", + "main", + ]) + .current_dir(dir) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "by run --coverage failed:\n{stderr}" + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), stdout, "{stderr}"); + fs::read_to_string(dir.join("report.lcov")) + .unwrap_or_else(|error| panic!("no report ({error}):\n{stderr}")) +} + +/// the project's coverage configuration names project files, and a hand-written +/// `.py` runs as its project file, so `source` takes it in and `omit` leaves it out +/// as it does a `.by` +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_coverage_applies_the_configuration_to_project_files() { + let (python, importable) = coverage_interpreter(); + if !importable { + eprintln!("skipping: {python} cannot import coverage"); + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + project_with_partly_covered_modules(dir.path()); + let project = dir.path().canonicalize().unwrap(); + + fs::write( + dir.path().join("pyproject.toml"), + "[tool.coverage.run]\nsource = [\"pkg\"]\n", + ) + .unwrap(); + let report = coverage_report(dir.path(), &python, "4\n"); + assert_eq!( + lcov_lines(&report, &project.join("pkg/plain.py")), + Some(vec!["DA:1,1", "DA:2,1", "DA:5,1", "DA:6,0"]), + "{report}" + ); + assert_eq!( + lcov_lines(&report, &project.join("main.by")), + None, + "main.by is outside `source`:\n{report}" + ); + + fs::write( + dir.path().join("pyproject.toml"), + "[tool.coverage.run]\nomit = [\"pkg/plain.py\"]\n", + ) + .unwrap(); + let report = coverage_report(dir.path(), &python, "4\n"); + assert_eq!( + lcov_lines(&report, &project.join("pkg/plain.py")), + None, + "pkg/plain.py is omitted:\n{report}" + ); + assert!( + lcov_lines(&report, &project.join("pkg/calc.by")).is_some(), + "{report}" + ); +} + +/// coverage.py reports a `.py` under `source` that never ran, and a `.by` that +/// never ran is reported the same way, on every core coverage.py measures with +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_coverage_reports_a_by_module_that_never_ran() { + let (python, importable) = coverage_interpreter(); + if !importable { + eprintln!("skipping: {python} cannot import coverage"); + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + project_with_partly_covered_modules(dir.path()); + fs::write( + dir.path().join("pkg/unused.by"), + "def unused() -> int:\n return 5\n", + ) + .unwrap(); + fs::write( + dir.path().join("pyproject.toml"), + "[tool.coverage.run]\nsource = [\"pkg\"]\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + + for core in ["ctrace", "sysmon"] { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .env("COVERAGE_CORE", core) + .args([ + "run", + "--python", + &python, + "--coverage", + "report.lcov", + "main", + ]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{core}: by run failed:\n{stderr}"); + assert!( + !stderr.contains("CoverageWarning"), + "{core}: coverage.py warned:\n{stderr}" + ); + let report = fs::read_to_string(dir.path().join("report.lcov")).expect("a report"); + assert_eq!( + lcov_lines(&report, &project.join("pkg/unused.by")), + Some(vec!["DA:1,0", "DA:2,0"]), + "{core}: {report}" + ); + } + + fs::write( + dir.path().join("pyproject.toml"), + "[tool.coverage.run]\nsource = [\"pkg\"]\nomit = [\"pkg/unused.by\"]\n", + ) + .unwrap(); + let report = coverage_report(dir.path(), &python, "4\n"); + assert_eq!( + lcov_lines(&report, &project.join("pkg/unused.by")), + None, + "pkg/unused.by is omitted:\n{report}" + ); +} + +/// `--in-build` runs the program in the staged tree, and the configuration still +/// names the project's files: it is read where the project is +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_coverage_in_build_applies_the_configuration_to_project_files() { + let (python, importable) = coverage_interpreter(); + if !importable { + eprintln!("skipping: {python} cannot import coverage"); + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + project_with_partly_covered_modules(dir.path()); + fs::write( + dir.path().join("pyproject.toml"), + "[tool.coverage.run]\nsource = [\"pkg\"]\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--python", &python, "--in-build", "--coverage"]) + .arg(project.join("report.lcov")) + .arg("main") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "by run --coverage failed:\n{stderr}" + ); + let report = fs::read_to_string(project.join("report.lcov")).expect("a report"); + assert_eq!( + lcov_lines(&report, &project.join("pkg/calc.by")), + Some(vec!["DA:1,1", "DA:2,1", "DA:3,0", "DA:4,1"]), + "{report}" + ); + assert_eq!( + lcov_lines(&report, &project.join("pkg/plain.py")), + Some(vec!["DA:1,1", "DA:2,1", "DA:5,1", "DA:6,0"]), + "{report}" + ); +} + +/// a hand-written `.py` runs as its project file, so a traceback through it +/// names that file rather than a copy in a directory gone once the run ends +#[test] +fn run_names_a_hand_written_module_s_project_file_in_a_traceback() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("helper.py"), + "def fail() -> None:\n raise ValueError(\"no\")\n\nwhere = __file__\n", + ) + .unwrap(); + fs::write( + dir.path().join("main.by"), + "import helper\n\nprint(helper.where)\nhelper.fail()\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + let helper = project.join("helper.py"); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + Path::new(stdout.trim()).canonicalize().ok(), + Some(helper.clone()), + "`__file__` names a staged copy:\n{stdout}\n{stderr}" + ); + let raised_in = stderr + .lines() + .filter_map(|line| line.trim().strip_prefix("File \"")) + .filter_map(|line| line.split('"').next()) + .next_back() + .expect("a traceback"); + assert_eq!( + Path::new(raised_in).canonicalize().ok(), + Some(helper), + "the traceback names a staged copy:\n{stderr}" + ); +} + +/// with branch coverage on, coverage.py records the arcs between lines rather +/// than the lines, and the report still names the `.by` lines that ran +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_coverage_reports_lines_when_branches_are_measured() { + let (python, importable) = coverage_interpreter(); + if !importable { + eprintln!("skipping: {python} cannot import coverage"); + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + project_with_partly_covered_modules(dir.path()); + fs::write( + dir.path().join("pyproject.toml"), + "[tool.coverage.run]\nbranch = true\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + + let report = coverage_report(dir.path(), &python, "4\n"); + assert_eq!( + lcov_lines(&report, &project.join("pkg/calc.by")), + Some(vec!["DA:1,1", "DA:2,1", "DA:3,0", "DA:4,1"]), + "{report}" + ); +} + +/// a program has not ended until its other threads have and its `atexit` +/// handlers have run, and what they run is measured too +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_coverage_measures_threads_and_exit_handlers() { + let (python, importable) = coverage_interpreter(); + if !importable { + eprintln!("skipping: {python} cannot import coverage"); + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "import atexit\nimport threading\nimport time\n\ndef late():\n time.sleep(0.2)\n print(\"late\")\n\ndef bye():\n print(\"bye\")\n\natexit.register(bye)\nthreading.Thread(target=late).start()\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + + let report = coverage_report(dir.path(), &python, "late\nbye\n"); + let lines = lcov_lines(&report, &project.join("main.by")).expect("main.by is reported"); + for ran in ["DA:6,1", "DA:7,1", "DA:10,1"] { + assert!(lines.contains(&ran), "{ran} is not covered:\n{report}"); + } +} + +/// a test runner compiles a test module itself, rewriting its assertions, so +/// its code objects are the staged `.py`'s rather than the `.by`'s — and the +/// report still names the `.by` and its lines +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_coverage_maps_a_module_a_test_runner_compiled() { + let (python, importable) = coverage_interpreter(); + let pytest = Command::new(&python) + .args(["-c", "import pytest"]) + .status() + .is_ok_and(|status| status.success()); + if !importable || !pytest { + eprintln!("skipping: {python} cannot import both coverage and pytest"); + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + project_with_partly_covered_modules(dir.path()); + fs::create_dir_all(dir.path().join("tests")).unwrap(); + fs::write( + dir.path().join("tests/test_calc.by"), + "from pkg.calc import add\n\n\ndef test_add():\n assert add(1, 2) == 3\n\n\ndef test_big():\n assert add(1000, 2) == -1\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + let report = project.join("report.lcov"); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .arg("run") + .args(["--python", &python, "--in-build", "--coverage"]) + .arg(&report) + .args(["pytest", "-q", "-p", "no:cacheprovider"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + assert!( + output.status.success(), + "by run --coverage pytest failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let report = fs::read_to_string(&report).expect("a report"); + assert_eq!( + lcov_lines(&report, &project.join("tests/test_calc.by")), + Some(vec!["DA:1,1", "DA:4,1", "DA:5,1", "DA:8,1", "DA:9,1"]), + "{report}" + ); + assert_eq!( + lcov_lines(&report, &project.join("pkg/calc.by")), + Some(vec!["DA:1,1", "DA:2,1", "DA:3,1", "DA:4,1"]), + "{report}" + ); +} + /// `multiprocessing`'s spawn start method — the default on macos and windows — /// reads `__main__.__spec__.name` to tell the child what to re-import, and falls /// back to running `__file__` as a *path* when there is none. `__file__` is a diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 60db177a3f..57f63a6aec 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -94,6 +94,34 @@ layout it is a flag rather than something inferred because only the caller knows which of the two directories they mean, and the program's name does not say +`--coverage PATH` measures which lines the program runs, with coverage.py, and +writes them to `PATH` as an LCOV report — the format an editor's gutter, +`genhtml` and CI coverage services all read. coverage.py measures the python the +program actually ran, so the report is mapped back before it is written: a +transpiled module is reported against its `.by` file, a `.by` line counting as +covered when any statement transpiled from it ran, and a `.py` the build carries +over is reported against the file in the project rather than the staged copy +that ran. that holds for a module a test runner compiled itself, rewriting its +assertions, as much as for one python imported + +```console +$ by run --coverage coverage.lcov --in-build pytest +``` + +coverage.py has to be installed in the interpreter the program runs on: without +it the program is not started, and the error names that interpreter. the project's own +coverage.py configuration (`[tool.coverage]`, `.coveragerc`) applies, except that +no `.coverage` data file is written: the report is the result. its `source`, +`include` and `omit` name the project's files as they are in the project, with +`--in-build` too, since every module runs as the file it was written as. the one +exception is a module a test runner compiled itself, which runs as the staged +copy it compiled: a `source` or an `omit` naming it does not reach it. a `.by` +module under `source` that never ran is reported as not run, as coverage.py +reports a `.py` that never ran. a program that +leaves without returning to `by run` — `os._exit` — leaves no report, and +`--compiled` cannot be combined with it, since a native module runs no python +lines to measure + the emitted code targets that interpreter's version by default. an explicit `--min-version` wins, but must not exceed the interpreter — `by run` refuses rather than emit code the interpreter cannot parse. an interpreter *older* than From 7b88c6784e39876aec2d12df08f31eb4c76a21fd Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:52:01 +1000 Subject: [PATCH 04/15] answer by/evaluatableExpression, the expression a debugger evaluates for a position: the whole reference whose last name the position is on, and the call around a callee when calls are allowed --- crates/ty_ide/Cargo.toml | 2 +- crates/ty_ide/src/evaluatable_expression.rs | 271 ++++++++++++++++++ crates/ty_ide/src/lib.rs | 2 + crates/ty_server/src/server/api.rs | 7 + crates/ty_server/src/server/api/requests.rs | 2 + .../api/requests/evaluatable_expression.rs | 106 +++++++ .../tests/e2e/evaluatable_expression.rs | 85 ++++++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 30 ++ 9 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 crates/ty_ide/src/evaluatable_expression.rs create mode 100644 crates/ty_server/src/server/api/requests/evaluatable_expression.rs create mode 100644 crates/ty_server/tests/e2e/evaluatable_expression.rs diff --git a/crates/ty_ide/Cargo.toml b/crates/ty_ide/Cargo.toml index d09a291452..bbef7c325d 100644 --- a/crates/ty_ide/Cargo.toml +++ b/crates/ty_ide/Cargo.toml @@ -23,6 +23,7 @@ ruff_python_ast = { workspace = true } ruff_python_codegen = { workspace = true } ruff_python_importer = { workspace = true } ruff_python_literal = { workspace = true } +ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } ruff_python_trivia = { workspace = true } ruff_source_file = { workspace = true } @@ -51,7 +52,6 @@ tracing = { workspace = true } unicode-width = { workspace = true } [dev-dependencies] -ruff_python_parser = { workspace = true } ruff_ranged_value = { workspace = true } ty_project = { workspace = true, features = ["testing"] } ty_static = { workspace = true } diff --git a/crates/ty_ide/src/evaluatable_expression.rs b/crates/ty_ide/src/evaluatable_expression.rs new file mode 100644 index 0000000000..b4fa497982 --- /dev/null +++ b/crates/ty_ide/src/evaluatable_expression.rs @@ -0,0 +1,271 @@ +//! the expression a debugger evaluates for a position: what a value tooltip +//! shows on hover, and what *Evaluate Expression* and *Quick Evaluate* start +//! from when nothing is selected +//! +//! it is the reference the name at the position is the end of. on `d` in +//! `a.b[c].d` that is all of `a.b[c].d`, on `b` it is `a.b`, and on `c` it is +//! `c`, the name inside the subscript. a client that guesses from the text +//! cannot tell `d` there from a name of its own, and a guess that stops at the +//! `]` answers nothing at all +//! +//! a debugger evaluates what this answers in the stopped program, so whether +//! the expression may run code the program would not have run is part of the +//! question. reading an attribute or an item is what a value tooltip does in +//! every python debugger and is always answered. a call is answered only when +//! the client says running one is fine, which is the difference between +//! hovering a name and asking for it to be evaluated: on `f` in `f(x)` the +//! answer is `f` when calls are not allowed and `f(x)` when they are, and a +//! reference whose own value is a call's result, like `make().d`, is not +//! answered at all when they are not +//! +//! the stopped program is python, so in a basedpython file only an expression +//! that python reads the same way is answered: not `user?.name` + +use ruff_db::PythonFile; +use ruff_db::parsed::parsed_module; +use ruff_db::source::source_text; +use ruff_python_ast::comparable::ComparableExpr; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::helpers::any_over_expr; +use ruff_python_ast::token::TokenKind; +use ruff_python_ast::{AnyNodeRef, Expr, ExprContext, Mod}; +use ruff_python_parser::{Mode, ParseOptions, parse}; +use ruff_text_size::{Ranged, TextRange, TextSize}; + +use crate::Db; + +/// the range of the expression a debugger evaluates for `offset`, or `None` +/// when the position is not on a name an expression ends with +/// +/// `calls` says whether the expression may call anything; see the module +/// documentation +pub fn evaluatable_expression( + db: &dyn Db, + file: PythonFile<'_>, + offset: TextSize, + calls: bool, +) -> Option { + let parsed = parsed_module(db, file).load(db); + // the name is found from the tokens rather than from the tree, so that a + // position between `.` and a name, or right after a name, is on that name + let token = parsed + .tokens() + .at_offset(offset) + .find(|token| token.kind() == TokenKind::Name || token.kind().is_soft_keyword())?; + let covering = covering_node(parsed.syntax().into(), token.range()); + + let mut ancestors = covering.ancestors(); + let mut node = ancestors.next()?; + if node.is_identifier() { + let parent = ancestors.next()?; + match parent { + // the attribute a reference ends with is the whole reference + AnyNodeRef::ExprAttribute(_) => node = parent, + // a parameter is bound in the frame the debugger is stopped in + AnyNodeRef::Parameter(_) => return Some(token.range()), + // the name a `def`, a class, an import or a keyword argument + // declares is not an expression, and evaluating the same text + // could mean something else in the stopped frame + _ => return None, + } + } + + let expression = match node { + AnyNodeRef::ExprName(name) => { + // a name the lowering made up spans the whole construct it stands + // for, and has no text of its own to evaluate + if name.ctx == ExprContext::Invalid { + return None; + } + name.range() + } + AnyNodeRef::ExprAttribute(attribute) => { + if !calls && calls_anything(&attribute.value) { + return None; + } + attribute.range() + } + _ => return None, + }; + + let answer = if calls + && let Some(AnyNodeRef::ExprCall(call)) = ancestors.next() + && call.func.range() == expression + { + call.range() + } else { + expression + }; + + let source = source_text(db, file.file(db)); + (!file.file(db).source_type(db).is_basedpython() || reads_the_same_in_python(&source[answer])) + .then_some(answer) +} + +/// whether python reads `text`, an expression in a basedpython file, as the same expression: +/// the program a debugger is stopped in is python, whatever its source was written in +fn reads_the_same_in_python(text: &str) -> bool { + let read = |basedpython: bool| { + let options = ParseOptions::from(Mode::Expression).with_basedpython(basedpython); + match parse(text, options).ok()?.into_syntax() { + Mod::Expression(expression) => Some(expression.body), + Mod::Module(_) => None, + } + }; + let (Some(python), Some(basedpython)) = (read(false), read(true)) else { + return false; + }; + ComparableExpr::from(&python) == ComparableExpr::from(&basedpython) +} + +/// whether evaluating `expression` would call something, or otherwise run code +/// the program decides on, rather than read a value +fn calls_anything(expression: &Expr) -> bool { + any_over_expr(expression, |expression| { + matches!( + expression, + Expr::Call(_) + | Expr::Await(_) + | Expr::Yield(_) + | Expr::YieldFrom(_) + | Expr::Named(_) + | Expr::Lambda(_) + | Expr::ListComp(_) + | Expr::SetComp(_) + | Expr::DictComp(_) + | Expr::Generator(_) + ) + }) +} + +#[cfg(test)] +mod tests { + use crate::tests::CursorTest; + + use super::evaluatable_expression; + + /// the text of the answer for the `` in `source`, a `.by` file + fn evaluatable(source: &str, calls: bool) -> Option { + let test = CursorTest::builder().source("main.by", source).build(); + let file = test.program_file(test.cursor.file).python_file(&test.db); + evaluatable_expression(&test.db, file, test.cursor.offset, calls) + .map(|range| test.cursor.source[range].to_string()) + } + + fn both(source: &str) -> (Option, Option) { + (evaluatable(source, false), evaluatable(source, true)) + } + + const CHAIN: &str = "a.b[c].d + compute(len(c))"; + + fn at(marker: &str) -> String { + let index = CHAIN.find(marker).expect("the marker is in the line"); + format!("{}{}\n", &CHAIN[..index], &CHAIN[index..]) + } + + #[test] + fn the_last_name_of_a_reference_is_the_whole_reference() { + assert_eq!(evaluatable(&at("d +"), false).as_deref(), Some("a.b[c].d")); + } + + #[test] + fn a_name_inside_a_reference_is_the_reference_up_to_it() { + assert_eq!(evaluatable(&at("a."), false).as_deref(), Some("a")); + assert_eq!(evaluatable(&at("b["), false).as_deref(), Some("a.b")); + } + + #[test] + fn a_name_in_a_subscript_is_itself() { + assert_eq!(evaluatable(&at("c]"), false).as_deref(), Some("c")); + } + + #[test] + fn a_callee_is_the_call_only_when_calls_are_allowed() { + assert_eq!( + both(&at("compute")), + ( + Some("compute".to_string()), + Some("compute(len(c))".to_string()) + ) + ); + assert_eq!( + both(&at("len")), + (Some("len".to_string()), Some("len(c)".to_string())) + ); + } + + #[test] + fn a_method_is_the_call_only_when_calls_are_allowed() { + assert_eq!( + both("items.append(1)\n"), + ( + Some("items.append".to_string()), + Some("items.append(1)".to_string()) + ) + ); + } + + #[test] + fn a_reference_through_a_call_is_answered_only_when_calls_are_allowed() { + assert_eq!( + both("make().d\n"), + (None, Some("make().d".to_string())) + ); + } + + #[test] + fn a_position_just_after_a_name_or_just_before_it_is_on_it() { + assert_eq!( + evaluatable("total = 1\n", false).as_deref(), + Some("total") + ); + assert_eq!(evaluatable("a.b\n", false).as_deref(), Some("a.b")); + } + + #[test] + fn a_parameter_is_its_name() { + assert_eq!( + evaluatable("def f(limit: int):\n pass\n", false).as_deref(), + Some("limit") + ); + } + + #[test] + fn declared_names_are_not_expressions() { + assert_eq!( + evaluatable("def compute():\n pass\n", true), + None + ); + assert_eq!(evaluatable("class Point:\n pass\n", true), None); + assert_eq!(evaluatable("import os\n", true), None); + assert_eq!(evaluatable("f(name=1)\n", true), None); + } + + #[test] + fn literals_keywords_operators_and_comments_are_nothing() { + assert_eq!(evaluatable("x = \"text\"\n", true), None); + assert_eq!(evaluatable("x = None\n", true), None); + assert_eq!(evaluatable("x = 1 + 2\n", true), None); + assert_eq!(evaluatable("# a note\n", true), None); + } + + /// the running program is python, so an expression it would read differently, or not at + /// all, is not answered + #[test] + fn basedpython_only_syntax_is_nothing() { + assert_eq!(evaluatable("user?.name\n", false), None); + assert_eq!(evaluatable("user?.name.first\n", false), None); + assert_eq!( + evaluatable("user?.name\n", false).as_deref(), + Some("user") + ); + } + + #[test] + fn a_name_in_an_f_string_is_itself() { + assert_eq!( + evaluatable("f\"{count} items\"\n", false).as_deref(), + Some("count") + ); + } +} diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 9964c2ed86..91160ec1a8 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -14,6 +14,7 @@ mod django_template; mod doc_highlights; mod docstring; mod document_symbols; +mod evaluatable_expression; mod find_references; mod folding_range; mod goto; @@ -65,6 +66,7 @@ pub use django_template::{ }; pub use doc_highlights::document_highlights; pub use document_symbols::document_symbols; +pub use evaluatable_expression::evaluatable_expression; pub use find_references::find_references; pub use folding_range::{FoldingRange, FoldingRangeKind, folding_ranges}; pub use goto::{goto_declaration, goto_definition, goto_type_definition}; diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 239fcea404..7e01065acd 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -211,6 +211,13 @@ pub(super) fn request(req: server::Request) -> Task { requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), + // asked while a person holds the mouse over a value, like hover + requests::EvaluatableExpressionRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } requests::DocumentSuperMembersRequestHandler::METHOD => background_document_request_task::< requests::DocumentSuperMembersRequestHandler, >( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 4c9d730f2a..cc9be78875 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -28,6 +28,7 @@ mod doc_highlights; mod document_super_members; mod document_symbols; mod entry_point; +mod evaluatable_expression; mod execute_command; mod explain_rule; mod explain_transpilation; @@ -77,6 +78,7 @@ pub(super) use doc_highlights::DocumentHighlightRequestHandler; pub(super) use document_super_members::DocumentSuperMembersRequestHandler; pub(super) use document_symbols::DocumentSymbolRequestHandler; pub(super) use entry_point::EntryPointHandler; +pub(super) use evaluatable_expression::EvaluatableExpressionRequestHandler; pub(super) use execute_command::ExecuteCommand; pub(super) use explain_rule::ExplainRuleHandler; pub(super) use explain_transpilation::ExplainTranspilationHandler; diff --git a/crates/ty_server/src/server/api/requests/evaluatable_expression.rs b/crates/ty_server/src/server/api/requests/evaluatable_expression.rs new file mode 100644 index 0000000000..90c6d323c0 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/evaluatable_expression.rs @@ -0,0 +1,106 @@ +//! `by/evaluatableExpression`: the expression a debugger evaluates for a position +//! +//! what a debugger's value tooltip, *Evaluate Expression* and *Quick Evaluate* start from when +//! nothing is selected: on `d` in `a.b[c].d`, all of `a.b[c].d`. LSP has no request for it — the +//! closest is `textDocument/inlineValue`, which answers for a range of lines at a stop rather than +//! for a position — and a client guessing from the text cannot tell where a reference starts. see +//! [`ty_ide::evaluatable_expression`] for what is answered + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, +}; +use ty_ide::evaluatable_expression; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{PositionExt, ToRangeExt}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum EvaluatableExpressionRequest {} + +impl Request for EvaluatableExpressionRequest { + type Params = EvaluatableExpressionParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/evaluatableExpression"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the position to answer for, and whether the expression may call anything +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct EvaluatableExpressionParams { + text_document: TextDocumentIdentifier, + position: Position, + /// whether the expression may call something: a debugger asking because the user asked for + /// an evaluation says yes, and one asking for a value tooltip says no. on `f` in `f(x)` that is + /// the difference between `f(x)` and `f` + calls: bool, +} + +/// the expression, as where it is in the document +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct EvaluatableExpression { + range: Range, +} + +pub(crate) struct EvaluatableExpressionRequestHandler; + +impl RequestHandler for EvaluatableExpressionRequestHandler { + type RequestType = EvaluatableExpressionRequest; +} + +impl BackgroundDocumentRequestHandler for EvaluatableExpressionRequestHandler { + fn document_uri(params: &EvaluatableExpressionParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, for a template, and when the position is not on a + /// name an expression ends with + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: EvaluatableExpressionParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.is_django_template() { + return Ok(None); + } + + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + + let python_file = db.program_file(file).python_file(db); + let Some(range) = evaluatable_expression(db, python_file, offset, params.calls) else { + return Ok(None); + }; + let Some(range) = range + .to_lsp_range(db, file, snapshot.encoding()) + .map(|range| range.local_range()) + else { + return Ok(None); + }; + Ok(Some(EvaluatableExpression { range })) + } +} + +impl RetriableRequestHandler for EvaluatableExpressionRequestHandler {} diff --git a/crates/ty_server/tests/e2e/evaluatable_expression.rs b/crates/ty_server/tests/e2e/evaluatable_expression.rs new file mode 100644 index 0000000000..8f156c1758 --- /dev/null +++ b/crates/ty_server/tests/e2e/evaluatable_expression.rs @@ -0,0 +1,85 @@ +//! `by/evaluatableExpression` over the wire +//! +//! which expression a position is on is `ty_ide`'s and its unit tests cover it. what these cover is +//! the contract a client reads: a position and whether the expression may call anything in, and out +//! the range of the expression in the document, or `null` where there is nothing to evaluate + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder, text_hash}; + +/// the request as a client sends it, in json throughout +enum EvaluatableExpression {} + +impl Request for EvaluatableExpression { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/evaluatableExpression"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const MAIN: &str = "\ +def main(a: object, c: str) -> None: + total = a.b[c].d + compute(len(c)) +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + Ok(server) +} + +fn ask(server: &mut TestServer, character: u32, calls: bool) -> Option { + let uri = server.file_uri(SystemPath::new("src/main.by")); + server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 1, "character": character }, + "calls": calls, + "textHash": text_hash(MAIN), + })) +} + +fn range(start: u32, end: u32) -> serde_json::Value { + serde_json::json!({ + "range": { + "start": { "line": 1, "character": start }, + "end": { "line": 1, "character": end }, + } + }) +} + +#[test] +fn the_last_name_of_a_reference_answers_the_whole_reference() -> Result<()> { + let mut server = server()?; + + // on `d`, the end of `a.b[c].d` + assert_eq!(ask(&mut server, 19, false), Some(range(12, 20))); + Ok(()) +} + +#[test] +fn a_callee_is_its_call_only_when_calls_are_allowed() -> Result<()> { + let mut server = server()?; + + // on `compute` + assert_eq!(ask(&mut server, 24, false), Some(range(23, 30))); + assert_eq!(ask(&mut server, 24, true), Some(range(23, 38))); + Ok(()) +} + +#[test] +fn an_operator_answers_null() -> Result<()> { + let mut server = server()?; + + // on `+` + assert_eq!(ask(&mut server, 21, true), None); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index f8deb3949e..72430cf1a4 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -38,6 +38,7 @@ mod completions; mod configuration; mod data_flow; mod django_templates; +mod evaluatable_expression; mod file_watching; mod folding_range; mod goto_definition; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index ad6b56d7d2..ab41203b07 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -591,6 +591,36 @@ between: bind — a global it only reads, an attribute it never assigns — is a value nothing in that scope can vouch for +## the expression under the cursor + +a debugger's value tooltip, and *Evaluate Expression* or *Quick Evaluate* with +nothing selected, start from the expression the cursor is on. which expression +that is depends on the code rather than the characters: on `d` in +`a.b[c].d + compute(len(c))` it is `a.b[c].d`, on `b` it is `a.b`, and on `c` +it is `c`. the editor asks with `by/evaluatableExpression`: + +```json +{ + "textDocument": { "uri": "file:///src/main.by" }, + "position": { "line": 1, "character": 19 }, + "calls": false +} +``` + +and the answer is the expression's `range`, or `null` where the position is not +on a name an expression ends with: a literal, an operator, a keyword, a comment, +or a name that a `def`, a class, an import or a keyword argument declares. a +parameter's name is the parameter. the program the debugger is stopped in is +python, so in a `.by` file an expression python would not read the same way, +like `user?.name`, is `null` too + +`calls` says whether the expression may call anything, which is the difference +between hovering a name and asking for an evaluation. on `compute` the answer is +`compute` when it may not and `compute(len(c))` when it may, and a reference +whose value comes out of a call, like `make().d`, is answered only when it may. +reading an attribute or an item is answered either way, as every python +debugger's value tooltip does + ## renaming a module renaming `util.by` to `helpers.by` renames the module `alpha.util`, and every From 87e99e4e967f219ff75891b424db35c31c6935e8 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:36:14 +1000 Subject: [PATCH 05/15] add by/documentUsageCounts: each declaration of a document with how many places use it and how many classes subclass it or members override it, counted as textDocument/references and textDocument/implementation answer at its name, for an editor drawing usage counts on every pass --- crates/ty_ide/src/lib.rs | 4 + crates/ty_ide/src/usage_counts.rs | 355 ++++++++++++++++++ crates/ty_server/src/server/api.rs | 5 + crates/ty_server/src/server/api/requests.rs | 2 + .../api/requests/document_usage_counts.rs | 174 +++++++++ crates/ty_server/tests/e2e/main.rs | 1 + crates/ty_server/tests/e2e/usage_counts.rs | 232 ++++++++++++ docs/basedpython/features/editor.md | 31 ++ 8 files changed, 804 insertions(+) create mode 100644 crates/ty_ide/src/usage_counts.rs create mode 100644 crates/ty_server/src/server/api/requests/document_usage_counts.rs create mode 100644 crates/ty_server/tests/e2e/usage_counts.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 91160ec1a8..9e1800437c 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -40,6 +40,7 @@ mod super_members; mod symbols; mod syntax_outline; mod type_hierarchy; +mod usage_counts; mod workspace_symbols; pub use add_dependency::{AddDependency, DependencyTarget}; @@ -99,6 +100,9 @@ pub use syntax_outline::{ pub use type_hierarchy::{ TypeHierarchyItem, prepare_type_hierarchy, type_hierarchy_subtypes, type_hierarchy_supertypes, }; +pub use usage_counts::{ + DeclarationKind, DeclarationUsages, UsageCountKinds, document_usage_counts, +}; pub use workspace_symbols::{WorkspaceSymbolInfo, workspace_symbols}; use ruff_db::{ diff --git a/crates/ty_ide/src/usage_counts.rs b/crates/ty_ide/src/usage_counts.rs new file mode 100644 index 0000000000..567a0aa52d --- /dev/null +++ b/crates/ty_ide/src/usage_counts.rs @@ -0,0 +1,355 @@ +//! how many places use each declaration of a document, and how many classes +//! subclass it or members override it: the counts an editor draws above each +//! declaration ("3 usages", "2 overrides"), for the whole document at once +//! +//! a count is what asking of the declaration's name answers, found by the same +//! code: [`crate::find_references()`] without the declaration for the usages, +//! and [`crate::goto_implementation()`] less the declaration itself for the +//! subclasses and overrides. so the count over a declaration is the number of +//! places its click goes to +//! +//! the declarations counted are the document's outline ([`crate::document_symbols()`]) +//! at the depth a reader scans a module at: its classes and functions, and the +//! methods and properties of those classes. a dunder is left out, being called +//! by the language rather than by name, and so is an `extension` block, which +//! names a class it does not declare. a nested function or a class inside a +//! class is not counted + +use rayon::prelude::*; +use ruff_db::parsed::parsed_module; +use ruff_python_ast::AnyNodeRef; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::helpers::is_dunder; +use ruff_text_size::TextRange; +use ty_project::parallel::ParallelIteratorExt; +use ty_python_core::ProgramFile; +use ty_python_semantic::types::ide_support::{class_member_definitions, overridden_members}; + +use crate::symbols::symbols_for_file; +use crate::{Db, SymbolKind, find_references, goto_implementation}; + +/// which counts to work out: each is a search of the workspace per declaration +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct UsageCountKinds { + /// how many places use each declaration + pub references: bool, + /// how many classes subclass each class, and how many members override each + /// method + pub implementations: bool, +} + +/// what a declaration is, for the counts it has: a function is neither +/// subclassed nor overridden +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeclarationKind { + Class, + Function, + /// a method or a property of a class + Method, +} + +/// a declaration of the document, and its counts +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeclarationUsages { + pub name: String, + pub kind: DeclarationKind, + /// the whole declaration, its decorators included + pub full_range: TextRange, + /// its name where it is declared + pub name_range: TextRange, + /// how many places use it, the declaration itself not among them. `None` + /// when not asked for + pub references: Option, + /// how many classes subclass a class, directly or not, and how many members + /// of those override a method. `None` when not asked for, and for a function + pub implementations: Option, + /// whether a method is abstract, an `@abstractmethod` or a protocol member + /// with no body, so that what overrides it implements it + pub is_abstract: bool, +} + +/// the declarations of `file` with the counts `kinds` asks for, in source order +pub fn document_usage_counts( + db: &dyn Db, + file: ProgramFile<'_>, + kinds: UsageCountKinds, +) -> Vec { + let declarations = counted_declarations(db, file); + if !kinds.references && !kinds.implementations { + return declarations; + } + let source_file = file.file(db); + let program = file.program(db); + + let mut declarations: Vec<_> = declarations + .into_par_iter() + .map_with_db(db, |db, mut declaration| { + let file = ProgramFile::new(db, source_file, program); + let offset = declaration.name_range.start(); + if kinds.references { + declaration.references = Some( + find_references(db, file, offset, false) + .map_or(0, |references| references.len()), + ); + } + if kinds.implementations && declaration.kind != DeclarationKind::Function { + declaration.implementations = + Some(goto_implementation(db, file, offset).map_or(0, |found| { + found + .value + .iter() + .filter(|target| { + // the declaration is its own first implementation + target.file() != source_file + || target.focus_range() != declaration.name_range + }) + .count() + })); + } + declaration + }) + .collect::>(); + + // abstract or not only changes how a method's overrides are named, so it is + // only worked out for one that has some + if declarations.iter().any(|declaration| { + declaration.kind == DeclarationKind::Method + && declaration.implementations.is_some_and(|count| count > 0) + }) { + let members = class_member_definitions(db, file); + for declaration in &mut declarations { + if declaration.kind == DeclarationKind::Method + && declaration.implementations.is_some_and(|count| count > 0) + { + declaration.is_abstract = members + .iter() + .filter(|member| member.name_range == declaration.name_range) + .any(|member| overridden_members(db, member).is_abstract); + } + } + } + declarations +} + +/// the declarations of `file` that are counted, with no counts yet +fn counted_declarations(db: &dyn Db, file: ProgramFile<'_>) -> Vec { + let module = parsed_module(db, file.python_file(db)).load(db); + let symbols = symbols_for_file(db, file).to_hierarchical(); + let mut declarations = Vec::new(); + + let mut add = |name: &str, kind, full_range, name_range| { + if !is_dunder(name) { + declarations.push(DeclarationUsages { + name: name.to_string(), + kind, + full_range, + name_range, + references: None, + implementations: None, + is_abstract: false, + }); + } + }; + + for (id, symbol) in symbols.iter() { + match symbol.kind { + SymbolKind::Function => { + add( + &symbol.name, + DeclarationKind::Function, + symbol.full_range, + symbol.name_range, + ); + } + SymbolKind::Class => { + let is_extension = matches!( + covering_node(module.syntax().into(), symbol.full_range).node(), + AnyNodeRef::StmtClassDef(class) if class.is_extension() + ); + if !is_extension { + add( + &symbol.name, + DeclarationKind::Class, + symbol.full_range, + symbol.name_range, + ); + } + for (_, member) in symbols.children(id) { + if matches!( + member.kind, + SymbolKind::Method | SymbolKind::Constructor | SymbolKind::Property + ) { + add( + &member.name, + DeclarationKind::Method, + member.full_range, + member.name_range, + ); + } + } + } + _ => {} + } + } + declarations.sort_by_key(|declaration| declaration.full_range.start()); + declarations +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + + use crate::tests::{CursorTest, cursor_test}; + use insta::assert_snapshot; + use ruff_db::source::source_text; + use ruff_source_file::LineIndex; + + use super::{DeclarationKind, UsageCountKinds, document_usage_counts}; + + impl CursorTest { + /// one line per declaration of the cursor's file: its kind, its name, + /// the line its name is on, then `refs=` and `impls=` with each count + /// asked for, and `(abstract)` for an abstract method + fn usage_counts(&self) -> String { + let file = self.cursor.file; + let source = source_text(&self.db, file); + let index = LineIndex::from_source_text(&source); + let declarations = document_usage_counts( + &self.db, + self.program_file(file), + UsageCountKinds { + references: true, + implementations: true, + }, + ); + if declarations.is_empty() { + return "[]".to_string(); + } + declarations + .iter() + .map(|declaration| { + let kind = match declaration.kind { + DeclarationKind::Class => "class", + DeclarationKind::Function => "function", + DeclarationKind::Method => "method", + }; + let mut line = format!( + "{kind} {} {}", + declaration.name, + index.line_index(declaration.name_range.start()) + ); + if let Some(references) = declaration.references { + let _ = write!(line, " refs={references}"); + } + if let Some(implementations) = declaration.implementations { + let _ = write!(line, " impls={implementations}"); + } + if declaration.is_abstract { + line.push_str(" (abstract)"); + } + line + }) + .collect::>() + .join("\n") + } + } + + #[test] + fn each_declaration_is_counted() { + let test = cursor_test( + " +from abc import ABC, abstractmethod + +class Animal(ABC): + @abstractmethod + def speak(self) -> str: ... + + def name(self) -> str: + return 'animal' + +class Dog(Animal): + def speak(self) -> str: + return 'woof' + +class Puppy(Dog): + def speak(self) -> str: + return 'yip' + +def make(animal: Animal) -> str: + return animal.speak() + animal.name() + +make(Dog()) +make(Puppy()) +", + ); + assert_snapshot!(test.usage_counts(), @" + class Animal 4 refs=2 impls=2 + method speak 6 refs=1 impls=2 (abstract) + method name 8 refs=1 impls=0 + class Dog 11 refs=2 impls=1 + method speak 12 refs=0 impls=1 + class Puppy 15 refs=1 impls=0 + method speak 16 refs=0 impls=0 + function make 19 refs=2 + "); + } + + #[test] + fn usages_in_other_files_are_counted() { + let test = CursorTest::builder() + .source( + "lib.py", + " +class Shape: + def area(self) -> float: + return 0.0 + +def unused() -> None: ... +", + ) + .source( + "main.py", + " +from lib import Shape + +class Square(Shape): + def area(self) -> float: + return 1.0 + +print(Shape().area(), Square().area()) +", + ) + .build(); + assert_snapshot!(test.usage_counts(), @" + class Shape 2 refs=2 impls=1 + method area 3 refs=1 impls=1 + function unused 6 refs=0 + "); + } + + #[test] + fn dunders_nested_functions_and_extensions_are_not_counted() { + let test = CursorTest::builder() + .source( + "main.by", + " +class A: + def __init__(self) -> None: ... + + def f(self) -> None: + def inner() -> None: ... + inner() + +extension list: + def second(self) -> int: + return 0 +", + ) + .build(); + assert_snapshot!(test.usage_counts(), @" + class A 2 refs=0 impls=0 + method f 5 refs=0 impls=0 + method second 10 refs=0 impls=0 + "); + } +} diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 7e01065acd..85236e3f21 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -223,6 +223,11 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + requests::DocumentUsageCountsRequestHandler::METHOD => background_document_request_task::< + requests::DocumentUsageCountsRequestHandler, + >( + req, BackgroundSchedule::Worker + ), requests::PrepareTypeHierarchyRequestHandler::METHOD => background_document_request_task::< requests::PrepareTypeHierarchyRequestHandler, >( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index cc9be78875..e6229d6a60 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -27,6 +27,7 @@ mod diagnostic; mod doc_highlights; mod document_super_members; mod document_symbols; +mod document_usage_counts; mod entry_point; mod evaluatable_expression; mod execute_command; @@ -77,6 +78,7 @@ pub(super) use diagnostic::DocumentDiagnosticRequestHandler; pub(super) use doc_highlights::DocumentHighlightRequestHandler; pub(super) use document_super_members::DocumentSuperMembersRequestHandler; pub(super) use document_symbols::DocumentSymbolRequestHandler; +pub(super) use document_usage_counts::DocumentUsageCountsRequestHandler; pub(super) use entry_point::EntryPointHandler; pub(super) use evaluatable_expression::EvaluatableExpressionRequestHandler; pub(super) use execute_command::ExecuteCommand; diff --git a/crates/ty_server/src/server/api/requests/document_usage_counts.rs b/crates/ty_server/src/server/api/requests/document_usage_counts.rs new file mode 100644 index 0000000000..94cc6ec606 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/document_usage_counts.rs @@ -0,0 +1,174 @@ +//! `by/documentUsageCounts`: how many places use each declaration of a document, +//! and how many classes subclass it or members override it +//! +//! the counts an editor draws above each declaration, "3 usages" or "2 +//! overrides", for the whole document at once. each count is what +//! `textDocument/references` (without the declaration) or +//! `textDocument/implementation` (less the declaration itself) answers at the +//! declaration's name, from the same code, so a client goes to as many places as +//! the count says by sending that request when one is clicked. asking either of +//! those of each declaration instead would be two requests per declaration on +//! every pass over the document +//! +//! not a `textDocument/codeLens`, because what a client wants of these is not +//! what it wants of a lens: it draws them as its own kind of hint, under a +//! setting of its own, and asks again whenever anything in the workspace changes, +//! where a lens is asked for when its document changes. the counts are a search +//! of the workspace for each declaration, so a client asks only for the ones it +//! draws +//! +//! one document at a time, whole. `textHash` says which revision of this +//! document the answer is about, but the counts also read every other file of the +//! workspace, and no change to those is announced for this request. a client asks +//! again after it opens, edits or closes any document of the workspace + +use std::borrow::Cow; + +use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, TextDocumentIdentifier, Uri}; +use ty_ide::{DeclarationKind, UsageCountKinds, document_usage_counts}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::ToRangeExt; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum DocumentUsageCountsRequest {} + +impl Request for DocumentUsageCountsRequest { + type Params = DocumentUsageCountsParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/documentUsageCounts"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// which document to answer for, and which counts to work out +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct DocumentUsageCountsParams { + text_document: TextDocumentIdentifier, + /// the counts wanted. a count not named is not worked out, and is left out + /// of the answer + counts: Vec, +} + +/// a count of [`DeclarationUsages`] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum UsageCount { + References, + Implementations, +} + +/// what a declaration is +#[derive(Debug, Clone, Copy, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum Kind { + Class, + Function, + /// a method or a property of a class + Method, +} + +/// a declaration of the document, with its counts +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DeclarationUsages { + name: String, + kind: Kind, + /// the whole declaration, its decorators included + range: Range, + /// its name, where `textDocument/references` and `textDocument/implementation` + /// answer for it + selection_range: Range, + /// how many places use it, as `textDocument/references` answers without the + /// declaration. there when asked for + #[serde(skip_serializing_if = "Option::is_none")] + references: Option, + /// how many classes subclass a class, directly or not, and how many of their + /// members override a method: what `textDocument/implementation` answers, + /// less the declaration itself. there when asked for, on a class or a method + #[serde(skip_serializing_if = "Option::is_none")] + implementations: Option, + /// whether a method is abstract, an `@abstractmethod` or a protocol member + /// with no body, so that what overrides it implements it. only said of a + /// method with implementations counted and found + #[serde(rename = "abstract")] + is_abstract: bool, +} + +pub(crate) struct DocumentUsageCountsRequestHandler; + +impl RequestHandler for DocumentUsageCountsRequestHandler { + type RequestType = DocumentUsageCountsRequest; +} + +impl BackgroundDocumentRequestHandler for DocumentUsageCountsRequestHandler { + // what the project's files say, which is the same whether this one is a buffer or the file on + // disk; the answer reads no version + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &DocumentUsageCountsParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, or for a document whose declarations are not + /// counted here: a notebook, whose ranges are per cell, or a template. otherwise the + /// declarations in source order: the document's classes and functions, and the methods and + /// properties of those classes, less dunders and `extension` blocks + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: DocumentUsageCountsParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let encoding = snapshot.encoding(); + let kinds = UsageCountKinds { + references: params.counts.contains(&UsageCount::References), + implementations: params.counts.contains(&UsageCount::Implementations), + }; + + Ok(Some( + document_usage_counts(db, db.program_file(file), kinds) + .into_iter() + .filter_map(|declaration| { + Some(DeclarationUsages { + range: declaration + .full_range + .to_lsp_range(db, file, encoding)? + .local_range(), + selection_range: declaration + .name_range + .to_lsp_range(db, file, encoding)? + .local_range(), + name: declaration.name, + kind: match declaration.kind { + DeclarationKind::Class => Kind::Class, + DeclarationKind::Function => Kind::Function, + DeclarationKind::Method => Kind::Method, + }, + references: declaration.references, + implementations: declaration.implementations, + is_abstract: declaration.is_abstract, + }) + }) + .collect(), + )) + } +} + +impl RetriableRequestHandler for DocumentUsageCountsRequestHandler {} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 72430cf1a4..49e11bceb8 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -62,6 +62,7 @@ mod super_members; mod syntax_outline; mod transpile; mod type_hierarchy; +mod usage_counts; mod will_rename_files; mod workspace_folders; diff --git a/crates/ty_server/tests/e2e/usage_counts.rs b/crates/ty_server/tests/e2e/usage_counts.rs new file mode 100644 index 0000000000..67aaa04e30 --- /dev/null +++ b/crates/ty_server/tests/e2e/usage_counts.rs @@ -0,0 +1,232 @@ +//! `by/documentUsageCounts` over the wire +//! +//! what counts as a usage or an override is `ty_ide`'s and its unit tests cover it. what these +//! cover is the contract a client reads: each declaration with its ranges, only the counts asked +//! for, and each count the number of places `textDocument/references` or +//! `textDocument/implementation` goes to from the declaration's name, which is what a client asks +//! when the count is clicked + +use anyhow::Result; +use lsp_types::{ + Definition, ImplementationParams, ImplementationRequest, ImplementationResponse, + LspRequestMethod, MessageDirection, PartialResultParams, Position, ReferenceContext, + ReferenceParams, ReferencesRequest, Request, TextDocumentIdentifier, + TextDocumentPositionParams, WorkDoneProgressParams, +}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder}; + +/// the request as a client sends it, in json throughout +enum DocumentUsageCounts {} + +impl Request for DocumentUsageCounts { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/documentUsageCounts"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const SHAPES: &str = "\ +from abc import ABC, abstractmethod + +class Shape(ABC): + @abstractmethod + def area(self) -> float: ... + + def describe(self) -> str: + return 'a shape' + +def total(shapes: list[Shape]) -> float: + return sum(shape.area() for shape in shapes) +"; + +const MAIN: &str = "\ +from shapes import Shape, total + +class Square(Shape): + def area(self) -> float: + return 1.0 + +print(total([Square()]), Square().describe()) +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes.by"), SHAPES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/shapes.by"), SHAPES, 1); + Ok(server) +} + +fn ask(server: &mut TestServer, counts: &[&str]) -> Vec { + let uri = server.file_uri(SystemPath::new("src/shapes.by")); + server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "counts": counts, + })) + .expect("an answer for a document") + .as_array() + .expect("a list of declarations") + .clone() +} + +fn position(declaration: &serde_json::Value) -> Position { + serde_json::from_value(declaration["selectionRange"]["start"].clone()).expect("a position") +} + +#[test] +fn each_declaration_comes_with_its_ranges_and_kind() -> Result<()> { + let mut server = server()?; + + let declarations = ask(&mut server, &[]); + let names: Vec<_> = declarations + .iter() + .map(|declaration| { + format!( + "{} {}", + declaration["kind"].as_str().unwrap_or_default(), + declaration["name"].as_str().unwrap_or_default() + ) + }) + .collect(); + assert_eq!( + names, + [ + "class Shape", + "method area", + "method describe", + "function total" + ] + ); + + // a decorated method's range starts at its decorator, and its name is where it is asked about + let area = &declarations[1]; + assert_eq!( + area["range"]["start"], + serde_json::json!({ "line": 3, "character": 4 }) + ); + assert_eq!( + area["selectionRange"], + serde_json::json!({ + "start": { "line": 4, "character": 8 }, + "end": { "line": 4, "character": 12 }, + }), + ); + // nothing asked for, nothing counted + assert_eq!(area.get("references"), None); + assert_eq!(area.get("implementations"), None); + Ok(()) +} + +#[test] +fn only_the_counts_asked_for_are_worked_out() -> Result<()> { + let mut server = server()?; + + for declaration in ask(&mut server, &["references"]) { + assert!(declaration["references"].is_number(), "{declaration:#}"); + assert_eq!(declaration.get("implementations"), None, "{declaration:#}"); + } + for declaration in ask(&mut server, &["implementations"]) { + assert_eq!(declaration.get("references"), None, "{declaration:#}"); + // a function is neither subclassed nor overridden + assert_eq!( + declaration["implementations"].is_number(), + declaration["kind"] != "function", + "{declaration:#}" + ); + } + Ok(()) +} + +#[test] +fn a_count_is_the_number_of_places_the_request_it_stands_for_goes_to() -> Result<()> { + let mut server = server()?; + let uri = server.file_uri(SystemPath::new("src/shapes.by")); + + let declarations = ask(&mut server, &["references", "implementations"]); + let counts: Vec<_> = declarations + .iter() + .map(|declaration| { + ( + declaration["name"].as_str().unwrap_or_default().to_string(), + declaration["references"].as_u64(), + declaration["implementations"].as_u64(), + declaration["abstract"].as_bool(), + ) + }) + .collect(); + assert_eq!( + counts, + [ + ("Shape".to_string(), Some(2), Some(1), Some(false)), + ("area".to_string(), Some(1), Some(1), Some(true)), + ("describe".to_string(), Some(1), Some(0), Some(false)), + ("total".to_string(), Some(1), None, Some(false)), + ] + ); + + for declaration in &declarations { + let at = TextDocumentPositionParams { + text_document: TextDocumentIdentifier { uri: uri.clone() }, + position: position(declaration), + }; + let references = server + .send_request_await::(ReferenceParams { + text_document_position_params: at.clone(), + work_done_progress_params: WorkDoneProgressParams::default(), + partial_result_params: PartialResultParams::default(), + context: ReferenceContext { + include_declaration: false, + }, + }) + .unwrap_or_default(); + assert_eq!( + declaration["references"].as_u64(), + Some(references.len() as u64), + "{declaration:#}" + ); + + if declaration["kind"] == "function" { + continue; + } + let implementations = + match server.send_request_await::(ImplementationParams { + text_document_position_params: at.clone(), + work_done_progress_params: WorkDoneProgressParams::default(), + partial_result_params: PartialResultParams::default(), + }) { + Some(ImplementationResponse::Definition(Definition::Location(location))) => { + vec![(location.uri, location.range.start)] + } + Some(ImplementationResponse::Definition(Definition::LocationList(locations))) => { + locations + .into_iter() + .map(|location| (location.uri, location.range.start)) + .collect() + } + Some(ImplementationResponse::DefinitionLinkList(links)) => links + .into_iter() + .map(|link| (link.target_uri, link.target_selection_range.start)) + .collect(), + None => Vec::new(), + }; + // less the declaration itself, which is the first of its own implementations + let others = implementations + .iter() + .filter(|(target, start)| *target != uri || *start != at.position) + .count(); + assert_eq!( + declaration["implementations"].as_u64(), + Some(others as u64), + "{declaration:#}" + ); + } + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index ab41203b07..670116dfc3 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -432,6 +432,37 @@ overrides, and `by/superMembers` still answers one. like the other document requests it takes `textHash`, and answers a document the client has not opened from the file on disk +## usage counts + +`by/documentUsageCounts` answers, for a whole document, how many places use +each of its declarations and how many classes subclass it or members override +it: what an editor draws above each one as "3 usages" or "2 overrides". the +declarations are the document's outline at the depth a reader scans a module +at — its classes and functions, and the methods and properties of those +classes. a dunder is left out, being called by the language rather than by +name, and so is an `extension` block, which names a class it does not declare + +each declaration comes with its `name`, its `kind` (`class`, `function` or +`method`), its whole `range` from its first decorator, its `selectionRange` on +its name, and the counts named in the request's `counts`: + +| count | what it is | +| ----------------- | ------------------------------------------------------------------------------------------- | +| `references` | the places `textDocument/references` finds from the name, the declaration not among them | +| `implementations` | of a class or a method, what `textDocument/implementation` finds from the name, less itself | + +so the count is the number of places the click on it goes to, when a client +sends that request from the name. a method with implementations is marked +`abstract` when it is an `@abstractmethod` or a protocol member with no body, +so that what overrides it implements it. each count is a search of the +workspace per declaration, and a count not named is not worked out + +a count depends on every file of the workspace, and the server announces no +change to it: a client asks again after any document of the workspace changes. +it is not a `textDocument/codeLens`, since a client that draws lenses asks for +them again only when their own document changes. like the other document +requests it takes `textHash` + ## the whole project's diagnostics with `diagnosticMode` set to `workspace`, the server checks every file in the From 5f3cdebd12121b6bf1be23e91ef68f45b093a2af Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:08:24 +1000 Subject: [PATCH 06/15] answer which tests call which code with by/testSubjects and by/subjectTests, with where a new test goes, and a file's or a package's dotted name with by/moduleName --- crates/ty_ide/src/lib.rs | 2 + crates/ty_ide/src/test_links.rs | 847 ++++++++++++++++++ crates/ty_server/src/server/api.rs | 9 + crates/ty_server/src/server/api/requests.rs | 4 + .../src/server/api/requests/module_name.rs | 95 ++ .../src/server/api/requests/test_subjects.rs | 265 ++++++ crates/ty_server/tests/e2e/main.rs | 1 + crates/ty_server/tests/e2e/test_subjects.rs | 231 +++++ docs/basedpython/features/editor.md | 43 + 9 files changed, 1497 insertions(+) create mode 100644 crates/ty_ide/src/test_links.rs create mode 100644 crates/ty_server/src/server/api/requests/module_name.rs create mode 100644 crates/ty_server/src/server/api/requests/test_subjects.rs create mode 100644 crates/ty_server/tests/e2e/test_subjects.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 9e1800437c..f670520980 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -39,6 +39,7 @@ mod stub_mapping; mod super_members; mod symbols; mod syntax_outline; +mod test_links; mod type_hierarchy; mod usage_counts; mod workspace_symbols; @@ -97,6 +98,7 @@ pub use syntax_outline::{ OutlineCall, OutlineClause, OutlineModifier, OutlineStatement, OutlineString, SyntaxOutline, syntax_outline, }; +pub use test_links::{NewTest, SubjectTests, TestLink, subject_tests, test_subjects}; pub use type_hierarchy::{ TypeHierarchyItem, prepare_type_hierarchy, type_hierarchy_subtypes, type_hierarchy_supertypes, }; diff --git a/crates/ty_ide/src/test_links.rs b/crates/ty_ide/src/test_links.rs new file mode 100644 index 0000000000..e8132613c2 --- /dev/null +++ b/crates/ty_ide/src/test_links.rs @@ -0,0 +1,847 @@ +//! which tests exercise which code: what an editor's *go to test* and *go to +//! test subject* ask +//! +//! a test exercises a subject when it calls it, the way call hierarchy sees a +//! call: a function or a method called by name or as an attribute, or a class +//! constructed. so a test's subjects are its outgoing calls, and a subject's +//! tests are the collected tests among its incoming calls. nothing here goes by +//! names: `test_circle.by` is not the test of `circle.by` unless something in +//! it calls something in `circle.by`, and a test of `circle.by` is found +//! wherever it is and whatever it is called +//! +//! which declarations are tests is pytest's collection, as +//! [`collected_pytest_tests`] models it. a subject is the project's own code: a +//! file the project checks that is not a test module (one tests are collected +//! from) and not a `conftest`, which holds fixtures for tests rather than code +//! under test + +use ruff_db::files::{File, system_path_to_file}; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_db::system::{SystemPath, SystemPathBuf}; +use ruff_diagnostics::Edit; +use ruff_python_ast::{self as ast}; +use ruff_python_codegen::Stylist; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::{FxHashMap, FxHashSet}; +use ty_python_core::ProgramFile; +use ty_python_core::program::Program; +use ty_python_semantic::{CollectedTest, collected_pytest_tests}; + +use crate::importer::{ImportRequest, Importer}; +use crate::{CallHierarchyItem, Db, SymbolKind, incoming_calls, outgoing_calls}; + +/// a test, or a subject a test exercises +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TestLink { + /// for a test, the `::`-separated names after the file in its pytest node + /// id: `TestTotal::test_total`. for a subject, its name within its module, + /// with the classes it is declared in: `Circle.area` + pub name: String, + pub kind: SymbolKind, + /// the module the test or the subject is declared in + pub module: Option, + pub file: File, + /// the whole declaration + pub full_range: TextRange, + /// its name + pub focus_range: TextRange, +} + +/// the subjects a test exercises: the project's own functions, methods and +/// classes it calls, each once, in the order the test first calls them +/// +/// at `offset` in `file` is a test, a class tests are collected under, or +/// anywhere else in a module tests are collected from, which stand for that +/// test, every test collected under that class, and every test in the module. +/// `None` when no test is collected from `file` +pub fn test_subjects( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option> { + let tests = collected_pytest_tests(db, file); + if tests.is_empty() { + return None; + } + let program = file.program(db); + let project_files = db.project().files(db); + let mut subject_files: FxHashMap = FxHashMap::default(); + let mut seen = FxHashSet::default(); + let mut subjects = Vec::new(); + for test in tests_at(&tests, offset) { + let mut calls = outgoing_calls(db, file, test.focus_range.start()); + calls.sort_by_key(|call| call.from_ranges.iter().map(Ranged::start).min()); + for call in calls { + let callee = call.to; + if !matches!( + callee.kind, + SymbolKind::Class + | SymbolKind::Function + | SymbolKind::Method + | SymbolKind::Constructor + ) { + continue; + } + let is_subject_file = *subject_files.entry(callee.file).or_insert_with(|| { + callee.file != file.file(db) + && project_files.contains(callee.file) + && !is_test_support(db, ProgramFile::new(db, callee.file, program)) + }); + if !is_subject_file || !seen.insert((callee.file, callee.selection_range)) { + continue; + } + subjects.push(subject_link(db, program, callee)); + } + } + Some(subjects) +} + +/// the tests that exercise a subject, and where a new one would go +#[derive(Debug, Clone)] +pub struct SubjectTests { + /// the collected tests that call the subject, by file and then in source + /// order + pub tests: Vec, + /// how a test of the subject would be added. `None` when the subject's + /// module has no name to import it by, is not a `.by` or a `.py` file, or is + /// itself a test module + pub new_test: Option, +} + +/// where a new test of a subject goes, and what it imports +#[derive(Debug, Clone)] +pub struct NewTest { + /// the test module: `test_` and the last part of the subject's module name, + /// written in what the subject's file is written in, `.by` or `.py`, in the + /// directory most of the project's test modules are in, or in `tests` under + /// the project root when it has none + pub path: SystemPathBuf, + /// whether that module exists already + pub exists: bool, + /// the subject's module + pub module: String, + /// the name to import from [`Self::module`]: the subject's own, or for a + /// method the class it is declared in. `None` for a module, which a test + /// imports whole + pub import: Option, + /// a name for the test function: `test_` and the subject's name in snake + /// case, `test_circle_area` for `Circle.area` + pub test_name: String, + /// for a test module that exists, the edit that imports [`Self::import`] + /// into it; `None` when it imports it already, or there is nothing to + /// import + pub import_edit: Option, +} + +/// the tests that exercise the subject at `offset` in `file`, and where a new +/// one would go +/// +/// the subject is the function, method or class whose declaration `offset` is +/// in, the outermost one a test could call: a function local to another stands +/// for the function it is local to. a class stands for itself and the methods it +/// declares, so a test that only calls a method of the class counts. anywhere +/// else, the subject is the module, standing for every function and class it +/// declares +pub fn subject_tests(db: &dyn Db, file: ProgramFile<'_>, offset: TextSize) -> SubjectTests { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let subject = Subject::at(parsed.suite(), offset); + let program = file.program(db); + + let mut collected: FxHashMap> = FxHashMap::default(); + let mut seen = FxHashSet::default(); + let mut tests = Vec::new(); + for name in subject.names() { + for call in incoming_calls(db, file, name.start()) { + let caller = call.from; + let caller_tests = collected.entry(caller.file).or_insert_with(|| { + collected_pytest_tests(db, ProgramFile::new(db, caller.file, program)) + }); + let Some(test) = caller_tests + .iter() + .find(|test| test.focus_range == caller.selection_range) + else { + continue; + }; + if !seen.insert((caller.file, test.focus_range)) { + continue; + } + tests.push(TestLink { + name: node_id(test), + kind: if test.classes.is_empty() { + SymbolKind::Function + } else { + SymbolKind::Method + }, + module: caller.detail, + file: caller.file, + full_range: test.full_range, + focus_range: test.focus_range, + }); + } + } + tests.sort_by(|a, b| { + a.file + .path(db) + .as_str() + .cmp(b.file.path(db).as_str()) + .then(a.focus_range.start().cmp(&b.focus_range.start())) + }); + + SubjectTests { + tests, + new_test: new_test(db, file, &subject), + } +} + +/// what a request for tests is about +enum Subject<'ast> { + Function(&'ast ast::StmtFunctionDef), + Method { + class: &'ast ast::StmtClassDef, + method: &'ast ast::StmtFunctionDef, + }, + Class(&'ast ast::StmtClassDef), + Module(&'ast [ast::Stmt]), +} + +impl<'ast> Subject<'ast> { + /// the subject whose declaration `offset` is in + /// + /// a method of a class nested in another class is the nested class's; a + /// class or a function nested in a function is the function's, since only + /// the function is reachable from a test + fn at(suite: &'ast [ast::Stmt], offset: TextSize) -> Self { + let mut subject = Subject::Module(suite); + let mut body = suite; + loop { + let Some(statement) = body + .iter() + .find(|statement| statement.range().contains_inclusive(offset)) + else { + return subject; + }; + match statement { + ast::Stmt::ClassDef(class) => { + subject = Subject::Class(class); + body = &class.body; + } + ast::Stmt::FunctionDef(function) => { + return match subject { + Subject::Class(class) => Subject::Method { + class, + method: function, + }, + _ => Subject::Function(function), + }; + } + _ => return subject, + } + } + } + + /// the names to find calls to: of the subject, and for a class or a module + /// of what it declares + fn names(&self) -> Vec { + fn class_names(class: &ast::StmtClassDef, names: &mut Vec) { + names.push(class.name.range()); + for statement in &class.body { + if let ast::Stmt::FunctionDef(method) = statement { + names.push(method.name.range()); + } + } + } + let mut names = Vec::new(); + match self { + Subject::Function(function) + | Subject::Method { + method: function, .. + } => names.push(function.name.range()), + Subject::Class(class) => class_names(class, &mut names), + Subject::Module(suite) => { + for statement in *suite { + match statement { + ast::Stmt::ClassDef(class) => class_names(class, &mut names), + ast::Stmt::FunctionDef(function) => names.push(function.name.range()), + _ => {} + } + } + } + } + names + } + + /// the name a test imports the subject by, from its module + fn import(&self) -> Option<&'ast str> { + match self { + Subject::Function(function) => Some(function.name.as_str()), + Subject::Method { class, .. } | Subject::Class(class) => Some(class.name.as_str()), + Subject::Module(_) => None, + } + } + + /// what a test of the subject is called after: `None` for a module, which is + /// named for its file + fn test_name(&self) -> Option { + match self { + Subject::Function(function) => Some(snake_case(function.name.as_str())), + Subject::Method { class, method } => Some(format!( + "{}_{}", + snake_case(class.name.as_str()), + snake_case(method.name.as_str()) + )), + Subject::Class(class) => Some(snake_case(class.name.as_str())), + Subject::Module(_) => None, + } + } +} + +/// the tests at `offset`: the test whose declaration it is in, else every test +/// collected under the innermost class it is in, else every test there is +fn tests_at(tests: &[CollectedTest], offset: TextSize) -> Vec<&CollectedTest> { + if let Some(test) = tests + .iter() + .find(|test| test.full_range.contains_inclusive(offset)) + { + return vec![test]; + } + let class = tests + .iter() + .flat_map(|test| &test.classes) + .filter(|class| class.full_range.contains_inclusive(offset)) + .min_by_key(|class| class.full_range.len()); + match class { + Some(class) => tests + .iter() + .filter(|test| { + test.classes + .iter() + .any(|under| under.full_range == class.full_range) + }) + .collect(), + None => tests.iter().collect(), + } +} + +/// a test's node id after its file: its classes and its name, `::`-separated +fn node_id(test: &CollectedTest) -> String { + let mut id = String::new(); + for class in &test.classes { + id.push_str(&class.name); + id.push_str("::"); + } + id.push_str(&test.name); + id +} + +/// whether `file` is for tests rather than under test: tests are collected from +/// it, or it is a `conftest` +fn is_test_support(db: &dyn Db, file: ProgramFile<'_>) -> bool { + file.file(db) + .path(db) + .as_system_path() + .and_then(SystemPath::file_stem) + .is_some_and(|stem| stem == "conftest") + || !collected_pytest_tests(db, file).is_empty() +} + +/// a subject as a [`TestLink`], named with the classes it is declared in +fn subject_link(db: &dyn Db, program: Program<'_>, item: CallHierarchyItem) -> TestLink { + let parsed = + parsed_module(db, ProgramFile::new(db, item.file, program).python_file(db)).load(db); + let mut name = enclosing_classes(parsed.suite(), item.full_range).join("."); + if !name.is_empty() { + name.push('.'); + } + name.push_str(item.name.as_str()); + TestLink { + name, + kind: item.kind, + module: item.detail, + file: item.file, + full_range: item.full_range, + focus_range: item.selection_range, + } +} + +/// the names of the classes whose bodies hold the declaration at `range`, +/// outermost first +fn enclosing_classes(suite: &[ast::Stmt], range: TextRange) -> Vec<&str> { + let mut names = Vec::new(); + let mut body = suite; + while let Some(class) = body.iter().find_map(|statement| match statement { + ast::Stmt::ClassDef(class) + if class.range().contains_range(range) && class.range() != range => + { + Some(class) + } + _ => None, + }) { + names.push(class.name.as_str()); + body = &class.body; + } + names +} + +/// where a new test of `subject`, in `file`, goes +fn new_test(db: &dyn Db, file: ProgramFile<'_>, subject: &Subject<'_>) -> Option { + if is_test_support(db, file) { + return None; + } + let module = ty_module_resolver::file_to_module(db, file.resolver_file(db))? + .name(db) + .to_string(); + let stem = module.rsplit('.').next().unwrap_or(&module); + let test_name = format!( + "test_{}", + subject.test_name().unwrap_or_else(|| snake_case(stem)) + ); + // a new test module is written in what the subject is, so a project that writes python goes on + // writing python; a test module there already is used whatever it is written in, since a `.by` + // one beside a `.py` one of the same name would shadow it, and the `.by` one of two is the + // module the name imports + let extension = file + .file(db) + .path(db) + .extension() + .filter(|extension| matches!(*extension, "by" | "py"))?; + let directory = test_directory(db, file); + let (path, existing) = ["by", "py"] + .into_iter() + .find_map(|extension| { + let path = directory.join(format!("test_{stem}.{extension}")); + let existing = system_path_to_file(db, &path).ok()?; + Some((path, Some(existing))) + }) + .unwrap_or_else(|| (directory.join(format!("test_{stem}.{extension}")), None)); + let import = subject.import().map(str::to_string); + + let test_name = match existing { + Some(test_file) => unused_name( + db, + ProgramFile::new(db, test_file, file.program(db)), + test_name, + ), + None => test_name, + }; + let import_edit = existing + .zip(import.as_deref()) + .and_then(|(test_file, name)| { + import_edit( + db, + ProgramFile::new(db, test_file, file.program(db)), + &module, + name, + ) + }); + Some(NewTest { + path, + exists: existing.is_some(), + module, + import, + test_name, + import_edit, + }) +} + +/// `name`, or when the module binds it at the top level already, `name` with the +/// first of `_2`, `_3` and so on that it does not, so that a new test does not +/// redefine one that is there +fn unused_name(db: &dyn Db, file: ProgramFile<'_>, name: String) -> String { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let bound: FxHashSet<&str> = parsed + .suite() + .iter() + .filter_map(|statement| match statement { + ast::Stmt::FunctionDef(function) => Some(function.name.as_str()), + ast::Stmt::ClassDef(class) => Some(class.name.as_str()), + _ => None, + }) + .collect(); + if !bound.contains(name.as_str()) { + return name; + } + (2..) + .map(|suffix| format!("{name}_{suffix}")) + .find(|candidate| !bound.contains(candidate.as_str())) + .unwrap_or(name) +} + +/// the directory most of the project's test modules are in, nearest the +/// project root among those holding as many; `tests` under the project root +/// when no test is collected anywhere +fn test_directory(db: &dyn Db, file: ProgramFile<'_>) -> SystemPathBuf { + let program = file.program(db); + let mut counts: FxHashMap = FxHashMap::default(); + for project_file in &db.project().files(db) { + let Some(directory) = project_file + .path(db) + .as_system_path() + .and_then(SystemPath::parent) + else { + continue; + }; + if collected_pytest_tests(db, ProgramFile::new(db, project_file, program)).is_empty() { + continue; + } + *counts.entry(directory.to_path_buf()).or_default() += 1; + } + counts + .into_iter() + .max_by(|(a, a_count), (b, b_count)| { + a_count + .cmp(b_count) + .then_with(|| b.components().count().cmp(&a.components().count())) + .then_with(|| b.cmp(a)) + }) + .map_or_else( + || db.project().root(db).join("tests"), + |(directory, _)| directory, + ) +} + +/// the edit that imports `name` from `module` into `file`, or `None` when it is +/// imported there already +fn import_edit(db: &dyn Db, file: ProgramFile<'_>, module: &str, name: &str) -> Option { + let parsed: ParsedModuleRef = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let stylist = Stylist::from_tokens(parsed.tokens(), source.as_str()); + let importer = Importer::new(db, &stylist, file, source.as_str(), &parsed); + // in scope at the end of the module, which is where a new test goes + let end = TextSize::of(source.as_str()); + let members = importer.members_in_scope_at(parsed.syntax().into(), end); + importer + .import(ImportRequest::import_from(module, name), &members) + .import() + .cloned() +} + +/// `CamelCase` and `snake_case` names in snake case: `CircleArea` is +/// `circle_area`, `HTTPServer` is `http_server` +fn snake_case(name: &str) -> String { + let characters: Vec = name.chars().collect(); + let mut snake = String::with_capacity(name.len() + 4); + for (index, &character) in characters.iter().enumerate() { + if character.is_uppercase() { + let previous = index.checked_sub(1).map(|previous| characters[previous]); + let next = characters.get(index + 1); + let starts_word = previous.is_some_and(|previous| { + previous.is_lowercase() + || previous.is_ascii_digit() + || (previous.is_uppercase() && next.is_some_and(|next| next.is_lowercase())) + }); + if starts_word && !snake.ends_with('_') { + snake.push('_'); + } + snake.extend(character.to_lowercase()); + } else { + snake.push(character); + } + } + snake +} + +#[cfg(test)] +mod tests { + use crate::tests::CursorTest; + use insta::assert_snapshot; + use ruff_db::source::source_text; + use ruff_source_file::LineIndex; + + use super::{TestLink, snake_case}; + + const SHAPES: &str = "\ +class Shape: + def area(self) -> float: + return 0.0 + +class Circle(Shape): + def __init__(self, radius: float) -> None: + self.radius = radius + + def area(self) -> float: + return 3.14 * self.radius ** 2 + +def total(shapes: list[Shape]) -> float: + return sum(shape.area() for shape in shapes) + +def unused() -> None: ... +"; + + const TESTS: &str = "\ +from shapes import Circle, total + +def helper() -> Circle: + return Circle(1.0) + +def test_area() -> None: + assert Circle(1.0).area() > 3 + +def test_total() -> None: + assert total([helper()]) > 3 + +class TestShapes: + def test_circle(self) -> None: + assert Circle(2.0).radius == 2.0 +"; + + impl CursorTest { + fn links(&self, links: &[TestLink]) -> String { + links + .iter() + .map(|link| { + let source = source_text(&self.db, link.file); + let line = + LineIndex::from_source_text(&source).line_index(link.focus_range.start()); + format!( + "{} {} ({}) {}:{line}", + link.kind.to_string(), + link.name, + link.module.as_deref().unwrap_or("-"), + link.file.path(&self.db).as_str(), + ) + }) + .collect::>() + .join("\n") + } + + fn test_subjects(&self) -> String { + match super::test_subjects( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + ) { + None => "None".to_string(), + Some(subjects) => self.links(&subjects), + } + } + + fn subject_tests(&self) -> String { + let answer = super::subject_tests( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + ); + let new_test = match &answer.new_test { + None => "no new test".to_string(), + Some(new_test) => format!( + "new test {} {} in {}{} importing {} from {}{}", + new_test.test_name, + if new_test.exists { "added" } else { "created" }, + new_test.path, + "", + new_test.import.as_deref().unwrap_or("nothing"), + new_test.module, + match &new_test.import_edit { + Some(edit) => format!(" with {:?}", edit.content().unwrap_or_default()), + None => String::new(), + }, + ), + }; + format!("{}\n{new_test}", self.links(&answer.tests)) + } + } + + fn test(shapes: &str, tests: &str) -> CursorTest { + CursorTest::builder() + .source("shapes.py", shapes) + .source("tests/test_shapes.py", tests) + .build() + } + + #[test] + fn a_test_exercises_what_it_calls_in_the_order_it_first_calls_it() { + let test = test( + SHAPES, + &TESTS.replace("def test_area", "def test_area"), + ); + assert_snapshot!(test.test_subjects(), @" + Class Circle (shapes) /shapes.py:5 + Method Circle.area (shapes) /shapes.py:9 + "); + } + + #[test] + fn a_test_class_exercises_what_its_tests_call() { + let test = test( + SHAPES, + &TESTS.replace("class TestShapes", "class TestShapes"), + ); + assert_snapshot!(test.test_subjects(), @"Class Circle (shapes) /shapes.py:5"); + } + + #[test] + fn a_test_module_exercises_what_all_its_tests_call_and_not_its_helpers() { + let test = test(SHAPES, &TESTS.replace("from shapes", "from shapes")); + assert_snapshot!(test.test_subjects(), @" + Class Circle (shapes) /shapes.py:5 + Method Circle.area (shapes) /shapes.py:9 + Function total (shapes) /shapes.py:12 + "); + } + + #[test] + fn a_module_no_test_is_collected_from_has_no_test_subjects() { + let test = test(&SHAPES.replace("def total", "def total"), TESTS); + assert_snapshot!(test.test_subjects(), @"None"); + } + + #[test] + fn a_method_is_exercised_by_the_tests_that_call_it() { + let test = test( + &SHAPES.replace(" return 3.14", " return 3.14"), + TESTS, + ); + assert_snapshot!(test.subject_tests(), @" + Function test_area (tests.test_shapes) /tests/test_shapes.py:6 + new test test_circle_area added in /tests/test_shapes.py importing Circle from shapes + "); + } + + #[test] + fn a_class_is_exercised_by_the_tests_that_construct_it_or_call_its_methods() { + let test = test( + &SHAPES.replace("class Circle", "class Circle"), + TESTS, + ); + assert_snapshot!(test.subject_tests(), @" + Function test_area (tests.test_shapes) /tests/test_shapes.py:6 + Method TestShapes::test_circle (tests.test_shapes) /tests/test_shapes.py:13 + new test test_circle added in /tests/test_shapes.py importing Circle from shapes + "); + } + + #[test] + fn a_function_no_test_calls_is_exercised_by_none() { + let test = test(&SHAPES.replace("def unused", "def unused"), TESTS); + assert_snapshot!(test.subject_tests(), @r#" + + new test test_unused added in /tests/test_shapes.py importing unused from shapes with ", unused" + "#); + } + + #[test] + fn a_module_is_exercised_by_the_tests_that_call_anything_it_declares() { + let test = test(&format!("# shapes\n{SHAPES}"), TESTS); + assert_snapshot!(test.subject_tests(), @" + Function test_area (tests.test_shapes) /tests/test_shapes.py:6 + Function test_total (tests.test_shapes) /tests/test_shapes.py:9 + Method TestShapes::test_circle (tests.test_shapes) /tests/test_shapes.py:13 + new test test_shapes added in /tests/test_shapes.py importing nothing from shapes + "); + } + + #[test] + fn a_new_test_goes_beside_the_project_tests_in_a_module_named_for_the_subjects() { + let test = CursorTest::builder() + .source( + "geometry.by", + "def perimeter(sides: list[float]) -> float: ...\n", + ) + .source("tests/test_shapes.py", "def test_nothing() -> None: ...\n") + .build(); + assert_snapshot!(test.subject_tests(), @" + + new test test_perimeter created in /tests/test_geometry.by importing perimeter from geometry + "); + } + + #[test] + fn a_new_test_in_a_test_module_that_exists_comes_with_the_import_it_needs() { + let test = CursorTest::builder() + .source( + "geometry.py", + "def perimeter(sides: list[float]) -> float: ...\n", + ) + .source( + "tests/test_geometry.py", + "import math\n\ndef test_nothing() -> None: ...\n", + ) + .build(); + assert_snapshot!(test.subject_tests(), @r#" + + new test test_perimeter added in /tests/test_geometry.py importing perimeter from geometry with "from geometry import perimeter\n" + "#); + } + + #[test] + fn a_new_test_in_a_test_module_that_imports_the_subject_needs_no_import() { + let test = CursorTest::builder() + .source( + "geometry.py", + "def perimeter(sides: list[float]) -> float: ...\n", + ) + .source( + "tests/test_geometry.py", + "from geometry import perimeter\n\ndef test_nothing() -> None: ...\n", + ) + .build(); + assert_snapshot!(test.subject_tests(), @" + + new test test_perimeter added in /tests/test_geometry.py importing perimeter from geometry + "); + } + + /// a test module written in the other language is the one a new test goes in: a `.by` + /// module beside a `.py` one of the same name would shadow it + #[test] + fn a_new_test_goes_in_the_test_module_there_whatever_it_is_written_in() { + let test = CursorTest::builder() + .source( + "geometry.by", + "def perimeter(sides: list[float]) -> float: ...\n", + ) + .source( + "tests/test_geometry.py", + "from geometry import perimeter\n\ndef test_nothing() -> None: ...\n", + ) + .build(); + assert_snapshot!(test.subject_tests(), @" + + new test test_perimeter added in /tests/test_geometry.py importing perimeter from geometry + "); + } + + #[test] + fn a_new_test_is_named_so_as_not_to_redefine_one_there() { + let test = CursorTest::builder() + .source("geometry.py", "def perimeter(sides: list[float]) -> float: ...\n") + .source( + "tests/test_geometry.py", + "from geometry import perimeter\n\ndef test_perimeter() -> None: ...\n\ndef test_perimeter_2() -> None: ...\n", + ) + .build(); + assert_snapshot!(test.subject_tests(), @""); + } + + #[test] + fn with_no_tests_anywhere_a_new_test_goes_in_tests() { + let test = CursorTest::builder() + .source( + "geometry.py", + "def perimeter(sides: list[float]) -> float: ...\n", + ) + .build(); + assert_snapshot!(test.subject_tests(), @" + + new test test_perimeter created in /tests/test_geometry.py importing perimeter from geometry + "); + } + + #[test] + fn a_test_module_is_no_subject_to_add_a_test_of() { + let test = test(SHAPES, &TESTS.replace("def helper", "def helper")); + assert_snapshot!(test.subject_tests(), @" + Function test_total (tests.test_shapes) /tests/test_shapes.py:9 + no new test + "); + } + + #[test] + fn names_go_into_snake_case() { + assert_eq!(snake_case("CircleArea"), "circle_area"); + assert_eq!(snake_case("HTTPServer"), "http_server"); + assert_eq!(snake_case("area"), "area"); + assert_eq!(snake_case("total_area"), "total_area"); + } +} diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 85236e3f21..fc0828d373 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -208,6 +208,15 @@ pub(super) fn request(req: server::Request) -> Task { BackgroundSchedule::LatencySensitive, ) } + requests::TestSubjectsRequestHandler::METHOD => background_document_request_task::< + requests::TestSubjectsRequestHandler, + >(req, BackgroundSchedule::Worker), + requests::SubjectTestsRequestHandler::METHOD => background_document_request_task::< + requests::SubjectTestsRequestHandler, + >(req, BackgroundSchedule::Worker), + requests::ModuleNameHandler::METHOD => { + background_request_task::(req, BackgroundSchedule::Worker) + } requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index e6229d6a60..46d7dedf81 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -41,6 +41,7 @@ mod goto_type_definition; mod hover; mod injections; mod inlay_hints; +mod module_name; mod prepare_call_hierarchy; mod prepare_rename; mod prepare_type_hierarchy; @@ -55,6 +56,7 @@ mod signature_help; mod super_members; mod syntax_outline; mod test_items; +mod test_subjects; mod transpile; mod transpile_for_build; mod type_hierarchy_subtypes; @@ -92,6 +94,7 @@ pub(super) use goto_type_definition::GotoTypeDefinitionRequestHandler; pub(super) use hover::HoverRequestHandler; pub(super) use injections::InjectionsRequestHandler; pub(super) use inlay_hints::InlayHintRequestHandler; +pub(super) use module_name::ModuleNameHandler; pub(super) use prepare_call_hierarchy::PrepareCallHierarchyRequestHandler; pub(super) use prepare_rename::PrepareRenameRequestHandler; pub(super) use prepare_type_hierarchy::PrepareTypeHierarchyRequestHandler; @@ -106,6 +109,7 @@ pub(super) use signature_help::SignatureHelpRequestHandler; pub(super) use super_members::SuperMembersRequestHandler; pub(super) use syntax_outline::SyntaxOutlineRequestHandler; pub(super) use test_items::TestItemsHandler; +pub(super) use test_subjects::{SubjectTestsRequestHandler, TestSubjectsRequestHandler}; pub(super) use transpile::TranspileRequestHandler; pub(super) use transpile_for_build::TranspileForBuildRequestHandler; pub(super) use type_hierarchy_subtypes::TypeHierarchySubtypesRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/module_name.rs b/crates/ty_server/src/server/api/requests/module_name.rs new file mode 100644 index 0000000000..64b14dd1e4 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/module_name.rs @@ -0,0 +1,95 @@ +//! `by/moduleName`: the name a file or a directory is imported by +//! +//! what an editor's *copy reference* puts on the clipboard for a module or a package, and what +//! anything that names a module after its file needs. the name is the module resolver's: the search +//! path the file is under and the path from there, so a src-layout project's `src/shapes/circle.by` +//! is `shapes.circle`, its `src/shapes/__init__.by` and its `src/shapes` are both `shapes`, and a +//! file that resolution would never reach — shadowed by another file under the same name earlier +//! on the search path — has no name, rather than one that imports something else + +use lsp_types::{LspRequestMethod, MessageDirection, Request, Uri}; +use ruff_db::Db as _; +use ruff_db::files::system_path_to_file; +use ruff_db::system::SystemPathBuf; +use ty_module_resolver::{file_to_module, path_to_module_name}; +use ty_project::{Db as _, SemanticDb as _}; + +use crate::server::api::traits::{ + BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::SessionSnapshot; +use crate::session::client::Client; + +pub(crate) enum ModuleNameRequest {} + +impl Request for ModuleNameRequest { + type Params = ModuleNameParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/moduleName"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the file or directory to name +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ModuleNameParams { + uri: Uri, +} + +pub(crate) struct ModuleNameHandler; + +impl RequestHandler for ModuleNameHandler { + type RequestType = ModuleNameRequest; +} + +impl BackgroundRequestHandler for ModuleNameHandler { + /// `null` for anything outside every project, and for a path no module is imported from: one + /// on no search path, a directory or a file whose path is not a dotted name, or a file that + /// resolution does not reach + fn run( + snapshot: &SessionSnapshot, + _client: &Client, + params: ModuleNameParams, + ) -> crate::server::Result> { + let Some(path) = params + .uri + .to_file_path() + .ok() + .and_then(|path| SystemPathBuf::from_path_buf(path).ok()) + else { + return Ok(None); + }; + let Some(db) = snapshot + .projects() + .iter() + .filter(|db| path.starts_with(db.project().root(*db))) + .max_by_key(|db| db.project().root(*db).components().count()) + else { + return Ok(None); + }; + let name = if db.system().is_directory(&path) { + let environment = db.project().program(db).resolver_environment(db); + path_to_module_name(db, environment, &path).map(|name| name.to_string()) + } else { + system_path_to_file(db, &path).ok().and_then(|file| { + let module = file_to_module(db, db.program_file(file).resolver_file(db))?; + Some(module.name(db).to_string()) + }) + }; + Ok(name) + } +} + +impl RetriableRequestHandler for ModuleNameHandler {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_params_a_client_sends_parse() { + let params: ModuleNameParams = serde_json::from_str(r#"{"uri":"file:///p/src/shapes"}"#) + .expect("a client names a path"); + assert_eq!(params.uri.as_str(), "file:///p/src/shapes"); + } +} diff --git a/crates/ty_server/src/server/api/requests/test_subjects.rs b/crates/ty_server/src/server/api/requests/test_subjects.rs new file mode 100644 index 0000000000..c7f87e47ba --- /dev/null +++ b/crates/ty_server/src/server/api/requests/test_subjects.rs @@ -0,0 +1,265 @@ +//! `by/testSubjects` and `by/subjectTests`: which tests exercise which code +//! +//! what an editor's *go to test subject* and *go to test* ask, from a test to the code it calls +//! and from code to the tests that call it. a test exercises what it calls, as call hierarchy sees +//! a call, and which declarations are tests is pytest's collection as `by/testItems` reports it: +//! nothing goes by file names. `by/subjectTests` also says where a new test of the code would go, +//! for an editor's *create test* +//! +//! both answer about the project's files as they are, a document the client has not opened +//! included, and read no version: what a test calls is in other files too + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, SymbolKind, + TextDocumentIdentifier, TextEdit, Uri, +}; +use ruff_text_size::{Ranged, TextRange}; +use ty_ide::{SubjectTests, TestLink, subject_tests, test_subjects}; +use ty_project::{Db as _, ProjectDatabase, SemanticDb as _}; + +use crate::PositionEncoding; +use crate::document::{PositionExt, ToRangeExt}; +use crate::server::api::symbols::convert_symbol_kind; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; +use crate::system::file_to_uri; + +pub(crate) enum TestSubjectsRequest {} + +impl Request for TestSubjectsRequest { + type Params = TestLinksParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/testSubjects"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +pub(crate) enum SubjectTestsRequest {} + +impl Request for SubjectTestsRequest { + type Params = TestLinksParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/subjectTests"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// where to start from: for `by/testSubjects` a test, a class tests are collected under or anywhere +/// in a module tests are collected from; for `by/subjectTests` a function, a method, a class, or +/// anywhere else in a module, which stands for everything the module declares +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct TestLinksParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +/// a test, or code a test exercises +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TestLinkItem { + /// for a test, the `::`-separated names after the file in its pytest node id, + /// `TestTotal::test_total`; for code under test, its name in its module, with the classes it is + /// declared in, `Circle.area` + name: String, + kind: SymbolKind, + /// the module it is declared in + #[serde(skip_serializing_if = "Option::is_none")] + module: Option, + uri: Uri, + /// the whole declaration + range: Range, + /// its name, which is where to go + selection_range: Range, +} + +impl TestLinkItem { + fn from_ide(db: &ProjectDatabase, link: TestLink, encoding: PositionEncoding) -> Option { + let range = |range: TextRange| { + range + .to_lsp_range(db, link.file, encoding) + .map(|range| range.local_range()) + }; + Some(TestLinkItem { + uri: file_to_uri(db, link.file)?, + range: range(link.full_range)?, + selection_range: range(link.focus_range)?, + name: link.name, + kind: convert_symbol_kind(link.kind), + module: link.module, + }) + } +} + +/// the tests that exercise the code, and how to add one +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SubjectTestsResponse { + /// the collected tests that call the code, by file and then in source order + tests: Vec, + /// where a new test of the code goes; `null` when its module has no name a test could import + /// it by, or is a test module itself + new_test: Option, +} + +/// where a new test goes, and what it imports +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct NewTestItem { + /// the test module: `test_`, in the directory most of the project's test modules + /// are in, or in `tests` under the project root when there are none; the one there already + /// whatever it is written in, or a new one written in what the subject is + uri: Uri, + /// whether the test module exists already, so that a new test is added to it rather than + /// created with it + exists: bool, + /// the module the code is in + module: String, + /// the name a test imports from `module`: the function's or the class's, or for a method its + /// class's. absent for a module, which a test imports whole + #[serde(skip_serializing_if = "Option::is_none")] + import: Option, + /// a name for the test function, `test_` and the code's name in snake case + test_name: String, + /// for a test module that exists, the edits that import `import` into it, empty when it does + /// already + import_edits: Vec, +} + +impl NewTestItem { + fn from_ide( + db: &ProjectDatabase, + new_test: ty_ide::NewTest, + encoding: PositionEncoding, + ) -> Option { + let uri = Uri::from_file_path(new_test.path.as_std_path()).ok()?; + let import_edits = match &new_test.import_edit { + None => Vec::new(), + Some(edit) => { + let file = ruff_db::files::system_path_to_file(db, &new_test.path).ok()?; + vec![TextEdit { + range: edit.range().to_lsp_range(db, file, encoding)?.local_range(), + new_text: edit.content().unwrap_or_default().to_string(), + }] + } + }; + Some(NewTestItem { + uri, + exists: new_test.exists, + module: new_test.module, + import: new_test.import, + test_name: new_test.test_name, + import_edits, + }) + } +} + +pub(crate) struct TestSubjectsRequestHandler; + +impl RequestHandler for TestSubjectsRequestHandler { + type RequestType = TestSubjectsRequest; +} + +impl BackgroundDocumentRequestHandler for TestSubjectsRequestHandler { + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &TestLinksParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, for a template, and when no test is collected from + /// the document; otherwise what the tests at the position call, each once, in the order they + /// first call it, and an empty list when that is nothing of the project's own + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: TestLinksParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + Ok( + test_subjects(db, db.program_file(file), offset).map(|subjects| { + subjects + .into_iter() + .filter_map(|link| TestLinkItem::from_ide(db, link, snapshot.encoding())) + .collect() + }), + ) + } +} + +impl RetriableRequestHandler for TestSubjectsRequestHandler {} + +pub(crate) struct SubjectTestsRequestHandler; + +impl RequestHandler for SubjectTestsRequestHandler { + type RequestType = SubjectTestsRequest; +} + +impl BackgroundDocumentRequestHandler for SubjectTestsRequestHandler { + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &TestLinksParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, for a template, and for a file outside the project, + /// whose code no test of the project's is looked for + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: TestLinksParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + if !db.project().files(db).contains(file) { + return Ok(None); + } + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + let SubjectTests { tests, new_test } = subject_tests(db, db.program_file(file), offset); + Ok(Some(SubjectTestsResponse { + tests: tests + .into_iter() + .filter_map(|link| TestLinkItem::from_ide(db, link, snapshot.encoding())) + .collect(), + new_test: new_test + .and_then(|new_test| NewTestItem::from_ide(db, new_test, snapshot.encoding())), + })) + } +} + +impl RetriableRequestHandler for SubjectTestsRequestHandler {} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 49e11bceb8..cf0fb28d86 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -60,6 +60,7 @@ mod semantic_tokens; mod signature_help; mod super_members; mod syntax_outline; +mod test_subjects; mod transpile; mod type_hierarchy; mod usage_counts; diff --git a/crates/ty_server/tests/e2e/test_subjects.rs b/crates/ty_server/tests/e2e/test_subjects.rs new file mode 100644 index 0000000000..3f07a1e31f --- /dev/null +++ b/crates/ty_server/tests/e2e/test_subjects.rs @@ -0,0 +1,231 @@ +//! `by/testSubjects`, `by/subjectTests` and `by/moduleName` over the wire +//! +//! which tests call what is `ty_ide`'s and its unit tests cover it. what these cover is the contract +//! a client reads: a position in, the tests or the code as items it can show and go to, `null` kept +//! apart from an empty list, where a new test goes and the import it needs, and the dotted name of a +//! file or a package in a src-layout project, which is the search path's and not the directory's + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use serde_json::{Value, json}; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder}; + +enum TestSubjects {} + +impl Request for TestSubjects { + type Params = Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/testSubjects"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +enum SubjectTests {} + +impl Request for SubjectTests { + type Params = Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/subjectTests"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +enum ModuleName {} + +impl Request for ModuleName { + type Params = Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/moduleName"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const CIRCLE: &str = "\ +class Circle: + def __init__(self, radius: float) -> None: + self.radius = radius + + def area(self) -> float: + return 3.14 * self.radius ** 2 +"; + +const TEST_CIRCLE: &str = "\ +from shapes.circle import Circle + +def test_area() -> None: + assert Circle(1.0).area() > 3 + +class TestCircle: + def test_radius(self) -> None: + assert Circle(2.0).radius == 2.0 +"; + +/// a src-layout project with the code and its test open, as an editor asking from either has them: +/// what the answers read from every other file is the files on disk +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("project"), None)? + .with_file(SystemPath::new("project/src/shapes/__init__.by"), "")? + .with_file(SystemPath::new("project/src/shapes/circle.by"), CIRCLE)? + .with_file(SystemPath::new("project/tests/test_circle.by"), TEST_CIRCLE)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("project/src/shapes/circle.by"), CIRCLE, 1); + server.open_text_document( + SystemPath::new("project/tests/test_circle.by"), + TEST_CIRCLE, + 1, + ); + Ok(server) +} + +fn at(server: &TestServer, path: &str, line: u32, character: u32) -> Value { + json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new(path)) }, + "position": { "line": line, "character": character }, + }) +} + +#[test] +fn a_test_answers_what_it_calls_as_items_to_go_to() -> Result<()> { + let mut server = server()?; + let circle = server.file_uri(SystemPath::new("project/src/shapes/circle.by")); + + // in `test_area`'s body + let params = at(&server, "project/tests/test_circle.by", 3, 12); + let subjects = server + .send_request_await::(params) + .expect("tests are collected from the document"); + + let names: Vec<_> = subjects + .as_array() + .expect("a list of subjects") + .iter() + .map(|subject| subject["name"].as_str().unwrap_or_default()) + .collect(); + assert_eq!(names, ["Circle", "Circle.area"], "{subjects:#}"); + let area = &subjects[1]; + assert_eq!(area["kind"], 6, "a method"); + assert_eq!(area["module"], "shapes.circle"); + assert_eq!(area["uri"], circle.as_str()); + assert_eq!( + area["selectionRange"], + json!({ "start": { "line": 4, "character": 8 }, "end": { "line": 4, "character": 12 } }), + ); + assert_eq!(area["range"]["start"]["line"], 4); + Ok(()) +} + +#[test] +fn a_document_no_test_is_collected_from_has_no_test_subjects() -> Result<()> { + let mut server = server()?; + let params = at(&server, "project/src/shapes/circle.by", 4, 9); + assert_eq!(server.send_request_await::(params), None); + Ok(()) +} + +#[test] +fn a_class_answers_the_tests_that_call_it_and_where_another_goes() -> Result<()> { + let mut server = server()?; + let tests = server.file_uri(SystemPath::new("project/tests/test_circle.by")); + + // on `class Circle` + let params = at(&server, "project/src/shapes/circle.by", 0, 7); + let answer = server + .send_request_await::(params) + .expect("the document is the project's"); + + let names: Vec<_> = answer["tests"] + .as_array() + .expect("a list of tests") + .iter() + .map(|test| test["name"].as_str().unwrap_or_default()) + .collect(); + assert_eq!( + names, + ["test_area", "TestCircle::test_radius"], + "{answer:#}" + ); + assert_eq!(answer["tests"][0]["module"], "tests.test_circle"); + assert_eq!(answer["tests"][0]["uri"], tests.as_str()); + assert_eq!( + answer["tests"][0]["selectionRange"], + json!({ "start": { "line": 2, "character": 4 }, "end": { "line": 2, "character": 13 } }), + ); + + // the test module exists and imports `Circle` already, so a new test needs nothing more + assert_eq!( + answer["newTest"], + json!({ + "uri": tests.as_str(), + "exists": true, + "module": "shapes.circle", + "import": "Circle", + "testName": "test_circle", + "importEdits": [], + }), + ); + Ok(()) +} + +#[test] +fn a_new_test_in_a_test_module_that_does_not_import_the_code_comes_with_the_import() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("project"), None)? + .with_file(SystemPath::new("project/src/shapes/__init__.by"), "")? + .with_file(SystemPath::new("project/src/shapes/circle.by"), CIRCLE)? + .with_file( + SystemPath::new("project/tests/test_circle.by"), + "import math\n\ndef test_pi() -> None:\n assert math.pi > 3\n", + )? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("project/src/shapes/circle.by"), CIRCLE, 1); + + // in `area`'s body + let params = at(&server, "project/src/shapes/circle.by", 5, 10); + let answer = server + .send_request_await::(params) + .expect("the document is the project's"); + + assert_eq!(answer["tests"], json!([])); + assert_eq!(answer["newTest"]["testName"], "test_circle_area"); + assert_eq!(answer["newTest"]["import"], "Circle"); + assert_eq!( + answer["newTest"]["importEdits"], + json!([{ + "range": { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 } }, + "newText": "from shapes.circle import Circle\n", + }]), + "{answer:#}", + ); + Ok(()) +} + +#[test] +fn a_module_and_a_package_are_named_from_their_search_path() -> Result<()> { + let mut server = server()?; + let mut name = |path: &str| { + let uri = server.file_uri(SystemPath::new(path)); + server.send_request_await::(json!({ "uri": uri })) + }; + + assert_eq!( + name("project/src/shapes/circle.by").as_deref(), + Some("shapes.circle") + ); + assert_eq!( + name("project/src/shapes/__init__.by").as_deref(), + Some("shapes") + ); + assert_eq!(name("project/src/shapes").as_deref(), Some("shapes")); + assert_eq!( + name("project/tests/test_circle.by").as_deref(), + Some("tests.test_circle") + ); + // the project root is the top of the module tree, not a module in it + assert_eq!(name("project"), None); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 670116dfc3..acb94d502e 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -463,6 +463,49 @@ it is not a `textDocument/codeLens`, since a client that draws lenses asks for them again only when their own document changes. like the other document requests it takes `textHash` +## tests and what they test + +`by/testSubjects` and `by/subjectTests` answer an editor's *go to test subject* +and *go to test*. a test exercises what it calls, as call hierarchy sees a call: +a function or a method called by name or as an attribute, or a class +constructed. which declarations are tests is pytest's collection, as +`by/testItems` reports it. nothing goes by file names: `test_circle.by` tests +whatever it calls, wherever that is, and the tests of a class are found whatever +their files are called + +`by/testSubjects` takes a position in a test, in a class tests are collected +under, or anywhere else in a module tests are collected from, standing for that +test, every test under the class, or every test in the module. it answers the +project's own functions, methods and classes those tests call, each once, in the +order they are first called: not what is declared in a test module or a +`conftest`, and nothing outside the project. each item has its `name` within its +module with the classes it is declared in (`Circle.area`), its `kind`, its +`module`, and its `uri`, `range` and `selectionRange`. the answer is `null` for +a document no test is collected from + +`by/subjectTests` takes a position in a function, a method or a class, or +anywhere else in a module, which stands for every function and class it +declares; a class stands for itself and its methods. it answers the collected +tests that call it, as items named by their pytest node id after the file +(`TestCircle::test_radius`), by file and then in source order, and `newTest`, +where a test of it would go: `test_` and the last part of its module's name, in +the directory most of the project's test modules are in, or in `tests` under the +project root when there are none. a test module of that name there already is +the one, whether it is `.by` or `.py`; a new one is written in what the subject's +file is. `newTest` says whether that module `exists`, the `module` to import +from, the name to `import` (absent for a module, which a test imports whole), a +`testName` the module does not define already, and for a module that exists the +`importEdits` that import the name into it, none when it does already. `newTest` +is `null` for a test module, or a module with no name to import it by. like the +other document requests both take `textHash`, and with it answer a document the +client has not opened from the file on disk + +`by/moduleName` takes the `uri` of a file or a directory and answers the dotted +name it is imported by, from the search path it is under: a src-layout +project's `src/shapes/circle.by` is `shapes.circle`, and `src/shapes` and +`src/shapes/__init__.by` are both `shapes`. it is `null` for a path no module is +imported from, and for a file that resolution would never reach + ## the whole project's diagnostics with `diagnosticMode` set to `workspace`, the server checks every file in the From 0e017e4efd1196025919caa8713f7ae608e5c1ec Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:44:00 +1000 Subject: [PATCH 07/15] add by/overridableMembers and by/overrideMembers, listing the methods the class at a position inherits and writing the basedpython declarations that override the ones picked, and stop importing names a .by file has implicitly when an annotation is written --- crates/ty_ide/src/inlay_hints.rs | 208 ++- crates/ty_ide/src/lib.rs | 6 + crates/ty_ide/src/override_members.rs | 1584 +++++++++++++++++ .../ty_python_semantic/src/types/display.rs | 35 +- .../src/types/ide_support.rs | 516 +++++- .../src/types/implicit_names.rs | 23 +- .../ty_python_semantic/src/types/typevar.rs | 6 + crates/ty_server/src/document.rs | 1 + crates/ty_server/src/document/range.rs | 4 +- crates/ty_server/src/server/api.rs | 13 + crates/ty_server/src/server/api/requests.rs | 4 + .../api/requests/overridable_members.rs | 145 ++ .../server/api/requests/override_members.rs | 135 ++ crates/ty_server/tests/e2e/main.rs | 1 + .../ty_server/tests/e2e/override_members.rs | 160 ++ docs/basedpython/features/editor.md | 58 + 16 files changed, 2857 insertions(+), 42 deletions(-) create mode 100644 crates/ty_ide/src/override_members.rs create mode 100644 crates/ty_server/src/server/api/requests/overridable_members.rs create mode 100644 crates/ty_server/src/server/api/requests/override_members.rs create mode 100644 crates/ty_server/tests/e2e/override_members.rs diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 7ba5689607..cfc8bf34e1 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -27,14 +27,17 @@ use ty_python_semantic::types::context_params::implicit_context_arguments; use ty_python_semantic::types::ide_support::{ BaseTypeArgument, InferredInvalidations, InferredStateReads, InlayHintCallArgumentDetails, StateRead, WriteSite, hintable_parameter_type, implicit_enum_member_value, - inferred_base_type_arguments, inferred_derived_dependencies, inferred_invalidations, - inferred_override, inferred_parameter_bound, inferred_property_type, inferred_raises, - inferred_return_annotation, inferred_state_reads, inferred_type_param_variance, - inherited_parameter_annotation, inherited_parameter_default, inlay_hint_call_argument_details, - is_composable_function, is_reveal_type_function, is_union_special_form, numeric_promotion, - parameter_stability, trailing_lambda_implicit_parameters, type_parameter_names, + implicit_type_name_module, inferred_base_type_arguments, inferred_derived_dependencies, + inferred_invalidations, inferred_override, inferred_parameter_bound, inferred_property_type, + inferred_raises, inferred_return_annotation, inferred_state_reads, + inferred_type_param_variance, inherited_parameter_annotation, inherited_parameter_default, + inlay_hint_call_argument_details, is_composable_function, is_reveal_type_function, + is_union_special_form, numeric_promotion, parameter_stability, + trailing_lambda_implicit_parameters, type_parameter_names, +}; +use ty_python_semantic::types::{ + BoundTypeVarInstance, DisplaySettings, Type, TypeDetail, TypeDisplayDetails, }; -use ty_python_semantic::types::{DisplaySettings, Type, TypeDetail, TypeDisplayDetails}; use ty_python_semantic::{HasType, SemanticModel, with_display_for_file}; #[derive(Debug, Clone)] @@ -64,14 +67,14 @@ impl InlayHint { named_type_arguments: bool, ) -> Option { let position = expr.range().end(); - let details = type_display_details(context.db, context.file, ty, named_type_arguments); + let details = type_display_details(context.db, context.file, ty, named_type_arguments, &[]); // Filter out repetitive hints like `x: T = T()` if call_matches_name(rhs, &details.label) { return None; } - let spelled = spell_type(context, expr.into(), expr.range().start(), &details); + let spelled = spell_type(context, expr.into(), expr.range().start(), &details, &[]); let text_edits = if spelled.editable && allow_edits { let mut text_edits = vec![InlayHintTextEdit { @@ -1277,15 +1280,18 @@ impl Default for InlayHintSettings { } } -/// `ty` rendered to a string, with the spans of the types that make it up. +/// `ty` rendered to a string, with the spans of the types that make it up. The +/// type parameters in `type_params_in_scope` are written by the names they come with. fn type_display_details<'db>( db: &'db dyn Db, file: ProgramFile<'db>, ty: Type<'db>, named_type_arguments: bool, + type_params_in_scope: &[(BoundTypeVarInstance<'db>, Name)], ) -> TypeDisplayDetails<'db> { let env = ProgramEnvironment::from_file(file); - let settings = DisplaySettings::from_possibly_ambiguous_types(db, &env, [ty]); + let settings = DisplaySettings::from_possibly_ambiguous_types(db, &env, [ty]) + .with_type_params_in_scope(db, type_params_in_scope.iter().cloned()); let settings = if named_type_arguments { settings.with_named_type_arguments() } else { @@ -1314,14 +1320,22 @@ struct SpelledType { /// Whether the annotation can be written at all: it is valid syntax, and /// every type it names can be reached from the file. editable: bool, + /// The imports this spelling added to the ones the context already held, + /// so that a caller dropping the spelling can drop them too. + imported: Vec, } /// Spell `details` for the scope around `scope_node`, importing what it names. +/// +/// A type parameter in `type_params_in_scope` is written by the name it comes +/// with, with nothing imported for it: one that what is being written declares +/// itself, or that a scope around it declares. fn spell_type<'db>( context: InlayHintImportContext<'_, 'db>, scope_node: AnyNodeRef<'_>, scope_offset: TextSize, details: &TypeDisplayDetails<'db>, + type_params_in_scope: &[(BoundTypeVarInstance<'db>, Name)], ) -> SpelledType { let InlayHintImportContext { db, @@ -1363,6 +1377,13 @@ fn spell_type<'db>( // Possibly import the current type and return the qualified name let mut qualified_name = |dynamic_importer: &mut DynamicImporter<'_, 'db>| { + if let Type::TypeVar(typevar) = ty + && type_params_in_scope + .iter() + .any(|(param, _)| param.is_same_occurrence(db, *typevar)) + { + return None; + } let type_definition = ty.definition(db, &env)?; let definition = type_definition.definition()?; @@ -1386,6 +1407,12 @@ fn spell_type<'db>( .as_deref() .unwrap_or(&details.label[start..end]); + // basedpython: a name a `.by` file reaches without importing it, which is + // imported from the module the language takes it from when the file binds + // the name to something else + let implicit_module = + implicit_type_name_module(db, &env, file, definition_name, *ty); + let file = definition.program_file(db); let module = file_to_module(db, file.resolver_file(db))?; @@ -1393,7 +1420,7 @@ fn spell_type<'db>( return None; } - let module_name = module.name(db).as_str(); + let module_name = implicit_module.unwrap_or(module.name(db).as_str()); dynamic_importer.import_symbol( db, @@ -1402,6 +1429,7 @@ fn spell_type<'db>( module_name, definition_name, &details.label[start..end], + implicit_module.is_some(), ) }; @@ -1442,6 +1470,7 @@ fn spell_type<'db>( edit_label, import_edits: dynamic_importer.text_edits(), editable, + imported: dynamic_importer.imported_members, } } @@ -1462,7 +1491,7 @@ pub(crate) fn annotation_for_type<'db>( let mut dynamic_imports = FxHashMap::default(); // a basedpython file writes types in basedpython's own spelling with_display_for_file(db, file.file(db), || { - let details = type_display_details(db, file, ty, false); + let details = type_display_details(db, file, ty, false, &[]); let spelled = spell_type( InlayHintImportContext { db, @@ -1473,6 +1502,7 @@ pub(crate) fn annotation_for_type<'db>( scope_node, scope_offset, &details, + &[], ); spelled .editable @@ -1480,6 +1510,104 @@ pub(crate) fn annotation_for_type<'db>( }) } +/// Types written as annotations in one file, importing what they name. See +/// [`with_annotation_speller`]. +pub(crate) struct AnnotationSpeller<'a, 'db> { + db: &'db dyn Db, + file: ProgramFile<'db>, + importer: &'a Importer<'a>, + dynamic_imports: FxHashMap, + import_edits: Vec, +} + +impl<'db> AnnotationSpeller<'_, 'db> { + /// `ty` written as an annotation for a position in the scope around + /// `scope_node`, or `None` when no annotation the file could hold says it. + /// A type parameter in `type_params_in_scope` is written by the name it comes + /// with, with nothing imported for it: see [`spell_type`]. + pub(crate) fn spell( + &mut self, + scope_node: AnyNodeRef<'_>, + scope_offset: TextSize, + ty: Type<'db>, + type_params_in_scope: &[(BoundTypeVarInstance<'db>, Name)], + ) -> Option { + let db = self.db; + let file = self.file; + // a basedpython file writes types in basedpython's own spelling + let spelled = with_display_for_file(db, file.file(db), || { + let details = type_display_details(db, file, ty, false, type_params_in_scope); + spell_type( + InlayHintImportContext { + db, + file, + importer: self.importer, + dynamic_imports: &mut self.dynamic_imports, + }, + scope_node, + scope_offset, + &details, + type_params_in_scope, + ) + }); + if !spelled.editable { + // an import is remembered so that a second annotation naming the same + // type does not import it again. one remembered for an annotation + // that is not written would leave the second without its import + for member in &spelled.imported { + self.dynamic_imports.remove(member); + } + return None; + } + self.import_edits.extend(spelled.import_edits); + Some(spelled.edit_label) + } + + /// `name`, which `module` binds at its top level, or the module itself when there is no + /// `name`, written for a position in the scope around `scope_node` and imported there: by + /// its name, or through its module when the file binds the name to something else. + pub(crate) fn reference( + &mut self, + scope_node: AnyNodeRef<'_>, + scope_offset: TextSize, + module: &str, + name: Option<&str>, + ) -> String { + let mut importer = DynamicImporter::new( + self.importer, + scope_node, + scope_offset, + &mut self.dynamic_imports, + ); + let text = importer.import_name(module, name); + let edits = importer.text_edits(); + self.import_edits.extend(edits); + text + } +} + +/// Runs `spell` with an [`AnnotationSpeller`] for `file`, and answers what it +/// answered together with the imports the annotations it wrote need, each once. +pub(crate) fn with_annotation_speller<'db, R>( + db: &'db dyn Db, + file: ProgramFile<'db>, + spell: impl FnOnce(&mut AnnotationSpeller<'_, 'db>) -> R, +) -> (R, Vec) { + let ast = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let stylist = Stylist::from_tokens(ast.tokens(), source.as_str()); + let importer = Importer::new(db, &stylist, file, source.as_str(), &ast); + let mut speller = AnnotationSpeller { + db, + file, + importer: &importer, + dynamic_imports: FxHashMap::default(), + import_edits: Vec::new(), + }; + let answer = spell(&mut speller); + (answer, speller.import_edits) +} + struct InlayHintImportContext<'a, 'db> { db: &'db dyn Db, file: ProgramFile<'db>, @@ -2900,6 +3028,11 @@ impl<'a, 'db> DynamicImporter<'a, 'db> { /// Attempts to import a given symbol. /// If the symbol in the text edit needs to be qualified, we return the qualified symbol text. + /// + /// basedpython: `implicit` says the file reaches the symbol by its name without importing + /// it, as a `.by` file does `Iterator`, so that it needs no import unless the file binds the + /// name to something else. + #[expect(clippy::too_many_arguments)] fn import_symbol( &mut self, db: &'db dyn Db, @@ -2908,6 +3041,7 @@ impl<'a, 'db> DynamicImporter<'a, 'db> { module_name: &str, symbol_name: &str, label_text: &str, + implicit: bool, ) -> Option { use std::collections::hash_map::Entry; @@ -2929,6 +3063,8 @@ impl<'a, 'db> DynamicImporter<'a, 'db> { // so we need to qualify this so we don't reference the // in scope member. is_possibly_qualified_name = true; + } else if implicit { + return None; } let key = DynamicallyImportedMember { @@ -2949,9 +3085,42 @@ impl<'a, 'db> DynamicImporter<'a, 'db> { self.imported_members.push(key); - qualified_symbol_text(action).map(str::to_string) + // the text the import makes refer to it: the name, or through its module + // when the import has to qualify it + Some(action.symbol_text().to_string()) + } + Entry::Occupied(entry) => Some(entry.get().symbol_text().to_string()), + } + } + + /// Imports `symbol_name` from `module_name`, or the module itself when there is no + /// `symbol_name`, and returns the text that refers to it. A name the file binds already is + /// taken to be something else, which is written through its module instead. + fn import_name(&mut self, module_name: &str, symbol_name: Option<&str>) -> String { + use std::collections::hash_map::Entry; + + let members = self.members.get_or_insert_with(|| { + self.importer + .members_in_scope_at(self.scope_node, self.scope_offset) + }); + let key = DynamicallyImportedMember { + module: module_name.to_string(), + name: symbol_name.unwrap_or_default().to_string(), + }; + match self.dynamic_imports.entry(key.clone()) { + Entry::Vacant(entry) => { + let request = match symbol_name { + None => ImportRequest::module(module_name), + Some(name) if members.find_member(name).is_some() => { + ImportRequest::import(module_name, name).force() + } + Some(name) => ImportRequest::import_from(module_name, name), + }; + let action = entry.insert(self.importer.import(request, members)); + self.imported_members.push(key); + action.symbol_text().to_string() } - Entry::Occupied(entry) => qualified_symbol_text(entry.get()).map(str::to_string), + Entry::Occupied(entry) => entry.get().symbol_text().to_string(), } } @@ -2972,15 +3141,6 @@ impl<'a, 'db> DynamicImporter<'a, 'db> { } } -/// If the import action requires qualifying the symbol (e.g. `import foo` instead of -/// `from foo import A`), returns the qualified symbol text. Otherwise returns `None`. -fn qualified_symbol_text(import_action: &ImportAction) -> Option<&str> { - if import_action.import().is_some() { - return None; - } - Some(import_action.symbol_text()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index f670520980..293bfd6f8b 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -29,6 +29,7 @@ mod injection; mod inlay_hints; mod markup; mod module_rename; +mod override_members; mod refactor; mod references; mod rename; @@ -83,6 +84,10 @@ pub use markup::MarkupKind; pub use module_rename::{ FileMove, ModuleRenameEdits, SkipReason, SkippedImport, module_rename_edits, }; +pub use override_members::{ + ClassMembersToOverride, MemberOverrides, MemberToOverride, members_to_override, + override_members, +}; pub use refactor::{Refactor, RefactorKind, RefactorOffer, refactor, refactors}; pub use references::ReferencesMode; pub use rename::{PreparedRename, can_rename, invalid_new_name, prepare_rename, rename}; @@ -99,6 +104,7 @@ pub use syntax_outline::{ syntax_outline, }; pub use test_links::{NewTest, SubjectTests, TestLink, subject_tests, test_subjects}; +pub use ty_python_semantic::types::ide_support::InheritedMethodKind; pub use type_hierarchy::{ TypeHierarchyItem, prepare_type_hierarchy, type_hierarchy_subtypes, type_hierarchy_supertypes, }; diff --git a/crates/ty_ide/src/override_members.rs b/crates/ty_ide/src/override_members.rs new file mode 100644 index 0000000000..acdd8134b0 --- /dev/null +++ b/crates/ty_ide/src/override_members.rs @@ -0,0 +1,1584 @@ +//! Override and implement members: which methods the class at a position could +//! override, which of them it has to implement, and the declarations that do it, +//! written the way basedpython writes them +//! +//! an editor asks twice: once for the members, to let its user pick +//! ([`members_to_override`]), and once for the declarations of the ones picked +//! ([`override_members`]). both answers come from one walk up the MRO, +//! [`inherited_methods`], and the members are listed with the declaration that +//! would be written for each, so that what is offered is what is written +//! +//! a declaration is the `def` header an override is written with, under the +//! `override` modifier: the method's parameters and return type as the class +//! inherits them, a generic superclass's type parameters filled in the way the +//! class fills them, each type spelled for the file and imported when the file +//! does not reach it already. its body calls the method it overrides, through +//! `super` (a static method, through the superclass's name), or for a method +//! that is abstract where the class inherits it, and so has nothing to call, is +//! `...` +//! +//! a parameter type that cannot be spelled from the file is left out rather +//! than written wrong: a basedpython override's unannotated parameter takes the +//! annotation of the parameter it overrides. a default that is a value is left +//! out of an implementation for the same reason. one that is an expression is +//! not inherited, and is written as its source wrote it, with each name it +//! reads written so that it means what it does where the method is declared + +use std::fmt::Write as _; + +use itertools::Itertools; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_diagnostics::Edit; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt}; +use ruff_python_codegen::Stylist; +use ruff_python_trivia::indentation_at_offset; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; +use ty_module_resolver::file_to_module; +use ty_python_core::ProgramFile; +use ty_python_semantic::ResolvedDefinition; +use ty_python_semantic::types::ide_support::{ + InheritedDefault, InheritedExpression, InheritedMethod, InheritedMethodKind, + InheritedParameterKind, InheritedSignature, definitions_for_name_in_class_body, + inherited_methods, +}; +use ty_python_semantic::types::{BoundTypeVarInstance, Type}; +use ty_python_semantic::{HasType, SemanticModel}; + +use crate::Db; +use crate::inlay_hints::{AnnotationSpeller, with_annotation_speller}; + +/// the members the class at a position inherits, for a user to pick from +#[derive(Debug, Clone)] +pub struct ClassMembersToOverride { + /// the class's name + pub class_name: Name, + /// in MRO order of the superclass each is inherited from + pub members: Vec, +} + +/// a method a class could override +#[derive(Debug, Clone)] +pub struct MemberToOverride { + pub name: Name, + /// the superclass it is inherited from + pub superclass: Name, + /// the module that declares the superclass + pub superclass_module: Option, + /// whether it is abstract where it is inherited from, so that the class has + /// to implement it + pub is_abstract: bool, + pub kind: InheritedMethodKind, + /// the declaration that overrides it, from its name on: `f(self, x: int) -> str`. + /// an overloaded method's first overload + pub signature: String, +} + +/// the declarations overriding some of a class's members +#[derive(Debug, Clone)] +pub struct MemberOverrides { + /// the edits that write them, with the imports they need + pub edits: Vec, + /// the body of the first declaration, in the text the edits leave behind: + /// what its user replaces first + pub selection: TextRange, +} + +/// the members the class at `offset` could override, in MRO order of the +/// superclass each is inherited from +/// +/// the class is the innermost one whose definition holds `offset`: its header, +/// its body, or an indented line after its last statement, which is where a new +/// member would go. `None` when there is no class there, and in a file that is +/// not basedpython source, whose overrides are written another way +pub fn members_to_override( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option { + let context = Context::new(db, file)?; + let class = context.class_at(offset)?; + let methods = context.inherited_methods(class); + let (members, _) = with_annotation_speller(db, file, |speller| { + let writer = Writer::new(&context, speller, class, offset); + methods + .iter() + .map(|method| MemberToOverride { + name: method.name.clone(), + superclass: method.superclass_name.clone(), + superclass_module: method.superclass_module.as_ref().map(ToString::to_string), + is_abstract: method.is_abstract, + kind: method.kind, + signature: writer.signature(method), + }) + .collect::>() + }); + Some(ClassMembersToOverride { + class_name: class.name.id.clone(), + members, + }) +} + +/// the declarations overriding the members `names` of the class at `offset`, as +/// [`members_to_override`] lists them, in the order it lists them +/// +/// they go where the class's user is: after the statement of the class body +/// that `offset` is in or follows, or at the end of the body when `offset` is in +/// the class's header. a body that is only a placeholder `pass` or `...` is +/// replaced. `None` when there is no class at `offset`, or none of `names` is a +/// member it could override +pub fn override_members( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, + names: &[Name], +) -> Option { + let context = Context::new(db, file)?; + let class = context.class_at(offset)?; + let methods: Vec<_> = context + .inherited_methods(class) + .into_iter() + .filter(|method| names.contains(&method.name)) + .collect(); + if methods.is_empty() { + return None; + } + let place = context.insertion(class, offset); + let (declarations, imports) = with_annotation_speller(db, file, |speller| { + let writer = Writer::new(&context, speller, class, place.range.start()); + methods + .iter() + .map(|method| writer.declaration(method)) + .collect::>() + }); + + let line_ending = context.stylist.line_ending().as_str(); + let mut text = place.leading.clone(); + let mut selection = None; + for (index, declaration) in declarations.iter().enumerate() { + if index > 0 { + text.push_str(line_ending); + text.push_str(line_ending); + } + for (line_index, line) in declaration.lines.iter().enumerate() { + if line_index > 0 { + text.push_str(line_ending); + } + if index > 0 || line_index > 0 || place.indent_first_line { + text.push_str(&place.indent); + } + if index == 0 && line_index + 1 == declaration.lines.len() { + selection = Some(TextRange::at( + text.text_len() + declaration.body_offset, + declaration.body_len, + )); + } + text.push_str(line); + } + } + text.push_str(&place.trailing); + + // the selection is in the text the edits leave behind, so it moves by what + // every import written above the declarations adds + let (added, removed) = imports + .iter() + .filter(|import| import.range.end() <= place.range.start()) + .fold( + (TextSize::default(), TextSize::default()), + |(added, removed), import| { + ( + added + import.new_text.text_len(), + removed + import.range.len(), + ) + }, + ); + let selection = selection?; + let selection = TextRange::at( + place.range.start() + selection.start() + added - removed, + selection.len(), + ); + + let mut edits = vec![Edit::range_replacement(text, place.range)]; + edits.extend( + imports + .into_iter() + .map(|import| Edit::range_replacement(import.new_text, import.range)), + ); + Some(MemberOverrides { edits, selection }) +} + +/// what the two questions read about the file they are asked of +struct Context<'db> { + db: &'db dyn Db, + model: SemanticModel<'db>, + parsed: ParsedModuleRef, + source: ruff_db::source::SourceText, + stylist: Stylist<'static>, +} + +impl<'db> Context<'db> { + fn new(db: &'db dyn Db, file: ProgramFile<'db>) -> Option { + if !file.file(db).source_type(db).is_basedpython() { + return None; + } + let parsed = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let stylist = Stylist::from_tokens(parsed.tokens(), source.as_str()).into_owned(); + Some(Self { + db, + model: SemanticModel::new(db, file), + parsed, + source, + stylist, + }) + } + + fn source(&self) -> &str { + self.source.as_str() + } + + /// the innermost class whose definition holds `offset`, or which `offset` + /// follows on an indented line with nothing between them but blank lines and + /// comments: the class body an editor's caret is in when it sits below the + /// last member, where python's own syntax tree has already ended the class + fn class_at(&self, offset: TextSize) -> Option<&ast::StmtClassDef> { + let mut classes = Classes::default(); + classes.visit_body(&self.parsed.syntax().body); + let source = self.source(); + let column = offset - source.line_start(offset); + classes + .found + .into_iter() + .filter(|class| { + if class.range().contains_inclusive(offset) { + return true; + } + if class.end() > offset { + return false; + } + let between = &source[TextRange::new(class.end(), offset)]; + let only_trivia = between + .lines() + .all(|line| line.trim().is_empty() || line.trim_start().starts_with('#')); + let class_column = class.start() - source.line_start(class.start()); + only_trivia && column > class_column + }) + .max_by_key(Ranged::start) + } + + fn inherited_methods(&self, class: &ast::StmtClassDef) -> Vec> { + let Some(Type::ClassLiteral(literal)) = class.inferred_type(&self.model) else { + return Vec::new(); + }; + inherited_methods(self.db, &self.model.program_environment(), literal) + } + + /// the type parameters `class` declares + fn class_type_params(&self, class: &ast::StmtClassDef) -> Vec> { + let Some(Type::ClassLiteral(literal)) = class.inferred_type(&self.model) else { + return Vec::new(); + }; + literal + .generic_context(self.db) + .into_iter() + .flat_map(|generic_context| generic_context.variables(self.db)) + .collect() + } + + /// where the declarations go in `class`, for a caret at `offset` + fn insertion(&self, class: &ast::StmtClassDef, offset: TextSize) -> Insertion { + let source = self.source(); + let line_ending = self.stylist.line_ending().as_str(); + let indent_unit = self.stylist.indentation().as_str(); + let class_indent = &source[TextRange::new(source.line_start(class.start()), class.start())]; + + // a class with no body: one being written, whose header ends in a colon with nothing + // indented after it yet, or a basedpython class declared without a body at all + let (Some(first), Some(last)) = (class.body.first(), class.body.last()) else { + let header = source[..class.end().to_usize()].trim_end(); + let after = &source[class.end().to_usize()..]; + let (at, leading) = if header.ends_with(':') { + (header.text_len(), line_ending.to_string()) + } else if let Some(colon) = after.trim_start().strip_prefix(':') { + ( + source.text_len() - colon.text_len(), + line_ending.to_string(), + ) + } else { + (class.end(), format!(":{line_ending}")) + }; + return Insertion { + range: TextRange::empty(at), + leading, + indent: format!("{class_indent}{indent_unit}"), + indent_first_line: true, + trailing: String::new(), + }; + }; + + if let [only] = class.body.as_slice() + && is_placeholder(only) + { + return match indentation_at_offset(only.start(), source) { + // `class A(B):` with the placeholder on a line of its own + Some(indent) => Insertion { + range: only.range(), + leading: String::new(), + indent: indent.to_string(), + indent_first_line: false, + trailing: String::new(), + }, + // `class A(B): ...`, whose placeholder is replaced from the colon + None => { + let before = source[..only.start().to_usize()].trim_end(); + Insertion { + range: TextRange::new(before.text_len(), only.end()), + leading: line_ending.to_string(), + indent: format!("{class_indent}{indent_unit}"), + indent_first_line: true, + trailing: String::new(), + } + } + }; + } + + // `class B(A): x = 1`, whose body moves onto a line of its own with the declarations + // below it + let Some(indent) = indentation_at_offset(first.start(), source) else { + let before = source[..first.start().to_usize()].trim_end(); + let end = source.line_end(last.end()); + let indent = format!("{class_indent}{indent_unit}"); + return Insertion { + range: TextRange::new(before.text_len(), end), + leading: format!( + "{line_ending}{indent}{}{line_ending}{line_ending}", + &source[TextRange::new(first.start(), end)] + ), + indent, + indent_first_line: true, + trailing: String::new(), + }; + }; + let indent = indent.to_string(); + // the statement the caret is in or follows; in the header, the body's last + let anchor = class + .body + .iter() + .rev() + .find(|statement| statement.start() <= offset) + .filter(|_| offset >= first.start()) + .unwrap_or(last); + let mut at = source.line_end(anchor.end()); + // a caret below the anchor goes below the comments between them too: they are what + // the user was writing after the anchor + let caret_line = source.line_start(offset); + let mut line = source.full_line_end(at); + while line < caret_line { + let end = source.line_end(line); + if !source[TextRange::new(line, end)].trim().is_empty() { + at = end; + } + line = source.full_line_end(line); + } + // a statement right below the anchor is kept a blank line away + let next_line_is_code = class + .body + .iter() + .any(|statement| statement.start() > anchor.end()) + && !source[TextRange::new(at, source.full_line_end(source.full_line_end(at)))] + .trim() + .is_empty(); + Insertion { + range: TextRange::empty(at), + leading: format!("{line_ending}{line_ending}"), + indent, + indent_first_line: true, + trailing: if next_line_is_code { + line_ending.to_string() + } else { + String::new() + }, + } + } +} + +/// a body statement that stands in for no body: `pass` or `...` +fn is_placeholder(statement: &Stmt) -> bool { + match statement { + Stmt::Pass(_) => true, + Stmt::Expr(expression) => matches!(*expression.value, Expr::EllipsisLiteral(_)), + _ => false, + } +} + +/// where the declarations are written, and what surrounds them there +struct Insertion { + /// what they replace: empty to insert them + range: TextRange, + /// what goes before the first + leading: String, + /// what each of their lines starts with + indent: String, + /// whether the first line starts with [`Self::indent`] too, which it does + /// not when it replaces a statement that is indented already + indent_first_line: bool, + /// what goes after the last + trailing: String, +} + +#[derive(Default)] +struct Classes<'a> { + found: Vec<&'a ast::StmtClassDef>, +} + +impl<'a> SourceOrderVisitor<'a> for Classes<'a> { + fn visit_stmt(&mut self, statement: &'a Stmt) { + if let Stmt::ClassDef(class) = statement { + self.found.push(class); + } + source_order::walk_stmt(self, statement); + } + + fn visit_expr(&mut self, _expression: &'a Expr) { + // a class is a statement, and no statement is inside an expression + } +} + +/// one declaration, a line at a time, indented from where the declaration is +struct Declaration { + /// the last is its body + lines: Vec, + /// where the body is in its line, after its indentation + body_offset: TextSize, + body_len: TextSize, +} + +/// writes declarations for a position in one class +struct Writer<'w, 'a, 'db> { + context: &'w Context<'db>, + speller: std::cell::RefCell<&'w mut AnnotationSpeller<'a, 'db>>, + class: &'w ast::StmtClassDef, + scope_node: AnyNodeRef<'w>, + offset: TextSize, + /// the type parameters the class declares, which its body names as they are + class_type_params: Vec<(BoundTypeVarInstance<'db>, Name)>, + /// one level of indentation, as the file writes it + indent_unit: &'w str, +} + +impl<'w, 'a, 'db> Writer<'w, 'a, 'db> { + fn new( + context: &'w Context<'db>, + speller: &'w mut AnnotationSpeller<'a, 'db>, + class: &'w ast::StmtClassDef, + offset: TextSize, + ) -> Self { + let db = context.db; + Self { + context, + speller: std::cell::RefCell::new(speller), + class, + scope_node: AnyNodeRef::from(class), + offset, + class_type_params: context + .class_type_params(class) + .into_iter() + .map(|typevar| (typevar, typevar.name(db).clone())) + .collect(), + indent_unit: context.stylist.indentation().as_str(), + } + } + + /// `ty` as an annotation, naming the type parameters in `type_params` and + /// the class's own as they are + fn spell( + &self, + ty: Type<'db>, + type_params: &[(BoundTypeVarInstance<'db>, Name)], + ) -> Option { + let in_scope: Vec<(BoundTypeVarInstance<'db>, Name)> = type_params + .iter() + .chain(&self.class_type_params) + .cloned() + .collect(); + self.speller + .borrow_mut() + .spell(self.scope_node, self.offset, ty, &in_scope) + } + + /// what [`MemberToOverride::signature`] says + fn signature(&self, method: &InheritedMethod<'db>) -> String { + let Some(signature) = method.signatures.first() else { + return format!("{}()", method.name); + }; + let mut text = method.name.to_string(); + self.write_signature(&mut text, method, signature, true); + text + } + + fn declaration(&self, method: &InheritedMethod<'db>) -> Declaration { + let mut lines = Vec::new(); + let body = if method.is_overloaded { + // each overload is a declaration with no body, and the run ends in + // an implementation that accepts whatever any of them does + for signature in &method.signatures { + let mut line = modifiers(method); + line.push_str(&method.name); + self.write_signature(&mut line, method, signature, true); + lines.push(line); + } + let receiver = receiver(method, method.signatures.first()); + let mut line = modifiers(method); + let _ = write!( + line, + "{}({}*args, **kwargs):", + method.name, + receiver.map(|name| format!("{name}, ")).unwrap_or_default() + ); + lines.push(line); + let returns_none = method + .signatures + .iter() + .all(|signature| signature.returns_none); + self.body(method, "*args, **kwargs", returns_none) + } else { + let signature = method.signatures.first(); + let mut line = modifiers(method); + line.push_str(&method.name); + if let Some(signature) = signature { + self.write_signature(&mut line, method, signature, false); + } else { + line.push_str("()"); + } + line.push(':'); + lines.push(line); + let arguments = signature + .map(|signature| arguments(method, signature)) + .unwrap_or_default(); + let returns_none = signature.is_some_and(|signature| signature.returns_none); + self.body(method, &arguments, returns_none) + }; + let body_len = body.text_len(); + lines.push(format!("{}{body}", self.indent_unit)); + Declaration { + body_offset: self.indent_unit.text_len(), + body_len, + lines, + } + } + + /// the type parameters, parameters and return type, after the name. a + /// `declaration` is a `def` with no body, an overload, which writes every + /// default it has; an implementation inherits the ones that are values + fn write_signature( + &self, + text: &mut String, + method: &InheritedMethod<'db>, + signature: &InheritedSignature<'db>, + declaration: bool, + ) { + // a method's type parameter named like one of the class's would hide it from the + // annotations that name the class's, so it is written under a name of its own + let mut type_params: Vec<(BoundTypeVarInstance<'db>, Name)> = Vec::new(); + for param in &signature.type_params { + let taken = |name: &str| { + self.class_type_params + .iter() + .chain(&type_params) + .any(|(_, taken)| taken == name) + }; + let name = if taken(¶m.name) { + // as many names are taken as there are type parameters, so one of that many + // and one more suffixes is free + (2..=self.class_type_params.len() + type_params.len() + 2) + .map(|suffix| Name::new(format!("{}{suffix}", param.name))) + .find(|candidate| !taken(candidate)) + .unwrap_or_else(|| param.name.clone()) + } else { + param.name.clone() + }; + type_params.push((param.typevar, name)); + } + if !signature.type_params.is_empty() { + let params = signature + .type_params + .iter() + .zip(&type_params) + .map(|(param, (_, name))| { + let mut written = name.to_string(); + if let Some(bound) = param.bound + && let Some(bound) = self.spell(bound, &type_params) + { + let _ = write!(written, ": {bound}"); + } else if !param.constraints.is_empty() + && let Some(constraints) = param + .constraints + .iter() + .map(|constraint| self.spell(*constraint, &type_params)) + .collect::>>() + { + let _ = write!(written, ": ({})", constraints.join(", ")); + } + written + }) + .join(", "); + let _ = write!(text, "[{params}]"); + } + + let has_receiver = method.kind != InheritedMethodKind::Static; + let has_variadic = signature + .parameters + .iter() + .any(|parameter| parameter.kind == InheritedParameterKind::Variadic); + // a receiver only python's positional-only marker covers is called the same either way, + // so it gets no marker of its own + let last_positional_only = signature + .parameters + .iter() + .rposition(|parameter| parameter.kind == InheritedParameterKind::PositionalOnly) + .filter(|&index| !(has_receiver && index == 0)); + let mut parts = Vec::new(); + let mut wrote_star = false; + for (index, parameter) in signature.parameters.iter().enumerate() { + if parameter.kind == InheritedParameterKind::KeywordOnly && !has_variadic && !wrote_star + { + parts.push("*".to_string()); + wrote_star = true; + } + let mut part = match parameter.kind { + InheritedParameterKind::Variadic => format!("*{}", parameter.name), + InheritedParameterKind::KeywordVariadic => format!("**{}", parameter.name), + _ => parameter.name.to_string(), + }; + // the receiver's type is implicit + let annotation = (!(has_receiver && index == 0)) + .then_some(parameter.annotation) + .flatten() + .and_then(|annotation| self.spell(annotation, &type_params)); + if let Some(annotation) = &annotation { + let _ = write!(part, ": {annotation}"); + } + let default = match ¶meter.default { + Some(InheritedDefault::Value(value)) if declaration => Some(value.clone()), + Some(InheritedDefault::Expression(expression)) => self.expression(expression), + _ => None, + }; + if let Some(default) = default { + // the spacing python style asks for + let separator = if annotation.is_some() { " = " } else { "=" }; + let _ = write!(part, "{separator}{default}"); + } + parts.push(part); + if Some(index) == last_positional_only { + parts.push("/".to_string()); + } + } + let _ = write!(text, "({})", parts.join(", ")); + + if let Some(returns) = signature.returns + && let Some(returns) = self.spell(returns, &type_params) + { + let _ = write!(text, " -> {returns}"); + } + } + + /// `expression` written in the class, each name it reads written so that it + /// means there what it means where the expression comes from. `None` when a + /// name cannot be: one bound only in a class or a function + fn expression(&self, expression: &InheritedExpression<'db>) -> Option { + let mut text = expression.text.clone(); + for name in expression.names.iter().rev() { + let here = + definitions_for_name_in_class_body(&self.context.model, self.class, &name.name); + let same = here.len() == name.definitions.len() + && here + .iter() + .all(|definition| name.definitions.contains(definition)); + if same { + continue; + } + let reference = self.reference(&name.definitions)?; + text.replace_range(name.range.to_std_range(), &reference); + } + Some(text) + } + + /// what refers to `definitions` from the class, importing it: a module, or + /// a name another module binds at its top level + fn reference(&self, definitions: &[ResolvedDefinition<'db>]) -> Option { + let db = self.context.db; + let here = self.context.model.program_file(); + let (file, name) = match definitions { + [ResolvedDefinition::Module(module)] => (*module, None), + [ResolvedDefinition::Definition(first), ..] => { + let file = first.program_file(db); + let name = first.name(db)?; + let importable = definitions.iter().all(|definition| { + matches!( + definition, + ResolvedDefinition::Definition(definition) + if definition.program_file(db) == file + && definition.file_scope(db).is_global() + && definition.name(db).as_deref() == Some(name.as_str()) + ) + }); + if !importable { + return None; + } + (file, Some(name)) + } + _ => return None, + }; + // a name of this file means something else here, and importing the file into itself + // would not reach it + if file == here { + return None; + } + let module = file_to_module(db, file.resolver_file(db))?; + Some(self.speller.borrow_mut().reference( + self.scope_node, + self.offset, + module.name(db).as_str(), + name.as_deref(), + )) + } + + /// the body: a call to the method being overridden, passing `arguments` on, + /// or `...` when there is nothing to call + /// + /// an abstract method has no implementation to call. a static method has no + /// `super` to call it through, since `super` needs the class or instance a + /// call was made on, so it is called on the superclass by name, or not at + /// all when the file cannot name it + fn body(&self, method: &InheritedMethod<'db>, arguments: &str, returns_none: bool) -> String { + if method.is_abstract { + return "...".to_string(); + } + let receiver = if method.kind == InheritedMethodKind::Static { + match self.spell(method.superclass_instance, &[]) { + Some(superclass) => superclass, + None => return "...".to_string(), + } + } else { + "super".to_string() + }; + let mut call = format!("{receiver}.{}({arguments})", method.name); + if method.is_async { + call = format!("await {call}"); + } + if returns_none { + call + } else { + format!("return {call}") + } + } +} + +/// what comes before the name: the modifiers and `def` +fn modifiers(method: &InheritedMethod<'_>) -> String { + let mut text = String::new(); + if method.is_protected { + text.push_str("protected "); + } + text.push_str("override "); + match method.kind { + InheritedMethodKind::Instance => {} + InheritedMethodKind::Class => text.push_str("class "), + InheritedMethodKind::Static => text.push_str("static "), + } + if method.is_async { + text.push_str("async "); + } + text.push_str("def "); + text +} + +/// the name of the parameter the method is called on: `self`, `cls`. `None` +/// for a static method, which is called on nothing +fn receiver<'s>( + method: &InheritedMethod<'_>, + signature: Option<&'s InheritedSignature<'_>>, +) -> Option<&'s Name> { + if method.kind == InheritedMethodKind::Static { + return None; + } + signature? + .parameters + .first() + .map(|parameter| ¶meter.name) +} + +/// what a call passing `signature`'s parameters on writes in its +/// parentheses +fn arguments(method: &InheritedMethod<'_>, signature: &InheritedSignature<'_>) -> String { + let skip = usize::from(method.kind != InheritedMethodKind::Static); + signature + .parameters + .iter() + .skip(skip) + .map(|parameter| match parameter.kind { + InheritedParameterKind::PositionalOnly + | InheritedParameterKind::PositionalOrKeyword => parameter.name.to_string(), + InheritedParameterKind::Variadic => format!("*{}", parameter.name), + InheritedParameterKind::KeywordOnly => { + format!("{name}={name}", name = parameter.name) + } + InheritedParameterKind::KeywordVariadic => format!("**{}", parameter.name), + }) + .join(", ") +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + use ruff_python_ast::name::Name; + use ruff_text_size::Ranged; + + use crate::tests::{CursorTest, cursor_test}; + + impl CursorTest { + /// one line per member: `Superclass.signature`, `(abstract)` when it is + fn members_to_override(&self) -> String { + let Some(members) = super::members_to_override( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + ) else { + return "None".to_string(); + }; + let mut lines = vec![format!("class {}", members.class_name)]; + lines.extend(members.members.iter().map(|member| { + format!( + "{}.{}{}", + member.superclass, + member.signature, + if member.is_abstract { + " (abstract)" + } else { + "" + } + ) + })); + lines.join("\n") + } + + /// the file once the overrides of `names` are written, with the + /// selection marked `…` + fn override_members(&self, names: &[&str]) -> String { + let names: Vec = names.iter().map(|name| Name::new(*name)).collect(); + let Some(overrides) = super::override_members( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + &names, + ) else { + return "None".to_string(); + }; + let source = ruff_db::source::source_text(&self.db, self.cursor.file); + let mut edits = overrides.edits.clone(); + edits.sort_by_key(|edit| std::cmp::Reverse(edit.start())); + let mut text = source.as_str().to_string(); + for edit in &edits { + text.replace_range( + edit.range().to_std_range(), + edit.content().unwrap_or_default(), + ); + } + let start = overrides.selection.start().to_usize(); + let end = overrides.selection.end().to_usize(); + format!( + "{}{}{}", + &text[..start], + &text[start..end], + &text[end..] + ) + } + } + + fn basedpython(source: &str) -> CursorTest { + CursorTest::builder().source("main.by", source).build() + } + + /// what [`CursorTest::members_to_override`] lists, less `object`'s members, + /// which every class inherits + fn without_object(listed: &str) -> String { + listed + .lines() + .filter(|line| !line.starts_with("object.")) + .collect::>() + .join("\n") + } + + #[test] + fn lists_what_a_class_inherits_with_the_declaration_of_each() { + let test = basedpython( + " +from abc import ABC, abstractmethod + +class A(ABC): + @abstractmethod + def f(self, x: int) -> str: ... + + def g(self, a: int = 1, *args: str, k: list[int] = [], **kw: object) -> int: + return a + +class B(A): + +", + ); + assert_snapshot!(test.members_to_override(), @r#" + class B + A.f(self, x: int) -> str (abstract) + A.g(self, a: int = 1, *args: str, k: list[int] = [], **kw: object) -> int + object.__setattr__(self, name: str, value: Any, /) + object.__delattr__(self, name: str, /) + object.__eq__(self, value: object, /) -> bool + object.__ne__(self, value: object, /) -> bool + object.__str__(self) -> str + object.__repr__(self) -> str + object.__hash__(self) -> int + object.__format__(self, format_spec: "", /) -> str + object.__getattribute__(self, name: str, /) -> Any + object.__sizeof__(self) -> int + object.__reduce__(self) -> str | (*: Any) + object.__reduce_ex__(self, protocol: SupportsIndex, /) -> str | (*: Any) + object.__getstate__(self) -> object + object.__dir__(self) -> Iterable[str] + object.__subclasshook__(cls, subclass: type, /) -> bool + "#); + } + + /// what the class defines itself, a `final` method, a `private` one and the + /// constructors are not offered + #[test] + fn leaves_out_what_cannot_be_overridden() { + let test = basedpython( + " +class A: + def kept(self): ... + def mine(self): ... + final def fixed(self): ... + private def hidden(self): ... + init(self): ... + +class B(A): + def mine(self): ... + +", + ); + assert_snapshot!(without_object(&test.members_to_override()), @" + class B + A.kept(self) + "); + } + + /// a method's nearest declaration decides whether it is still abstract + #[test] + fn an_abstract_method_implemented_further_down_is_not_abstract() { + let test = basedpython( + " +from typing import Protocol + +class Shape(Protocol): + def area(self) -> float: ... + def name(self) -> str: ... + +class Square(Shape): + override def area(self) -> float: + return 1.0 + +class Small(Square): + +", + ); + assert_snapshot!(without_object(&test.members_to_override()), @" + class Small + Square.area(self) -> float + Shape.name(self) -> str (abstract) + "); + } + + #[test] + fn no_class_at_the_position_is_none() { + let test = basedpython( + " +class A: + def f(self): ... + +def g(): + pass +", + ); + assert_snapshot!(test.members_to_override(), @"None"); + } + + /// python's overrides are written with decorators, not basedpython's keyword + #[test] + fn a_python_file_is_none() { + let test = cursor_test( + " +class A: + def f(self): ... + +class B(A): + pass +", + ); + assert_snapshot!(test.members_to_override(), @"None"); + } + + #[test] + fn writes_an_override_after_the_statement_the_caret_follows() { + let test = basedpython( + " +class A: + def f(self, x: int) -> str: + return '' + + def g(self) -> None: ... + +class B(A): + y = 1 + +", + ); + assert_snapshot!(test.override_members(&["f", "g"]), @" + + class A: + def f(self, x: int) -> str: + return '' + + def g(self) -> None: ... + + class B(A): + y = 1 + + override def f(self, x: int) -> str: + return super.f(x) + + override def g(self): + super.g() + "); + } + + /// a caret in the middle of the body puts the declarations there, a blank + /// line away from the statement below + #[test] + fn writes_between_two_statements() { + let test = basedpython( + " +class A: + def f(self): ... + +class B(A): + x = 1 + y = 2 +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + class A: + def f(self): ... + + class B(A): + x = 1 + + override def f(self): + super.f() + + y = 2 + "); + } + + #[test] + fn replaces_a_placeholder_body() { + let test = basedpython( + " +class A: + def f(self): ... + +class B(A): + pass +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + class A: + def f(self): ... + + class B(A): + override def f(self): + super.f() + "); + } + + #[test] + fn gives_a_bodyless_class_a_body() { + let test = basedpython( + " +class A: + def f(self): ... + +class B(A) +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + class A: + def f(self): ... + + class B(A): + override def f(self): + super.f() + "); + } + + /// an abstract method has nothing to call, so its body is `...` + #[test] + fn implements_an_abstract_method() { + let test = basedpython( + " +from abc import ABC, abstractmethod + +class A(ABC): + @abstractmethod + def f(self, x: int) -> str: ... + +class B(A): + +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + from abc import ABC, abstractmethod + + class A(ABC): + @abstractmethod + def f(self, x: int) -> str: ... + + class B(A): + override def f(self, x: int) -> str: + ... + "); + } + + /// a generic superclass's parameters are filled in the way the class fills + /// them. `Iterator` and `Overlapping` are names a `.by` file has without + /// importing them, so nothing is imported for them + #[test] + fn writes_types_as_the_class_inherits_them() { + let test = basedpython( + " +from collections.abc import Mapping + +class M(Mapping[str, int]): + +", + ); + assert_snapshot!(test.override_members(&["__getitem__", "__iter__"]), @" + + from collections.abc import Mapping + + class M(Mapping[str, int]): + override def __getitem__(self, key: Overlapping[str], /) -> int: + ... + + override def __iter__(self) -> Iterator[str]: + ... + "); + } + + /// a type the file does not reach yet is imported, once however many + /// declarations name it + #[test] + fn imports_what_the_declarations_name() { + let test = CursorTest::builder() + .source( + "shapes.by", + " +from pathlib import Path + +class Shape: + def save(self, to: Path) -> None: ... + def load(self, source: Path) -> Path: + return source +", + ) + .source( + "main.by", + " +from shapes import Shape + +class Square(Shape): + +", + ) + .build(); + assert_snapshot!(test.override_members(&["save", "load"]), @" + from pathlib import Path + + from shapes import Shape + + class Square(Shape): + override def save(self, to: Path): + super.save(to) + + override def load(self, source: Path) -> Path: + return super.load(source) + "); + } + + /// a classmethod, a staticmethod, an `async def` and a `protected` method + /// each say so again + #[test] + fn keeps_how_a_method_is_declared() { + let test = basedpython( + " +class A: + class def make(cls, n: int) -> int: + return n + + static def helper(n: int) -> int: + return n + + async def fetch(self, *, retries: int = 3) -> bytes: + return b'' + + protected def step(self) -> None: ... + +class B(A): + +", + ); + assert_snapshot!( + test.override_members(&["make", "helper", "fetch", "step"]), + @" + + class A: + class def make(cls, n: int) -> int: + return n + + static def helper(n: int) -> int: + return n + + async def fetch(self, *, retries: int = 3) -> bytes: + return b'' + + protected def step(self) -> None: ... + + class B(A): + override class def make(cls, n: int) -> int: + return super.make(n) + + override static def helper(n: int) -> int: + return A.helper(n) + + override async def fetch(self, *, retries: int) -> bytes: + return await super.fetch(retries=retries) + + protected override def step(self): + super.step() + " + ); + } + + /// an `async def` that yields is an async generator: a call to it returns + /// the generator rather than a coroutine, so the override returns what the + /// call to the method it overrides does, with nothing to await + #[test] + fn overrides_an_async_generator_without_awaiting_it() { + let test = basedpython( + " +from collections.abc import AsyncIterator + +class A: + async def stream(self) -> AsyncIterator[int]: + yield 1 + +class B(A): + +", + ); + assert_snapshot!(test.override_members(&["stream"]), @" + + from collections.abc import AsyncIterator + + class A: + async def stream(self) -> AsyncIterator[int]: + yield 1 + + class B(A): + override def stream(self) -> AsyncIterator[int]: + return super.stream() + "); + } + + /// a default expression names what it reads as the file the override is in + /// reaches it, importing what that file does not + #[test] + fn imports_what_a_default_expression_reads() { + let test = CursorTest::builder() + .source( + "shapes.by", + " +from pathlib import Path as P + +def make_config() -> list[int]: + return [] + +class Shape: + def draw(self, config: list[int] = make_config(), root: P = P('.')) -> None: ... +", + ) + .source( + "main.by", + " +from shapes import Shape + +class Square(Shape): + +", + ) + .build(); + assert_snapshot!(test.override_members(&["draw"]), @" + from pathlib import Path + + from shapes import Shape, make_config + + class Square(Shape): + override def draw(self, config: list[int] = make_config(), root: Path = Path('.')): + super.draw(config, root) + "); + } + + /// a name the file binds to something else is not the one the default + /// reads, which is written through its module instead + #[test] + fn a_default_expression_s_name_shadowed_where_it_is_written_is_qualified() { + let test = CursorTest::builder() + .source( + "shapes.by", + " +def make_config() -> list[int]: + return [] + +class Shape: + def draw(self, config: list[int] = make_config()) -> None: ... +", + ) + .source( + "main.by", + " +from shapes import Shape + +def make_config() -> list[int]: + return [1] + +class Square(Shape): + +", + ) + .build(); + assert_snapshot!(test.override_members(&["draw"]), @" + import shapes + + from shapes import Shape + + def make_config() -> list[int]: + return [1] + + class Square(Shape): + override def draw(self, config: list[int] = shapes.make_config()): + super.draw(config) + "); + } + + /// a name a `.by` file has without an import is written bare only where + /// nothing in the file binds it to something else + #[test] + fn an_implicit_name_the_file_shadows_is_qualified() { + let test = CursorTest::builder() + .source("mylib.by", "class Iterator: ...\n") + .source( + "main.by", + " +from collections.abc import Mapping +from mylib import Iterator + +class M(Mapping[str, int]): + +", + ) + .build(); + assert_snapshot!(test.override_members(&["__iter__"]), @" + import typing + + from collections.abc import Mapping + from mylib import Iterator + + class M(Mapping[str, int]): + override def __iter__(self) -> typing.Iterator[str]: + ... + "); + } + + /// a method's type parameter named like one of the class's is renamed, so + /// that each annotation keeps naming the one it names + #[test] + fn renames_a_method_type_parameter_the_class_s_would_shadow() { + let test = basedpython( + " +class A[T]: + def f[U](self, u: U, t: T) -> T: + return t + +class B[U](A[U]): + +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + class A[T]: + def f[U](self, u: U, t: T) -> T: + return t + + class B[U](A[U]): + override def f[U2](self, u: U2, t: U) -> U: + return super.f(u, t) + "); + } + + /// a body on the class's header line is moved below it before a + /// declaration goes after it + #[test] + fn writes_after_a_body_on_the_header_line() { + let test = basedpython( + " +class A: + def f(self): ... + +class B(A): x = 1 +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + class A: + def f(self): ... + + class B(A): + x = 1 + + override def f(self): + super.f() + "); + } + + /// a default that is a value is inherited and left out; one that is an + /// expression is not, and is written as its source wrote it + #[test] + fn writes_only_the_defaults_an_override_does_not_inherit() { + let test = basedpython( + " +class A: + def f(self, a: int = 1, b: list[int] = [], /, c: str | None = None) -> None: ... + +class B(A): + +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + class A: + def f(self, a: int = 1, b: list[int] = [], /, c: str | None = None) -> None: ... + + class B(A): + override def f(self, a: int, b: list[int] = [], /, c: str | None): + super.f(a, b, c) + "); + } + + /// a generic method declares its type parameters again + #[test] + fn writes_a_generic_methods_type_parameters() { + let test = basedpython( + " +class A: + def f[T: int](self, t: T) -> T: + return t + +class B(A): + +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + class A: + def f[T: int](self, t: T) -> T: + return t + + class B(A): + override def f[T: int](self, t: T) -> T: + return super.f(t) + "); + } + + /// each overload is a declaration of its own, and the implementation after + /// them passes on whatever it is called with + #[test] + fn writes_an_overloaded_method_as_its_overloads() { + let test = basedpython( + " +class D(dict[str, int]): + +", + ); + assert_snapshot!(test.override_members(&["get"]), @" + + class D(dict[str, int]): + override def get(self, key: Overlapping[str], default: None = None, /) -> int | None + override def get(self, key: Overlapping[str], default: int, /) -> int + override def get[Element](self, key: Overlapping[str], default: Element, /) -> int | Element + override def get(self, *args, **kwargs): + return super.get(*args, **kwargs) + "); + } + + /// a caret on an indented line below the last statement is still in the + /// class, though python's syntax tree ended it on the line above + #[test] + fn a_caret_below_the_last_statement_is_in_the_class() { + let test = basedpython( + " +class A: + def f(self): ... + +class B(A): + x = 1 + + # more to come + + +def after(): ... +", + ); + assert_snapshot!(test.override_members(&["f"]), @" + + class A: + def f(self): ... + + class B(A): + x = 1 + + # more to come + + override def f(self): + super.f() + + + def after(): ... + "); + } +} diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 22ce39e910..51410ed476 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -31,7 +31,7 @@ use crate::types::signatures::{ CallableSignature, Parameter, Parameters, ParametersKind, Signature, }; use crate::types::tuple::{RepeatedUnit, TupleSpec, VariableLengthTuple, VariableSegment}; -use crate::types::typevar::BoundTypeVarIdentity; +use crate::types::typevar::{BoundTypeVarIdentity, BoundTypeVarInstance}; use crate::types::visibility::private_symbols; use crate::types::visitor::TypeVisitor; use crate::types::{ @@ -167,9 +167,34 @@ pub struct DisplaySettings<'db> { /// it, but emitting one as python would evaluate `_I + 1` on a `TypeVar` object at /// import time. reduce_symbolic_operations: bool, + /// Type parameters that the place a type is written declares, or is inside the scope of, each + /// with the name it is written by there, which an annotation there can hold. Any other type + /// variable is shown with the scope that binds it, which no annotation can. + type_params_in_scope: Rc, ast::name::Name>>, } impl<'db> DisplaySettings<'db> { + /// Write each type parameter in `params` by the name it comes with, as an annotation in its + /// scope does. Two type parameters can share a name, so each is told apart by which one it + /// is rather than by what it is called. + #[must_use] + pub fn with_type_params_in_scope( + &self, + db: &'db dyn Db, + params: impl IntoIterator, ast::name::Name)>, + ) -> Self { + let mut in_scope = (*self.type_params_in_scope).clone(); + in_scope.extend( + params + .into_iter() + .map(|(param, name)| (param.identity(db), name)), + ); + Self { + type_params_in_scope: Rc::new(in_scope), + ..self.clone() + } + } + /// basedpython: name the type parameter each type argument fills. #[must_use] pub fn with_named_type_arguments(&self) -> Self { @@ -2258,6 +2283,14 @@ impl<'db> FmtDetailed<'db> for DisplayRepresentation<'_, 'db> { } } }, + Type::TypeVar(bound_typevar) + if let Some(name) = self + .settings + .type_params_in_scope + .get(&bound_typevar.identity(db)) => + { + f.with_type(self.ty).write_str(name) + } Type::TypeVar(bound_typevar) => { f.set_invalid_type_annotation(); write!( diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index d920869b8d..0a6347e7f5 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -32,16 +32,18 @@ use crate::types::receivers; use crate::types::repeated_underscore::{UnderscoreLowering, UnderscoreRefusal}; use crate::types::signatures::{ Parameter, ParameterKind, ParametersKind, ReturnCallableTypeVarScope, Signature, + is_placeholder_default, }; use crate::types::{ - CallDunderError, CallableTypes, ClassBase, ClassLiteral, ClassType, KnownClass, KnownFunction, - KnownUnion, PropertyAccessorRole, SpecialFormType, Type, TypeContext, - TypeVarBoundOrConstraints, TypeVarVariance, binding_type, + BoundTypeVarInstance, CallDunderError, CallableTypes, ClassBase, ClassLiteral, ClassType, + KnownClass, KnownFunction, KnownUnion, PropertyAccessorRole, SpecialFormType, Type, + TypeContext, TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, binding_type, }; use crate::{Db, HasDefinition, HasType, ProgramEnvironment, SemanticModel}; use ruff_db::files::{File, FileRange}; use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; use ruff_python_ast::{self as ast, AnyNodeRef, name::Name}; use ruff_python_stdlib::identifiers::is_mangled_private; use ruff_text_size::{Ranged, TextRange}; @@ -49,8 +51,8 @@ use rustc_hash::FxHashSet; use ty_module_resolver::{ ImportingFile, Module, ModuleName, ResolverFile, resolve_module_confident, }; -use ty_python_core::definition::{Definition, DefinitionKind}; -use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; +use ty_python_core::scope::{FileScopeId, NodeWithScopeRef, ScopeId}; use ty_python_core::symbol::ScopedSymbolId; use ty_python_core::{ProgramFile, attribute_scopes, semantic_index, use_def_map}; @@ -98,13 +100,44 @@ pub fn definitions_for_name<'db>( node: AnyNodeRef<'_>, alias_resolution: ImportAliasResolution, ) -> Vec> { - let db = model.db(); - let env = model.program_environment(); - // Get the scope for this name expression let Some(scope) = model.scope(node) else { return vec![]; }; + definitions_for_name_in_scope(model, scope, name_str, node, alias_resolution) +} + +/// basedpython: what `name_str` means read in the body of `class`, through any import of it: what +/// a default expression of one of its methods reads +pub fn definitions_for_name_in_class_body<'db>( + model: &SemanticModel<'db>, + class: &ast::StmtClassDef, + name_str: &str, +) -> Vec> { + let Some(scope) = semantic_index(model.db(), model.program_file()) + .try_node_scope(NodeWithScopeRef::Class(class)) + else { + return vec![]; + }; + definitions_for_name_in_scope( + model, + scope, + name_str, + class.into(), + ImportAliasResolution::ResolveAliases, + ) +} + +/// what `name_str`, read in `scope` at `node`, means +fn definitions_for_name_in_scope<'db>( + model: &SemanticModel<'db>, + scope: FileScopeId, + name_str: &str, + node: AnyNodeRef<'_>, + alias_resolution: ImportAliasResolution, +) -> Vec> { + let db = model.db(); + let env = model.program_environment(); // basedpython: inside a trailing lambda block whose callback declares a // receiver, the receiver outranks every binding outside the block, so it is @@ -3350,6 +3383,473 @@ pub fn overridable_members<'db>( members } +/// basedpython: the module `name` is taken from when, written bare in a type expression of +/// `file`, it means what defines `ty`: a name the language makes available in a `.by` file +/// without an import, as it does `typing`'s `Iterator`, so that writing `ty` there needs no import +/// unless the file binds `name` to something else +pub fn implicit_type_name_module<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: ProgramFile<'db>, + name: &str, + ty: Type<'db>, +) -> Option<&'static str> { + if !file.file(db).source_type(db).is_basedpython() { + return None; + } + let implicit = implicit_name(name)?; + if !implicit.position.admits(true) { + return None; + } + let Place::Defined(member) = implicit.resolve(db, env).place else { + return None; + }; + let definition = member.ty.definition(db, env)?; + if Some(definition) != ty.definition(db, env) { + return None; + } + implicit + .resolving_module(db, env) + .map(ty_module_resolver::KnownModule::as_str) +} + +/// basedpython: how a method is reached, which is how a declaration of it says so: `class def` +/// for a classmethod, `static def` for a staticmethod +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InheritedMethodKind { + Instance, + Class, + Static, +} + +/// basedpython: a method a class inherits and does not define itself, with everything a +/// declaration overriding it has to say +#[derive(Debug, Clone)] +#[expect( + clippy::struct_excessive_bools, + reason = "independent facts about the method" +)] +pub struct InheritedMethod<'db> { + /// the method's name + pub name: Name, + /// the name of the superclass the class inherits it from: the first in its MRO to bind + /// the name + pub superclass_name: Name, + /// the module that declares the superclass, when a module name reaches it + pub superclass_module: Option, + /// an instance of the superclass, specialized the way the class specializes it: what a + /// static method, which has no `super` to call it through, names it by + pub superclass_instance: Type<'db>, + /// whether it is abstract where the class inherits it from, and so left for the class to + /// implement: what `abstract-instantiation` counts as abstract + pub is_abstract: bool, + pub kind: InheritedMethodKind, + /// whether it is an `async def` whose call returns a coroutine, which an override awaits. + /// an async generator is not one: an override returns the generator its call returns + pub is_async: bool, + /// whether it is `protected`, which an override has to say again: a plain declaration over + /// an inherited `protected` member is `invalid-override-visibility` + pub is_protected: bool, + /// its signature as the class inherits it, specialized the way the class specializes the + /// superclass. one per overload for an overloaded method, otherwise one + pub signatures: Vec>, + /// whether it is overloaded, so that [`Self::signatures`] are its overloads + pub is_overloaded: bool, +} + +/// basedpython: one signature of an [`InheritedMethod`], in the parts a declaration writes +#[derive(Debug, Clone)] +pub struct InheritedSignature<'db> { + /// the type parameters the signature declares itself, in order. empty when it declares + /// none, and when it declares one a header cannot write with a name alone: a `ParamSpec` or a + /// variadic pack, whose annotations then go unwritten too + pub type_params: Vec>, + pub parameters: Vec>, + /// what it returns. `None` when nothing is worth writing: it returns `None`, which a + /// basedpython `def` that leaves out its return annotation already says, or nothing is known + pub returns: Option>, + /// whether it is known to return `None`, so that a call to it has nothing to pass on + pub returns_none: bool, +} + +/// basedpython: a type parameter an [`InheritedSignature`] declares +#[derive(Debug, Clone)] +pub struct InheritedTypeParam<'db> { + /// the type parameter itself, which the signature's types name + pub typevar: BoundTypeVarInstance<'db>, + pub name: Name, + /// the upper bound it is declared with, if any + pub bound: Option>, + /// the constraints it is declared with, if any + pub constraints: Vec>, +} + +/// basedpython: which of python's parameter kinds an [`InheritedParameter`] is +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InheritedParameterKind { + PositionalOnly, + PositionalOrKeyword, + Variadic, + KeywordOnly, + KeywordVariadic, +} + +/// basedpython: a parameter of an [`InheritedSignature`] +#[derive(Debug, Clone)] +pub struct InheritedParameter<'db> { + pub kind: InheritedParameterKind, + pub name: Name, + /// the type it is annotated with. `None` for the receiver, whose type is implicit, and for a + /// parameter whose annotation is nothing a declaration could write: none at all, the + /// anonymous type parameter an unannotated basedpython parameter opens, or a starred pack + pub annotation: Option>, + /// the default it is declared with, if any + pub default: Option>, +} + +/// basedpython: the default of an [`InheritedParameter`] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InheritedDefault<'db> { + /// a value, written the way a signature writes it: `1`, `"a"`, `None`. an override that + /// leaves it out inherits it + Value(String), + /// an expression rather than a value: `[]`, `make()`. basedpython re-evaluates it on every + /// call, so there is no value to inherit and an override has to write it out + Expression(InheritedExpression<'db>), + /// a default that cannot be written: one with no source, a stub's `...` placeholder, or an + /// expression reading a name that means nothing where it is written + Unwritable, +} + +/// basedpython: a default expression an override writes out, which it has to write so that each +/// name it reads means what it means where the overridden method is declared +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InheritedExpression<'db> { + /// as its source writes it + pub text: String, + /// the names it reads from the scope it is evaluated in, in source order. a name a lambda or + /// a comprehension in it binds is its own, and is not one of them + pub names: Vec>, +} + +/// basedpython: a name an [`InheritedExpression`] reads +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InheritedExpressionName<'db> { + /// where it is in [`InheritedExpression::text`] + pub range: TextRange, + pub name: Name, + /// what it means where the expression is written, through any import of it + pub definitions: Vec>, +} + +/// basedpython: the methods `class` inherits without defining them, each with what a declaration +/// overriding it has to say, in MRO order of the superclass each is inherited from +/// +/// these are the members an editor offers to override or implement. a method is left out when +/// it cannot be overridden: a `final` one, and a `private` one, which python mangles into a name +/// of its own. so are the constructors and the methods they call (`__init__`, `__new__`, +/// `__post_init__`, `__init_subclass__`), which basedpython declares with `init` and which the +/// override checks do not hold to what they override. members other than methods are not +/// offered: an attribute or a property is declared with syntax of its own. unlike +/// [`overridable_members`], `object`'s methods are offered: `__eq__`, `__hash__` and `__repr__` +/// are overridden all the time +/// +/// a method is abstract when the nearest declaration of it in the MRO is abstract, which is when +/// `abstract-instantiation` says the class leaves it unimplemented +pub fn inherited_methods<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: ClassLiteral<'db>, +) -> Vec> { + let class_type = class.identity_specialization(db); + let unimplemented = class_type.abstract_methods(db); + let mut seen: FxHashSet = FxHashSet::default(); + let mut methods = Vec::new(); + + // the MRO opens with the class itself, whose names seed `seen`: a name it binds is not one + // it inherits, and after that the first superclass to bind a name is the one it is inherited + // from + for (depth, superclass) in class_type + .iter_mro(db) + .filter_map(ClassBase::into_class) + .enumerate() + { + let Some((literal, _)) = superclass.static_class_literal(db) else { + continue; + }; + for member in all_end_of_scope_members(db, literal.body_scope(db)) { + let name = member.member.name; + if !seen.insert(name.clone()) + || depth == 0 + || is_constructor_like_method(&name) + || is_mangled_private(&name) + { + continue; + } + let Place::Defined(defined) = superclass + .own_class_member(db, env, None, &name) + .inner + .place + else { + continue; + }; + let Type::FunctionLiteral(function) = defined.ty else { + continue; + }; + if function.has_known_decorator(db, FunctionDecorators::FINAL) + || function.has_known_decorator(db, FunctionDecorators::PRIVATE) + { + continue; + } + let kind = if function.is_staticmethod(db) { + InheritedMethodKind::Static + } else if function.is_classmethod(db) { + InheritedMethodKind::Class + } else { + InheritedMethodKind::Instance + }; + let (overloads, _) = function.overloads_and_implementation(db); + let signature = function.signature(db); + // an `async def` that yields is an async generator, whose call returns the generator + // rather than a coroutine + let is_async = function.literal(db).last_definition.header(db).is_async + && signature.overloads.iter().all(|signature| { + signature + .return_ty + .known_specialization(db, env, KnownClass::CoroutineType) + .is_some() + }); + methods.push(InheritedMethod { + is_abstract: unimplemented.contains_key(&name), + superclass_name: literal.name(db).clone(), + superclass_module: ty_module_resolver::file_to_module( + db, + literal.program_file(db).resolver_file(db), + ) + .map(|module| module.name(db).clone()), + superclass_instance: Type::instance(db, env, superclass), + kind, + is_async, + is_protected: function.has_known_decorator(db, FunctionDecorators::PROTECTED), + is_overloaded: !overloads.is_empty() && signature.overloads.len() > 1, + signatures: signature + .overloads + .iter() + .map(|signature| inherited_signature(db, env, signature, is_async)) + .collect(), + name, + }); + } + } + methods +} + +/// `signature` in the parts a declaration writes. an `async def`'s signature returns the coroutine +/// a call makes, and its declaration writes what the coroutine returns +fn inherited_signature<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + signature: &Signature<'db>, + is_async: bool, +) -> InheritedSignature<'db> { + let mut type_params = Vec::new(); + for variable in signature + .generic_context + .iter() + .flat_map(|context| context.variables(db)) + { + let typevar = variable.typevar(db); + match variable.kind(db) { + // the receiver's `Self` is implicit, and an unannotated parameter's anonymous + // type parameter is never written + TypeVarKind::TypingSelf | TypeVarKind::Pep613Alias | TypeVarKind::InferredParameter => { + } + TypeVarKind::LegacyTypeVar | TypeVarKind::Pep695TypeVar => { + let (bound, constraints) = match typevar.bound_or_constraints(db, env) { + Some(TypeVarBoundOrConstraints::UpperBound(bound)) => (Some(bound), Vec::new()), + Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { + (None, constraints.elements(db).to_vec()) + } + None => (None, Vec::new()), + }; + type_params.push(InheritedTypeParam { + typevar: variable, + name: typevar.name(db).clone(), + bound, + constraints, + }); + } + // a `ParamSpec` or a variadic pack: a header cannot declare one by its name alone, + // and an annotation naming one is then left out rather than written wrong + _ => { + type_params.clear(); + break; + } + } + } + + let parameters = signature + .parameters() + .iter() + .filter_map(|parameter| { + let (kind, default) = match parameter.kind() { + ParameterKind::PositionalOnly { .. } => { + (InheritedParameterKind::PositionalOnly, true) + } + ParameterKind::PositionalOrKeyword { .. } => { + (InheritedParameterKind::PositionalOrKeyword, true) + } + ParameterKind::Variadic { .. } => (InheritedParameterKind::Variadic, false), + ParameterKind::KeywordOnly { .. } => (InheritedParameterKind::KeywordOnly, true), + ParameterKind::KeywordVariadic { .. } => { + (InheritedParameterKind::KeywordVariadic, false) + } + }; + // a list that repeats `_` is written with the `_`s its author wrote + let name = if parameter.is_repeated_underscore() { + Name::new_static("_") + } else { + parameter.name()?.clone() + }; + let annotated = parameter.annotated_type(); + let annotation = (parameter.should_annotation_be_displayed() + && !parameter.has_starred_annotation() + && !annotated.is_unknown() + && !annotated.is_inferred_parameter_hole(db)) + .then_some(annotated); + let default = default + .then(|| parameter.default_type(db)) + .flatten() + .map(|default| inherited_default(db, env, parameter, default)); + Some(InheritedParameter { + kind, + name, + annotation, + default, + }) + }) + .collect(); + + let returns = if is_async + && let Some(coroutine) = + signature + .return_ty + .known_specialization(db, env, KnownClass::CoroutineType) + && let [_, _, returned] = coroutine.types(db) + { + *returned + } else { + signature.return_ty + }; + InheritedSignature { + type_params, + parameters, + returns: (!returns.is_none(db) && !returns.is_unknown()).then_some(returns), + returns_none: returns.is_none(db), + } +} + +/// how `parameter`, whose default is of type `default`, says its default +fn inherited_default<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + parameter: &Parameter<'db>, + default: Type<'db>, +) -> InheritedDefault<'db> { + let Some(definition) = parameter.definition() else { + return default + .display_default_value(db, env) + .map_or(InheritedDefault::Unwritable, InheritedDefault::Value); + }; + // a stub's `...` stands in for a value it does not say + if is_placeholder_default(db, definition) { + return InheritedDefault::Unwritable; + } + if let Some(value) = default.display_default_value(db, env) { + return InheritedDefault::Value(value); + } + inherited_expression(db, definition) + .map_or(InheritedDefault::Unwritable, InheritedDefault::Expression) +} + +/// the default expression of the parameter `definition` declares, with what each name it reads +/// means where it is written; `None` when it has none, or reads a name that means nothing there +fn inherited_expression<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> Option> { + let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node)) = + definition.kind(db) + else { + return None; + }; + let module = parsed_module(db, definition.python_file(db)).load(db); + let expression = node.node(&module).default()?; + let program_file = definition.program_file(db); + let model = SemanticModel::new(db, program_file); + // a name bound in a scope that is not this one or one around it is bound by a lambda or a + // comprehension in the expression + let evaluated_in = model.scope(expression.into())?; + let index = semantic_index(db, program_file); + let around: FxHashSet = index + .ancestor_scopes(evaluated_in) + .map(|(scope, _)| scope) + .collect(); + + let mut names = Vec::new(); + let mut found = Vec::new(); + collect_loaded_names(expression, &mut found); + for name in found { + let definitions = definitions_for_name( + &model, + name.id.as_str(), + name.into(), + ImportAliasResolution::ResolveAliases, + ); + if definitions.is_empty() { + return None; + } + let is_own = definitions.iter().all(|resolved| { + matches!( + resolved, + ResolvedDefinition::Definition(bound) + if bound.program_file(db) == program_file + && !around.contains(&bound.file_scope(db)) + ) + }); + if is_own { + continue; + } + names.push(InheritedExpressionName { + range: name.range() - expression.start(), + name: name.id.clone(), + definitions, + }); + } + Some(InheritedExpression { + text: source_text(db, definition.file(db))[expression.range()].to_string(), + names, + }) +} + +/// the names `expression` reads, in source order +fn collect_loaded_names<'a>(expression: &'a ast::Expr, names: &mut Vec<&'a ast::ExprName>) { + struct Names<'a, 'n> { + names: &'n mut Vec<&'a ast::ExprName>, + } + impl<'a> SourceOrderVisitor<'a> for Names<'a, '_> { + fn visit_expr(&mut self, expression: &'a ast::Expr) { + if let ast::Expr::Name(name) = expression + && name.ctx.is_load() + { + self.names.push(name); + } + walk_expr(self, expression); + } + } + Names { names }.visit_expr(expression); +} + /// basedpython: the parameters a trailing lambda block binds implicitly, each as /// its name and the type the callee gives it. /// diff --git a/crates/ty_python_semantic/src/types/implicit_names.rs b/crates/ty_python_semantic/src/types/implicit_names.rs index 09ed428aaa..95813c5407 100644 --- a/crates/ty_python_semantic/src/types/implicit_names.rs +++ b/crates/ty_python_semantic/src/types/implicit_names.rs @@ -103,14 +103,21 @@ impl ImplicitName { db: &'db dyn Db, env: &ProgramEnvironment<'db>, ) -> Option> { - self.modules - .iter() - .find(|&&module| { - !known_module_symbol(db, env, module, self.member) - .place - .is_undefined() - }) - .and_then(|&module| core_module_scope(db, env, module)) + self.resolving_module(db, env) + .and_then(|module| core_module_scope(db, env, module)) + } + + /// The module [`ImplicitName::resolve`] takes the member from. + pub(crate) fn resolving_module<'db>( + self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ) -> Option { + self.modules.iter().copied().find(|&module| { + !known_module_symbol(db, env, module, self.member) + .place + .is_undefined() + }) } } diff --git a/crates/ty_python_semantic/src/types/typevar.rs b/crates/ty_python_semantic/src/types/typevar.rs index 5412de4769..87272b1b67 100644 --- a/crates/ty_python_semantic/src/types/typevar.rs +++ b/crates/ty_python_semantic/src/types/typevar.rs @@ -1612,6 +1612,12 @@ impl<'db> BoundTypeVarInstance<'db> { self.identity_inner(db) } + /// Whether `self` and `other` are the same occurrence of a typevar: the same typevar bound + /// by the same scope, whatever their bounds or constraints have been materialized to. + pub fn is_same_occurrence(self, db: &'db dyn Db, other: Self) -> bool { + self.identity(db) == other.identity(db) + } + pub fn name(self, db: &'db dyn Db) -> &'db Name { self.typevar(db).name(db) } diff --git a/crates/ty_server/src/document.rs b/crates/ty_server/src/document.rs index c55d29f1f4..c0318cec2a 100644 --- a/crates/ty_server/src/document.rs +++ b/crates/ty_server/src/document.rs @@ -15,6 +15,7 @@ pub(crate) use location::ToLink; pub use notebook::NotebookDocument; pub(crate) use range::{ FileRangeExt, LspRange, PositionExt, RangeExt, TextSizeExt, ToRangeExt, resolve_file_uri_range, + text_range_to_lsp_range, }; pub use text_document::TextDocument; pub(crate) use text_document::{DocumentVersion, LanguageId}; diff --git a/crates/ty_server/src/document/range.rs b/crates/ty_server/src/document/range.rs index 4dd96441b4..05c1d8d6e2 100644 --- a/crates/ty_server/src/document/range.rs +++ b/crates/ty_server/src/document/range.rs @@ -238,7 +238,9 @@ fn text_size_to_lsp_position( source_location_to_position(&source_location) } -fn text_range_to_lsp_range( +/// `range` in `text`, which need not be any document's text: an edit's answer can name a range in +/// the text its edits leave behind +pub(crate) fn text_range_to_lsp_range( range: TextRange, text: &str, index: &LineIndex, diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index fc0828d373..5ee6fea20d 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -237,6 +237,19 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + // both are asked by someone waiting on a shortcut they pressed + requests::OverridableMembersRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } + requests::OverrideMembersRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } requests::PrepareTypeHierarchyRequestHandler::METHOD => background_document_request_task::< requests::PrepareTypeHierarchyRequestHandler, >( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 46d7dedf81..0de96f8ff9 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -42,6 +42,8 @@ mod hover; mod injections; mod inlay_hints; mod module_name; +mod overridable_members; +mod override_members; mod prepare_call_hierarchy; mod prepare_rename; mod prepare_type_hierarchy; @@ -95,6 +97,8 @@ pub(super) use hover::HoverRequestHandler; pub(super) use injections::InjectionsRequestHandler; pub(super) use inlay_hints::InlayHintRequestHandler; pub(super) use module_name::ModuleNameHandler; +pub(super) use overridable_members::OverridableMembersRequestHandler; +pub(super) use override_members::OverrideMembersRequestHandler; pub(super) use prepare_call_hierarchy::PrepareCallHierarchyRequestHandler; pub(super) use prepare_rename::PrepareRenameRequestHandler; pub(super) use prepare_type_hierarchy::PrepareTypeHierarchyRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/overridable_members.rs b/crates/ty_server/src/server/api/requests/overridable_members.rs new file mode 100644 index 0000000000..5774c25162 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/overridable_members.rs @@ -0,0 +1,145 @@ +//! `by/overridableMembers`: the methods the class at a position could override, for an editor to +//! offer +//! +//! what an editor's *Override Members* and *Implement Members* show before writing anything: the +//! methods the class inherits, grouped by the superclass each comes from, with the abstract ones — +//! those the class has to implement — marked. `by/overrideMembers` writes the ones picked. LSP has +//! no request for either: a code action carries an edit worked out before anything is picked, and +//! here what the edit is depends on what is picked + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Request, TextDocumentIdentifier, Uri, +}; +use ty_ide::{InheritedMethodKind, members_to_override}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::PositionExt; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum OverridableMembersRequest {} + +impl Request for OverridableMembersRequest { + type Params = OverridableMembersParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/overridableMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// which class to ask about: the document, and a position in the class — its header, its body, +/// or an indented line below its last statement, where a member would be added +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct OverridableMembersParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +/// the class, and what it could override +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct OverridableMembers { + /// the class's name + class_name: String, + /// in MRO order of the superclass each is inherited from + members: Vec, +} + +/// one method the class inherits and does not define +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct OverridableMember { + /// the method's name, which is what `by/overrideMembers` is asked to write it by + name: String, + /// the superclass it is inherited from + container_name: String, + /// the module that declares the superclass + #[serde(skip_serializing_if = "Option::is_none")] + container_module: Option, + /// whether it is abstract where it is inherited from, so that the class has to implement it: + /// what `abstract-instantiation` counts as abstract + #[serde(rename = "abstract")] + is_abstract: bool, + kind: OverridableMemberKind, + /// the declaration `by/overrideMembers` writes for it, from its name on: + /// `greet(self, name: str) -> str`. an overloaded method's first overload + signature: String, +} + +/// how the method is reached, which is how its declaration says so +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum OverridableMemberKind { + Method, + ClassMethod, + StaticMethod, +} + +pub(crate) struct OverridableMembersRequestHandler; + +impl RequestHandler for OverridableMembersRequestHandler { + type RequestType = OverridableMembersRequest; +} + +impl BackgroundDocumentRequestHandler for OverridableMembersRequestHandler { + fn document_uri(params: &OverridableMembersParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, for a document that is not basedpython source, + /// and when there is no class at the position; a class that inherits nothing it could + /// override has an empty list + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: OverridableMembersParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + let Some(class) = members_to_override(db, db.program_file(file), offset) else { + return Ok(None); + }; + Ok(Some(OverridableMembers { + class_name: class.class_name.to_string(), + members: class + .members + .into_iter() + .map(|member| OverridableMember { + name: member.name.to_string(), + container_name: member.superclass.to_string(), + container_module: member.superclass_module, + is_abstract: member.is_abstract, + kind: match member.kind { + InheritedMethodKind::Instance => OverridableMemberKind::Method, + InheritedMethodKind::Class => OverridableMemberKind::ClassMethod, + InheritedMethodKind::Static => OverridableMemberKind::StaticMethod, + }, + signature: member.signature, + }) + .collect(), + })) + } +} + +impl RetriableRequestHandler for OverridableMembersRequestHandler {} diff --git a/crates/ty_server/src/server/api/requests/override_members.rs b/crates/ty_server/src/server/api/requests/override_members.rs new file mode 100644 index 0000000000..4a452734b3 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/override_members.rs @@ -0,0 +1,135 @@ +//! `by/overrideMembers`: the declarations overriding the methods an editor's user picked from +//! `by/overridableMembers` +//! +//! each is written the way basedpython writes an override — the `override` modifier, the +//! signature as the class inherits it, each type spelled for the document and imported when it is +//! not reached already — with a body that calls the method it overrides, or `...` for one the class +//! has to implement. the answer is the edits that write them and where the first one's body is +//! once they are applied, which is where the user goes next + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, TextEdit, + Uri, +}; +use ruff_python_ast::name::Name; +use ruff_source_file::LineIndex; +use ruff_text_size::Ranged; +use ty_ide::override_members; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{PositionExt, ToRangeExt, text_range_to_lsp_range}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum OverrideMembersRequest {} + +impl Request for OverrideMembersRequest { + type Params = OverrideMembersParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/overrideMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// which class, as `by/overridableMembers` is asked about it, and which of the members it answered +/// to write. the position is also where they are written: after the statement of the class body +/// it is in or follows +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct OverrideMembersParams { + text_document: TextDocumentIdentifier, + position: Position, + /// the names of the members to write, as `by/overridableMembers` gives them + members: Vec, +} + +/// the declarations, as edits to the document +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct OverrideMembers { + /// the declarations, and the imports they need. none of them overlap + edits: Vec, + /// the first declaration's body, in the document as the edits leave it: what the user + /// replaces first + selection: Range, +} + +pub(crate) struct OverrideMembersRequestHandler; + +impl RequestHandler for OverrideMembersRequestHandler { + type RequestType = OverrideMembersRequest; +} + +impl BackgroundDocumentRequestHandler for OverrideMembersRequestHandler { + fn document_uri(params: &OverrideMembersParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when `by/overridableMembers` would be, and when none of the members asked for is + /// one it lists + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: OverrideMembersParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + let names: Vec = params.members.iter().map(Name::new).collect(); + let Some(overrides) = override_members(db, db.program_file(file), offset, &names) else { + return Ok(None); + }; + + let mut edits = Vec::with_capacity(overrides.edits.len()); + for edit in &overrides.edits { + let Some(range) = edit.range().to_lsp_range(db, file, snapshot.encoding()) else { + return Ok(None); + }; + edits.push(TextEdit { + range: range.local_range(), + new_text: edit.content().unwrap_or_default().to_string(), + }); + } + + // the selection is in the text the edits leave behind, so it is a position in that text + let source = ruff_db::source::source_text(db, file); + let mut edited = source.as_str().to_string(); + let mut applied = overrides.edits.clone(); + applied.sort_by_key(|edit| std::cmp::Reverse(edit.start())); + for edit in &applied { + edited.replace_range( + edit.range().to_std_range(), + edit.content().unwrap_or_default(), + ); + } + let selection = text_range_to_lsp_range( + overrides.selection, + &edited, + &LineIndex::from_source_text(&edited), + snapshot.encoding(), + ); + + Ok(Some(OverrideMembers { edits, selection })) + } +} + +impl RetriableRequestHandler for OverrideMembersRequestHandler {} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index cf0fb28d86..5f9c8cb17f 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -49,6 +49,7 @@ mod injections; mod inlay_hints; mod named_text; mod notebook; +mod override_members; mod program_model; mod project_server; mod publish_diagnostics; diff --git a/crates/ty_server/tests/e2e/override_members.rs b/crates/ty_server/tests/e2e/override_members.rs new file mode 100644 index 0000000000..e2dcf6af93 --- /dev/null +++ b/crates/ty_server/tests/e2e/override_members.rs @@ -0,0 +1,160 @@ +//! `by/overridableMembers` and `by/overrideMembers` over the wire +//! +//! which members a class could override, and how a declaration of each is written, is `ty_ide`'s +//! and its unit tests cover it. what these cover is the contract a client reads: a position in a +//! class in, out the members with the superclass and module each comes from, and then for the +//! names picked the edits that write them, imports included, with the first one's body as a range +//! in the text the edits leave behind — and `null` where there is no class + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder}; + +/// `by/overridableMembers`, in json throughout +enum OverridableMembers {} + +impl Request for OverridableMembers { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/overridableMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// `by/overrideMembers`, in json throughout +enum OverrideMembers {} + +impl Request for OverrideMembers { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/overrideMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const BASES: &str = "\ +from abc import ABC, abstractmethod +from pathlib import Path + +class Shape(ABC): + @abstractmethod + def area(self) -> float: ... + + def save(self, to: Path) -> None: ... +"; + +const MAIN: &str = "\ +from bases import Shape + +class Square(Shape): + side = 1.0 + +def free() -> None: ... +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/bases.by"), BASES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + Ok(server) +} + +fn list(server: &mut TestServer, line: u32, character: u32) -> Option { + let uri = server.file_uri(SystemPath::new("src/main.by")); + server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": line, "character": character }, + })) +} + +#[test] +fn a_class_lists_what_it_inherits_with_where_each_comes_from() -> Result<()> { + let mut server = server()?; + + // in the body of `Square` + let answer = list(&mut server, 3, 4).expect("a class is there"); + assert_eq!(answer["className"], "Square"); + let members = answer["members"].as_array().expect("a list of members"); + assert_eq!( + members[0], + serde_json::json!({ + "name": "area", + "containerName": "Shape", + "containerModule": "bases", + "abstract": true, + "kind": "method", + "signature": "area(self) -> float", + }), + ); + assert_eq!(members[1]["name"], "save"); + assert_eq!(members[1]["abstract"], false); + assert_eq!(members[1]["signature"], "save(self, to: Path)"); + // what every class inherits comes last, from `object` + assert!( + members[2..] + .iter() + .all(|member| member["containerName"] == "object"), + "{answer:#}" + ); + Ok(()) +} + +#[test] +fn no_class_at_the_position_is_null() -> Result<()> { + let mut server = server()?; + // in `free` + assert_eq!(list(&mut server, 5, 5), None); + Ok(()) +} + +#[test] +fn the_members_picked_are_written_with_their_imports_and_the_first_body_selected() -> Result<()> { + let mut server = server()?; + let uri = server.file_uri(SystemPath::new("src/main.by")); + + let answer = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 3, "character": 14 }, + "members": ["area", "save"], + })) + .expect("the members are written"); + + let edits = answer["edits"].as_array().expect("a list of edits"); + assert_eq!( + edits[0], + serde_json::json!({ + "range": { + "start": { "line": 3, "character": 14 }, + "end": { "line": 3, "character": 14 }, + }, + "newText": "\n\n override def area(self) -> float:\n ...\n\n override def save(self, to: Path):\n super.save(to)", + }), + "{answer:#}" + ); + assert_eq!(edits.len(), 2, "{answer:#}"); + assert_eq!(edits[1]["newText"], "from pathlib import Path\n"); + // the `...` of `area`, once the import has moved it down a line + assert_eq!( + answer["selection"], + serde_json::json!({ + "start": { "line": 7, "character": 8 }, + "end": { "line": 7, "character": 11 }, + }), + ); + + // a name the class could not override is not written + let none = server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 3, "character": 14 }, + "members": ["side"], + })); + assert_eq!(none, None); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index acb94d502e..dc66c12619 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -506,6 +506,64 @@ project's `src/shapes/circle.by` is `shapes.circle`, and `src/shapes` and `src/shapes/__init__.by` are both `shapes`. it is `null` for a path no module is imported from, and for a file that resolution would never reach +## override and implement members + +an editor's *override methods* and *implement methods* ask twice: once for what +the class can override, to let its user pick, and once for the declarations of +the ones picked. a code action cannot carry that, since its edit is worked out +before anything is picked + +`by/overridableMembers` takes a position in a class — its header, its body, or +an indented line below its last statement, where a new member would go — and +answers the class's name and the methods it inherits without defining them, in +MRO order of the superclass each comes from. each comes with that superclass and +its module, whether it is `abstract` where it is inherited from (what +`abstract-instantiation` counts as left unimplemented, so the ones *implement +methods* offers), whether it is a `method`, a `classMethod` or a `staticMethod`, +and its `signature`: the declaration that would be written, from its name on. +`object`'s methods are listed, last; a `final` or `private` method, and the +constructors and the methods they call, are not. the answer is `null` where +there is no class, and for a file that is not basedpython + +`by/overrideMembers` takes the same position and the names picked, and answers +the edits that write their declarations, in the order they were listed: + +```by +from pathlib import Path + +class Square(Shape): + side = 1.0 + + override def area(self) -> float: + ... + + override def save(self, to: Path): + super.save(to) +``` + +each is declared `override`, with `class`, `static`, `async` or `protected` when +the method it overrides is — an async generator is not `async`, and returns the +generator its call makes — and with the signature the class inherits: a generic +superclass's type parameters filled in the way the class fills them, a method's +own type parameter renamed where it would hide one of the class's, an +overloaded method's overloads one declaration each. a type the file does not +reach yet is imported, and one it cannot name at all is left out, since a +parameter left unannotated takes the annotation of the one it overrides. a +default that is a value is left out of a method with a body for the same +reason, and one that is an expression is written as its base wrote it, with +each name in it imported, or written through its module where the file means +something else by it. the body +calls the method through `super`, or a static method, which has no `super`, +through its class's name; a method that is abstract has nothing to call, and +its body is `...`, which the checker reports until it is written. `-> None` is +not written + +the declarations go after the statement the position is in or follows, or at the +end of the body for a position in the header, and replace a body that is only +`pass` or `...`. the answer's `selection` is the first declaration's body as a +range in the text the edits leave behind, which is where the user goes next. it +is `null` when none of the names is a method the class could override + ## the whole project's diagnostics with `diagnosticMode` set to `workspace`, the server checks every file in the From 1b2d0a0bca5d600b0d4ec163f07329837e700cab Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:10:31 +1000 Subject: [PATCH 08/15] mark the one import that resolves an unresolved name in its quick fix's data, and answer by/copiedImports and by/pastedImports, the imports copied code needs and the ones a paste is missing --- crates/ty_ide/src/add_dependency.rs | 2 + crates/ty_ide/src/code_action.rs | 35 +- crates/ty_ide/src/completion.rs | 25 +- .../ty_ide/src/django_template/diagnostics.rs | 2 + crates/ty_ide/src/importer.rs | 68 +- crates/ty_ide/src/lib.rs | 2 + crates/ty_ide/src/pasted_imports.rs | 710 ++++++++++++++++++ crates/ty_server/src/server/api.rs | 13 + crates/ty_server/src/server/api/requests.rs | 4 + .../src/server/api/requests/code_action.rs | 34 +- .../src/server/api/requests/copied_imports.rs | 146 ++++ .../src/server/api/requests/pasted_imports.rs | 126 ++++ crates/ty_server/tests/e2e/main.rs | 1 + crates/ty_server/tests/e2e/pasted_imports.rs | 168 +++++ ...action_attribute_access_on_unimported.snap | 3 + docs/basedpython/features/editor.md | 27 + 16 files changed, 1338 insertions(+), 28 deletions(-) create mode 100644 crates/ty_ide/src/pasted_imports.rs create mode 100644 crates/ty_server/src/server/api/requests/copied_imports.rs create mode 100644 crates/ty_server/src/server/api/requests/pasted_imports.rs create mode 100644 crates/ty_server/tests/e2e/pasted_imports.rs diff --git a/crates/ty_ide/src/add_dependency.rs b/crates/ty_ide/src/add_dependency.rs index f3a88ccb4d..7c8cc28fad 100644 --- a/crates/ty_ide/src/add_dependency.rs +++ b/crates/ty_ide/src/add_dependency.rs @@ -81,6 +81,7 @@ pub(crate) fn code_actions(db: &dyn Db, file: File, range: TextRange) -> Vec Vec, + /// The fix is the one import that resolves an unresolved name: no other module + /// has a member of that name, so nothing is left to choose. An editor may add an + /// import like this one as the user types, without asking, as IDEs do for + /// unambiguous imports. + pub unambiguous_import: bool, } /// An edit and the file it applies to. @@ -52,6 +57,7 @@ impl QuickFix { preferred, create: None, add_dependency: None, + unambiguous_import: false, } } } @@ -120,11 +126,19 @@ fn unresolved_fixes( let parsed = parsed_module(db, file.python_file(db)).load(db); let node = covering_node(parsed.syntax().into(), diagnostic_range).node(); let symbol = &node.expr_name()?.id; - Some( - completion::unresolved_fixes(db, file, &parsed, symbol, node) - .into_iter() - .map(move |import| QuickFix::new(import.label, file.file(db), vec![import.edit], true)), - ) + let fixes = completion::unresolved_fixes(db, file, &parsed, symbol, node); + // every qualification names a module one of the imports imports from, so a + // single import is a single thing the name can mean + let unambiguous = fixes.imports.len() == 1; + let imports = fixes.imports.into_iter().map(move |import| QuickFix { + unambiguous_import: unambiguous, + ..QuickFix::new(import.label, file.file(db), vec![import.edit], true) + }); + let qualifications = fixes + .qualifications + .into_iter() + .map(move |import| QuickFix::new(import.label, file.file(db), vec![import.edit], true)); + Some(imports.chain(qualifications)) } #[cfg(test)] @@ -807,6 +821,7 @@ mod tests { 2 | dt.timedelta(days=1) | ^^ help: This is a preferred code action + help: This is the only import that resolves the name | 1 + import datetime as dt 2 | @@ -902,6 +917,7 @@ mod tests { 2 | ExecutionLoader | ^^^^^^^^^^^^^^^ help: This is a preferred code action + help: This is the only import that resolves the name | 1 + from importlib.abc import ExecutionLoader 2 | @@ -940,6 +956,7 @@ mod tests { 3 | ExecutionLoader | ^^^^^^^^^^^^^^^ help: This is a preferred code action + help: This is the only import that resolves the name | 1 + from importlib.abc import ExecutionLoader 2 | @@ -975,6 +992,7 @@ mod tests { 3 | ExecutionLoader | ^^^^^^^^^^^^^^^ help: This is a preferred code action + help: This is the only import that resolves the name | 1 + from importlib.abc import ExecutionLoader 2 | @@ -1093,6 +1111,13 @@ mod tests { )); } + if action.unambiguous_import { + diagnostic.sub(SubDiagnostic::new( + ruff_db::diagnostic::SubDiagnosticSeverity::Help, + "This is the only import that resolves the name", + )); + } + let mut edits = action.edits.into_iter().map(|edit| edit.edit); if let Some(first) = edits.next() { diagnostic.set_fix(Fix::safe_edits(first, edits)); diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 0db6380f12..ddfa0dd28f 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -2811,6 +2811,17 @@ pub(crate) struct ImportEdit { pub edit: Edit, } +/// The fixes that would resolve an unresolved name, by how each one does it. +pub(crate) struct UnresolvedFixes { + /// Imports that bind the name: one for each module with a member of that name, and the + /// import of a module under the common alias the name is, like `import numpy as np`. + pub imports: Vec, + /// Rewrites of the name into an attribute of a module the file imports already, like + /// `re.search` for `search` when the file has `import re`. Each names a module one of + /// [`Self::imports`] imports from too. + pub qualifications: Vec, +} + /// Get fixes that would resolve an unresolved reference pub(crate) fn unresolved_fixes<'db>( db: &'db dyn Db, @@ -2818,8 +2829,7 @@ pub(crate) fn unresolved_fixes<'db>( parsed: &ParsedModuleRef, symbol: &str, node: AnyNodeRef, -) -> Vec { - let mut results = Vec::new(); +) -> UnresolvedFixes { let query = UserQuery::exactly(symbol); let ctx = CollectionContext::none(); @@ -2843,7 +2853,7 @@ pub(crate) fn unresolved_fixes<'db>( }, &mut completions, ); - results.extend(completions.into_imports()); + let mut imports = completions.into_imports(); // Request qualifications we could apply to the symbol to make it resolve let mut completions = Completions::new(db, file, ctx, query); @@ -2858,13 +2868,16 @@ pub(crate) fn unresolved_fixes<'db>( }, &mut completions, ); - results.extend(completions.into_qualifications(node.range())); + let qualifications = completions.into_qualifications(node.range()); // an unresolved `np` is the name a common alias would have bound, so offer to write the // import that binds it — the same one the completions above `np.` carry - results.extend(alias_import(db, file, &importer, &members, symbol)); + imports.extend(alias_import(db, file, &importer, &members, symbol)); - results + UnresolvedFixes { + imports, + qualifications, + } } /// The import that would bind `symbol`, when it is a common alias of a module the project has. diff --git a/crates/ty_ide/src/django_template/diagnostics.rs b/crates/ty_ide/src/django_template/diagnostics.rs index 97a5f60370..fe5a885edd 100644 --- a/crates/ty_ide/src/django_template/diagnostics.rs +++ b/crates/ty_ide/src/django_template/diagnostics.rs @@ -1044,6 +1044,7 @@ pub(crate) fn code_actions( preferred: true, create: Some(destination), add_dependency: None, + unambiguous_import: false, }); } @@ -1059,6 +1060,7 @@ pub(crate) fn code_actions( preferred: false, create: None, add_dependency: None, + unambiguous_import: false, }); actions diff --git a/crates/ty_ide/src/importer.rs b/crates/ty_ide/src/importer.rs index 5363acaa7a..d91b939dfa 100644 --- a/crates/ty_ide/src/importer.rs +++ b/crates/ty_ide/src/importer.rs @@ -152,11 +152,7 @@ impl<'a> Importer<'a> { self.file.resolver_environment(self.db), ); let request = request.avoid_conflicts(self.db, importing_file, members); - let mut symbol_text: Box = request - .member - .or(request.asname) - .unwrap_or(request.module) - .into(); + let mut symbol_text: Box = request.symbol_text().into(); let Some(response) = self.find(importing_file, &request, members.at) else { let insertion = if let Some(future) = self.find_last_future_import(members.at) { Insertion::end_of_statement(future.stmt, self.source, self.stylist) @@ -206,17 +202,18 @@ impl<'a> Importer<'a> { let module = alias.asname.as_ref().unwrap_or(&alias.name).as_str(); ImportAction { import: None, - symbol_text: format!("{module}.{symbol_text}").into(), + symbol_text: format!("{module}.{member}").into(), } } ImportResponseKind::Partial(ast) => { + let imported = request.imported_member(); let import = if let Some(insertion) = Insertion::existing_import(response.import.stmt, self.tokens) { - insertion.into_edit(member) + insertion.into_edit(&imported) } else { Insertion::end_of_statement(response.import.stmt, self.source, self.stylist) - .into_edit(&format!("from {} import {member}", request.module)) + .into_edit(&format!("from {} import {imported}", request.module)) }; ImportAction { import: Some(import), @@ -386,7 +383,7 @@ impl<'ast> MembersInScope<'ast> { importing_file: ImportingFile<'_>, request: &ImportRequest<'_>, ) -> bool { - let symbol_text = request.member.or(request.asname).unwrap_or(request.module); + let symbol_text = request.symbol_text(); let Some(member) = self.find_member(symbol_text) else { return false; }; @@ -562,7 +559,15 @@ impl<'ast> AstImportKind<'ast> { let kind = ast .names .iter() - .find(|alias| alias.name.as_str() == "*" || alias.name.as_str() == member) + .find(|alias| match request.asname { + // `from x import y` binds `y`, not the `z` that was asked for + Some(asname) => { + alias.name.as_str() == member + && alias.asname.as_ref().map(ast::Identifier::as_str) + == Some(asname) + } + None => alias.name.as_str() == "*" || alias.name.as_str() == member, + }) .map(|alias| ImportResponseKind::Unqualified { ast, alias }) .unwrap_or_else(|| ImportResponseKind::Partial(ast)); Some(kind) @@ -582,10 +587,8 @@ pub(crate) struct ImportRequest<'a> { /// When `member` is absent, then this request reflects an import /// of the module itself. i.e., `import module`. member: Option<&'a str>, - /// The name to bind the module to (e.g., `np`, in `import numpy as np`). - /// - /// Only a request for a module itself carries one, since that is the - /// only import an alias is asked for today. + /// The name to bind the module or member to (e.g., `np`, in `import + /// numpy as np`, or `PP` in `from pathlib import PurePath as PP`). asname: Option<&'a str>, /// The preferred style to use when importing the symbol (e.g., /// `import foo` or `from foo import bar`). @@ -629,6 +632,33 @@ impl<'a> ImportRequest<'a> { } } + /// Create a new [`ImportRequest`] from a module and member, bound + /// under a name of its own, as in `from pathlib import PurePath as PP`. + /// + /// If `module` has no existing imports, the symbol should be + /// imported using the `import from` statement. + pub(crate) fn import_from_as(module: &'a str, member: &'a str, asname: &'a str) -> Self { + Self { + asname: Some(asname), + ..Self::import_from(module, member) + } + } + + /// The text the requested symbol is referred to by once it is + /// imported as asked: its alias, the member, or the module. + fn symbol_text(&self) -> &'a str { + self.asname.or(self.member).unwrap_or(self.module) + } + + /// The member as a `from ... import` lists it: `member`, or + /// `member as asname`. + fn imported_member(&self) -> String { + match (self.member, self.asname) { + (Some(member), Some(asname)) => format!("{member} as {asname}"), + (member, _) => member.unwrap_or(self.module).to_string(), + } + } + /// Create a new [`ImportRequest`] for bringing the given module /// into scope. /// @@ -678,6 +708,12 @@ impl<'a> ImportRequest<'a> { ..self }; }; + // a member asked for under an alias can only be imported by a + // `from ... import ... as ...`, so there is no other style to fall + // back on: whoever asks for an alias has chosen the name it binds + if self.asname.is_some() { + return self; + } match (members.map.get(self.module), members.map.get(member)) { // Neither symbol exists, so we can just proceed as // normal. @@ -742,8 +778,8 @@ impl<'a> ImportRequest<'a> { impl std::fmt::Display for ImportRequest<'_> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match (&self.style, self.member) { - (ImportStyle::ImportFrom, Some(member)) => { - write!(f, "from {} import {member}", self.module) + (ImportStyle::ImportFrom, Some(_)) => { + write!(f, "from {} import {}", self.module, self.imported_member()) } _ => { write!(f, "import {}", self.module)?; diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 293bfd6f8b..9cd6e6a488 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -30,6 +30,7 @@ mod inlay_hints; mod markup; mod module_rename; mod override_members; +mod pasted_imports; mod refactor; mod references; mod rename; @@ -88,6 +89,7 @@ pub use override_members::{ ClassMembersToOverride, MemberOverrides, MemberToOverride, members_to_override, override_members, }; +pub use pasted_imports::{CopiedCode, CopiedImport, PastedImports, copied_imports, pasted_imports}; pub use refactor::{Refactor, RefactorKind, RefactorOffer, refactor, refactors}; pub use references::ReferencesMode; pub use rename::{PreparedRename, can_rename, invalid_new_name, prepare_rename, rename}; diff --git a/crates/ty_ide/src/pasted_imports.rs b/crates/ty_ide/src/pasted_imports.rs new file mode 100644 index 0000000000..ce6683a9ef --- /dev/null +++ b/crates/ty_ide/src/pasted_imports.rs @@ -0,0 +1,710 @@ +//! The imports copied code depends on, and the ones it is missing where it is pasted. +//! +//! Code copied out of one module and pasted into another loses the imports it was +//! written against: a function that calls `Path(...)` under `from pathlib import Path` +//! is a function with an unresolved `Path` once it lands in a module without that +//! import. So an editor asks two questions. [`copied_imports`] answers which of the +//! names the copied text uses are bound by the global scope of the file it came from, +//! where it came from, and how to bind each of them again elsewhere. [`pasted_imports`] +//! answers which of those the file the text landed in does not bind where it landed, +//! and the edits that import them. +//! +//! The first is asked about the copied text, with the place it was copied from, rather +//! than about a range of the file: cutting a function is an edit that takes the text +//! out of the file, and the question is only worth asking once the paste has happened. +//! Its answer describes each import rather than pointing into the source file, so the +//! second question can be asked about another file. + +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_diagnostics::Edit; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr, ExprContext, Stmt}; +use ruff_python_codegen::Stylist; +use ruff_python_parser::{ParseOptions, parse_unchecked}; +use ruff_python_trivia::textwrap::dedent; +use ruff_text_size::{TextRange, TextSize}; +use rustc_hash::FxHashSet; +use ty_module_resolver::{ImportingFile, ModuleName, file_to_module}; +use ty_project::Db; +use ty_python_core::ProgramFile; +use ty_python_core::definition::{Definition, DefinitionKind}; +use ty_python_semantic::SemanticModel; + +use crate::importer::{ImportRequest, Importer}; + +/// A name copied code uses, and the import that binds it. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CopiedImport { + /// The name the copied code uses. + pub name: String, + /// The module to import, by its absolute name: a relative import in the file the + /// code was copied from means nothing from anywhere else. + pub module: String, + /// What to import from [`Self::module`], when the name is bound to a member of it + /// rather than to the module itself. A member imported under another name keeps + /// its own name here, and [`Self::name`] is the alias. + pub member: Option, +} + +impl CopiedImport { + /// How [`Importer::import`] is asked for this import. + fn request(&self) -> ImportRequest<'_> { + match &self.member { + Some(member) if *member == self.name => { + ImportRequest::import_from(&self.module, member).force() + } + Some(member) => ImportRequest::import_from_as(&self.module, member, &self.name).force(), + // `import a.b` binds `a`, so the import that binds `a` again is `import a.b`, + // which keeps `a.b` an attribute the copied code can read + None if self.module.split('.').next() == Some(self.name.as_str()) => { + ImportRequest::module(&self.module) + } + None => ImportRequest::module_as(&self.module, &self.name), + } + } +} + +/// Code copied out of a file: its text, and the offset in the file it was copied from. +#[derive(Debug, Clone, Copy)] +pub struct CopiedCode<'a> { + pub text: &'a str, + pub at: TextSize, +} + +/// The names `copies` use that `file`'s global scope binds where each was copied from, +/// and the import that binds each of them again in another module. +/// +/// The copied text is read on its own, not out of `file`: what a cut takes is no +/// longer in the file by the time anything is asked about it, and what a copy took +/// may have been edited since. It is the scope at [`CopiedCode::at`] that says what +/// each name meant, and a cut leaves that scope's bindings where they were. +/// +/// A name bound by an import is imported the same way, from the module that import +/// names. A name `file` defines itself — a function, a class, a module-level +/// variable — is imported from `file`'s own module, when `file` is a module anything +/// can import. Names the copied code binds itself travel with it, and names of any +/// scope but the global one, or the builtins, cannot be imported, so neither is +/// answered. Each name is answered once, in the order the code first uses it. +pub fn copied_imports( + db: &dyn Db, + file: ProgramFile<'_>, + copies: &[CopiedCode<'_>], +) -> Vec { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let model = SemanticModel::new(db, file); + let importing_file = ImportingFile::File(file.file(db), file.resolver_environment(db)); + let own_module = file_to_module(db, file.resolver_file(db)) + .map(|module| module.name(db).as_str().to_string()); + let options = ParseOptions::from(file.file(db).source_type(db)).with_basedpython(matches!( + file.file(db).path(db).extension(), + Some("by" | "byi") + )); + + let mut seen = FxHashSet::default(); + let mut imports = Vec::new(); + for copy in copies { + let Some(at) = (copy.at <= TextSize::of(source.as_str())).then_some(copy.at) else { + continue; + }; + let snippet = snippet_source(source.as_str(), at, copy.text); + let snippet = parse_unchecked(&snippet, options.clone()); + let mut names = SnippetNames::default(); + names.visit_mod(snippet.syntax()); + + // what each name meant where the code was copied from: the bindings of every scope + // around that place, innermost first, so that a local shadows the global it names + let scope = covering_node(parsed.syntax().into(), TextRange::empty(at)).node(); + let members = model.members_in_scope_at(scope); + + for name in names.read { + if names.bound.contains(&name) || !seen.insert(name.clone()) { + continue; + } + let Some(member) = members.get(name.as_str()) else { + continue; + }; + let definition = member.first_reachable_definition; + if definition.file(db) != file.file(db) || !definition.file_scope(db).is_global() { + continue; + } + imports.extend(import_of( + db, + importing_file, + &parsed, + own_module.as_deref(), + &name, + definition, + )); + } + } + imports +} + +/// The import that binds `name` as `definition`, a global binding of the file `parsed` is, +/// binds it. +fn import_of( + db: &dyn Db, + importing_file: ImportingFile<'_>, + parsed: &ParsedModuleRef, + own_module: Option<&str>, + name: &str, + definition: Definition<'_>, +) -> Option { + match definition.kind(db) { + DefinitionKind::Import(import) => { + let alias = import.alias(parsed); + Some(CopiedImport { + name: name.to_string(), + module: alias.name.to_string(), + member: None, + }) + } + DefinitionKind::ImportFrom(import) => { + absolute_module(db, importing_file, import.import(parsed)).map(|module| CopiedImport { + name: name.to_string(), + module, + member: Some(import.alias(parsed).name.to_string()), + }) + } + DefinitionKind::StarImport(import) => { + absolute_module(db, importing_file, import.import(parsed)).map(|module| CopiedImport { + name: name.to_string(), + module, + member: Some(name.to_string()), + }) + } + // `from a.b import c` also binds `a` in `a/__init__`, which is not a name code + // outside that package reads + DefinitionKind::ImportFromSubmodule(_) => None, + _ => own_module.map(|module| CopiedImport { + name: name.to_string(), + module: module.to_string(), + member: Some(name.to_string()), + }), + } +} + +/// `text`, copied from `at` in `source`, as source that parses on its own. +/// +/// A copy usually starts at the first character of a line's code, after the +/// indentation, and takes whole lines after that, each indented as it was: `return +/// x\n y = 1`. Read alone that is an unexpected indent, so the copy is given back +/// the indentation its first line had, as spaces, and then the whole is dedented. +fn snippet_source(source: &str, at: TextSize, text: &str) -> String { + let line_start = source[..at.to_usize()] + .rfind('\n') + .map_or(0, |newline| newline + 1); + let before = &source[line_start..at.to_usize()]; + let indent = if before.chars().all(char::is_whitespace) { + before.to_string() + } else { + " ".repeat(before.chars().count()) + }; + dedent(&format!("{indent}{text}")).into_owned() +} + +/// The names copied code reads, in the order it reads them, and the names it binds. +#[derive(Default)] +struct SnippetNames { + read: Vec, + bound: FxHashSet, +} + +impl<'a> SourceOrderVisitor<'a> for SnippetNames { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr { + match name.ctx { + ExprContext::Load => self.read.push(name.id.to_string()), + ExprContext::Store | ExprContext::Del | ExprContext::Invalid => { + self.bound.insert(name.id.to_string()); + } + } + } + source_order::walk_expr(self, expr); + } + + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::FunctionDef(function) => { + self.bound.insert(function.name.to_string()); + } + Stmt::ClassDef(class) => { + self.bound.insert(class.name.to_string()); + } + _ => {} + } + source_order::walk_stmt(self, stmt); + } + + fn visit_parameter(&mut self, parameter: &'a ast::Parameter) { + self.bound.insert(parameter.name.to_string()); + source_order::walk_parameter(self, parameter); + } + + fn visit_alias(&mut self, alias: &'a ast::Alias) { + let bound = alias.asname.as_ref().unwrap_or(&alias.name); + let bound = bound.as_str().split('.').next().unwrap_or_default(); + self.bound.insert(bound.to_string()); + } + + fn visit_except_handler(&mut self, handler: &'a ast::ExceptHandler) { + let ast::ExceptHandler::ExceptHandler(handler_inner) = handler; + if let Some(name) = &handler_inner.name { + self.bound.insert(name.to_string()); + } + source_order::walk_except_handler(self, handler); + } + + fn visit_pattern(&mut self, pattern: &'a ast::Pattern) { + match pattern { + ast::Pattern::MatchAs(ast::PatternMatchAs { + name: Some(name), .. + }) + | ast::Pattern::MatchStar(ast::PatternMatchStar { + name: Some(name), .. + }) + | ast::Pattern::MatchMapping(ast::PatternMatchMapping { + rest: Some(name), .. + }) => { + self.bound.insert(name.to_string()); + } + _ => {} + } + source_order::walk_pattern(self, pattern); + } + + fn visit_type_param(&mut self, type_param: &'a ast::TypeParam) { + self.bound.insert(type_param.name().to_string()); + source_order::walk_type_param(self, type_param); + } +} + +/// The absolute name of the module `import` imports from. +fn absolute_module( + db: &dyn Db, + importing_file: ImportingFile<'_>, + import: &ast::StmtImportFrom, +) -> Option { + ModuleName::from_import_statement(db, importing_file, import) + .ok() + .map(|module| module.as_str().to_string()) +} + +/// What [`pasted_imports`] answers: the imports the pasted code is missing, and the +/// edits that add them. +#[derive(Debug, Default)] +pub struct PastedImports { + /// The imports asked about that the pasted code is missing, in the order asked. + pub missing: Vec, + /// The edits that add every import of [`Self::missing`], all to the file the code + /// was pasted into and all computed against its text as it is. + pub edits: Vec, +} + +/// Which of `imports` the code pasted at `range` of `file` is missing, and the edits +/// that add them. +/// +/// An import is missing when its name is not bound in the scope the pasted code +/// landed in. A name that is bound there is left alone even when it is bound to +/// something else: an import would shadow that, or be shadowed by it, and either +/// changes code the paste did not write. An import of a module the file cannot +/// resolve is not offered, nor one of `file`'s own members — those are the names the +/// paste can only have been cut from here, and are gone. +pub fn pasted_imports( + db: &dyn Db, + file: ProgramFile<'_>, + range: TextRange, + imports: &[CopiedImport], +) -> PastedImports { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let stylist = Stylist::from_tokens(parsed.tokens(), source.as_str()); + let importer = Importer::new(db, &stylist, file, source.as_str(), &parsed); + let node = landing_node(&parsed, range); + let members = importer.members_in_scope_at(node, range.start()); + let model = SemanticModel::new(db, file); + let own_module = file_to_module(db, file.resolver_file(db)) + .map(|module| module.name(db).as_str().to_string()); + + let mut answer = PastedImports::default(); + let mut seen = FxHashSet::default(); + for import in imports { + if !seen.insert(import.name.as_str()) + || members.find_member(&import.name).is_some() + || own_module.as_deref() == Some(import.module.as_str()) + { + continue; + } + let resolves = ModuleName::new(&import.module) + .is_some_and(|module| model.resolve_module_name(&module).is_some()); + if !resolves { + continue; + } + let action = importer.import(import.request(), &members); + // an import the file can only satisfy by spelling the name another way — as + // `pathlib.Path` under an `import pathlib` — would leave the pasted name as + // unresolved as it was + if let Some(edit) = action.import() + && action.symbol_text() == import.name + { + answer.missing.push(import.clone()); + answer.edits.push(edit.clone()); + } + } + answer +} + +/// The node whose scope the code pasted at `range` landed in. +fn landing_node(parsed: &ParsedModuleRef, range: TextRange) -> AnyNodeRef<'_> { + covering_node(parsed.syntax().into(), range).node() +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + use ruff_db::files::{File, system_path_to_file}; + use ruff_db::source::source_text; + use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; + use ruff_python_trivia::textwrap::dedent; + use ruff_text_size::{Ranged, TextRange, TextSize}; + use ty_project::{ProjectMetadata, SemanticDb as _}; + + use super::{CopiedCode, CopiedImport, copied_imports, pasted_imports}; + + /// A project of files, each written with its ``/`` markers taken out and kept as + /// that file's ranges. + struct PasteTest { + db: ty_project::TestDb, + } + + impl PasteTest { + fn new(files: &[(&str, &str)]) -> (Self, Vec<(File, Vec)>) { + let mut db = + ty_project::TestDb::new(ProjectMetadata::new("test", SystemPathBuf::from("/"))); + let mut written = Vec::new(); + for (path, source) in files { + let mut text = dedent(source).into_owned(); + let mut ranges = Vec::new(); + while let Some(start) = text.find("") { + text.replace_range(start..start + "".len(), ""); + let end = text.find("").expect("an `` for every ``"); + text.replace_range(end..end + "".len(), ""); + ranges.push(TextRange::new( + TextSize::try_from(start).unwrap(), + TextSize::try_from(end).unwrap(), + )); + } + db.write_file(path, text) + .expect("write to memory file system to be successful"); + let file = system_path_to_file(&db, path).expect("newly written file to exist"); + written.push((file, ranges)); + } + (Self { db }, written) + } + + /// One line per import of the text in `ranges`, copied from where it is: `name <- + /// import statement`. + fn copied(&self, file: File, ranges: &[TextRange]) -> String { + let source = source_text(&self.db, file); + let copies: Vec<_> = ranges + .iter() + .map(|range| CopiedCode { + text: &source[*range], + at: range.start(), + }) + .collect(); + self.copied_code(file, &copies) + } + + /// One line per import of `copies`, copied from `file`. + fn copied_code(&self, file: File, copies: &[CopiedCode<'_>]) -> String { + copied_imports(&self.db, self.db.program_file(file), copies) + .iter() + .map(|import| format!("{} <- {}", import.name, statement(import))) + .collect::>() + .join("\n") + } + + /// The imports missing at `range` of `file`, then the file with the edits applied. + fn pasted(&self, file: File, range: TextRange, imports: &[CopiedImport]) -> String { + let pasted = pasted_imports(&self.db, self.db.program_file(file), range, imports); + let mut text = source_text(&self.db, file).as_str().to_string(); + let mut edits = pasted.edits; + // edits at one position appear in the order given, so the later one goes in first + edits.sort_by_key(Ranged::start); + for edit in edits.iter().rev() { + text.replace_range( + edit.start().to_usize()..edit.end().to_usize(), + edit.content().unwrap_or_default(), + ); + } + let missing = pasted + .missing + .iter() + .map(statement) + .collect::>() + .join("\n"); + format!("{missing}\n---\n{text}") + } + } + + fn statement(import: &CopiedImport) -> String { + match &import.member { + Some(member) if *member == import.name => { + format!("from {} import {member}", import.module) + } + Some(member) => format!("from {} import {member} as {}", import.module, import.name), + None if import.module.split('.').next() == Some(import.name.as_str()) => { + format!("import {}", import.module) + } + None => format!("import {} as {}", import.module, import.name), + } + } + + fn import(name: &str, module: &str, member: Option<&str>) -> CopiedImport { + CopiedImport { + name: name.to_string(), + module: module.to_string(), + member: member.map(str::to_string), + } + } + + /// Every name the copied code reads from the module's global scope is answered with the import + /// that binds it again: the imports the module has, under the names it has them, and the + /// module's own definitions, from the module. A parameter, a name the copied code binds itself + /// and a builtin are not imports at all. + #[test] + fn copied_code_needs_the_imports_and_definitions_of_its_module() { + let (test, files) = PasteTest::new(&[( + "shapes.py", + r#" + import math + import os.path + import numbers as num + from pathlib import Path as P + from collections import * + + SCALE = 2 + + def helper() -> int: + return 1 + + def area(r: float) -> float: + local = math.pi * r * SCALE + print(P(os.path.join("a")), helper(), local, deque, num.Number) + return local + "#, + )]); + let (file, ranges) = &files[0]; + + assert_snapshot!(test.copied(*file, ranges), @" + math <- import math + SCALE <- from shapes import SCALE + P <- from pathlib import Path as P + os <- import os.path + helper <- from shapes import helper + deque <- from collections import deque + num <- import numbers as num + "); + } + + /// A definition the copied ranges include travels with them, and needs no import. + #[test] + fn a_definition_copied_with_its_uses_needs_no_import() { + let (test, files) = PasteTest::new(&[( + "shapes.py", + r#" + def helper() -> int: + return 1 + + def area() -> int: + return helper() + "#, + )]); + let (file, ranges) = &files[0]; + + assert_snapshot!(test.copied(*file, ranges), @""); + } + + /// Cut code is no longer in the file it came from. What its names meant is what the scope it + /// was cut from still binds, and the imports at the top of the file outlive the cut. + #[test] + fn cut_code_needs_what_the_place_it_was_cut_from_binds() { + let (test, files) = PasteTest::new(&[( + "shapes.py", + r#" + import math + + def area(r: float) -> float: + + + def where(): ... + "#, + )]); + let (file, ranges) = &files[0]; + let cut = CopiedCode { + text: "return math.pi * r * r * where()\n x = 1", + at: ranges[0].start(), + }; + + assert_snapshot!(test.copied_code(*file, &[cut]), @" + math <- import math + where <- from shapes import where + "); + } + + /// A relative import is answered by the absolute module it names, which is what the code needs + /// from anywhere else. + #[test] + fn a_relative_import_is_answered_by_its_absolute_module() { + let (test, files) = PasteTest::new(&[ + ("pkg/__init__.py", ""), + ("pkg/sizes.py", "BIG = 10\n"), + ( + "pkg/shapes.py", + r#" + from .sizes import BIG + + x = BIG + "#, + ), + ]); + let (file, ranges) = &files[2]; + + assert_snapshot!(test.copied(*file, ranges), @"BIG <- from pkg.sizes import BIG"); + } + + /// Several carets copy several ranges, and the answer covers all of them, once per name. + #[test] + fn every_range_copied_is_answered_once_per_name() { + let (test, files) = PasteTest::new(&[( + "main.py", + r#" + import math + import json + + a = math.pi + b = json.dumps(1) + c = math.tau + "#, + )]); + let (file, ranges) = &files[0]; + + assert_snapshot!(test.copied(*file, ranges), @"math <- import math"); + } + + /// Pasted into a module that has some of the imports already, the paste is missing the rest: a + /// member of a module the file imports from goes into that import, and the others are new. + #[test] + fn a_paste_is_missing_the_imports_its_module_does_not_have() { + let (test, files) = PasteTest::new(&[ + ("shapes.py", "SCALE = 2\n"), + ( + "main.py", + r#" + import math + from pathlib import PurePath + + def f(): + return math.pi * SCALE, Path(), P(), np + "#, + ), + ]); + let (file, ranges) = &files[1]; + let imports = [ + import("math", "math", None), + import("SCALE", "shapes", Some("SCALE")), + import("Path", "pathlib", Some("Path")), + import("P", "pathlib", Some("PurePosixPath")), + import("np", "numbers", None), + ]; + + assert_snapshot!(test.pasted(*file, ranges[0], &imports), @" + from shapes import SCALE + from pathlib import Path + from pathlib import PurePosixPath as P + import numbers as np + --- + from shapes import SCALE + import numbers as np + + import math + from pathlib import PurePath, Path, PurePosixPath as P + + def f(): + return math.pi * SCALE, Path(), P(), np + "); + } + + /// A name bound where the code lands is left alone, whatever it is bound to: an import would + /// change what the code around the paste reads. + #[test] + fn a_name_bound_where_the_paste_lands_is_not_imported() { + let (test, files) = PasteTest::new(&[( + "main.py", + r#" + def Path(): + ... + + x = Path() + "#, + )]); + let (file, ranges) = &files[0]; + let imports = [import("Path", "pathlib", Some("Path"))]; + + assert_snapshot!(test.pasted(*file, ranges[0], &imports), @" + + --- + + def Path(): + ... + + x = Path() + "); + } + + /// Code cut from a module and pasted back into it needs nothing from itself: what it named in + /// the module is either still there, and bound, or went with the cut. + #[test] + fn a_paste_into_its_own_module_imports_nothing_from_it() { + let (test, files) = PasteTest::new(&[( + "shapes.py", + r#" + x = helper() + "#, + )]); + let (file, ranges) = &files[0]; + let imports = [import("helper", "shapes", Some("helper"))]; + + assert_snapshot!(test.pasted(*file, ranges[0], &imports), @" + + --- + + x = helper() + "); + } + + /// A module the file it lands in cannot resolve is not imported: the import would be as + /// unresolved as the name. + #[test] + fn a_module_that_does_not_resolve_is_not_imported() { + let (test, files) = PasteTest::new(&[( + "main.py", + r#" + x = thing + "#, + )]); + let (file, ranges) = &files[0]; + let imports = [import("thing", "elsewhere", Some("thing"))]; + + assert_snapshot!(test.pasted(*file, ranges[0], &imports), @" + + --- + + x = thing + "); + } +} diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 5ee6fea20d..06398006c7 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -250,6 +250,19 @@ pub(super) fn request(req: server::Request) -> Task { BackgroundSchedule::LatencySensitive, ) } + // asked as text is copied and pasted, with the user waiting on the paste to settle + requests::CopiedImportsRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } + requests::PastedImportsRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } requests::PrepareTypeHierarchyRequestHandler::METHOD => background_document_request_task::< requests::PrepareTypeHierarchyRequestHandler, >( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 0de96f8ff9..5e2d4dc136 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -21,6 +21,7 @@ mod code_action; mod code_action_resolve; mod code_lens; mod completion; +mod copied_imports; mod data_flow; mod detached_transpile; mod diagnostic; @@ -44,6 +45,7 @@ mod inlay_hints; mod module_name; mod overridable_members; mod override_members; +mod pasted_imports; mod prepare_call_hierarchy; mod prepare_rename; mod prepare_type_hierarchy; @@ -77,6 +79,7 @@ pub(super) use code_action::CodeActionRequestHandler; pub(super) use code_action_resolve::CodeActionResolveRequestHandler; pub(super) use code_lens::CodeLensRequestHandler; pub(super) use completion::CompletionRequestHandler; +pub(super) use copied_imports::CopiedImportsRequestHandler; pub(super) use data_flow::DataFlowRequestHandler; pub(super) use diagnostic::DocumentDiagnosticRequestHandler; pub(super) use doc_highlights::DocumentHighlightRequestHandler; @@ -99,6 +102,7 @@ pub(super) use inlay_hints::InlayHintRequestHandler; pub(super) use module_name::ModuleNameHandler; pub(super) use overridable_members::OverridableMembersRequestHandler; pub(super) use override_members::OverrideMembersRequestHandler; +pub(super) use pasted_imports::PastedImportsRequestHandler; pub(super) use prepare_call_hierarchy::PrepareCallHierarchyRequestHandler; pub(super) use prepare_rename::PrepareRenameRequestHandler; pub(super) use prepare_type_hierarchy::PrepareTypeHierarchyRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/code_action.rs b/crates/ty_server/src/server/api/requests/code_action.rs index 536ba75c85..b71c2686d4 100644 --- a/crates/ty_server/src/server/api/requests/code_action.rs +++ b/crates/ty_server/src/server/api/requests/code_action.rs @@ -130,6 +130,8 @@ impl BackgroundDocumentRequestHandler for CodeActionRequestHandler { )]) }); + let data = QuickFixData::of(&action); + // an action that installs something reaches past the files // the editor can edit, so it asks the server to run the // command instead. the client runs a command after the edit @@ -162,7 +164,7 @@ impl BackgroundDocumentRequestHandler for CodeActionRequestHandler { is_preferred: Some(action.preferred), command, disabled: None, - data: None, + data, tags: None, })); } @@ -306,6 +308,36 @@ pub(crate) struct RefactorData { pub(crate) range: types::Range, } +/// What a quick fix's `data` tells a client about it, beyond what the protocol's own +/// fields can say. +/// +/// `isPreferred` cannot say that a fix may be applied without asking: every import +/// that resolves a name is preferred, since each one properly addresses the +/// diagnostic, and a client's "auto fix" still asks which when there are several. +/// `unambiguousImport` is the narrower claim an editor needs before it adds an +/// import as the user types: this is the only import that resolves the name. +/// +/// Sent only on a fix that has something to say, and never on one that is resolved: +/// a quick fix always carries its edit, so a client has no `codeAction/resolve` to +/// send it back in. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct QuickFixData { + unambiguous_import: bool, +} + +impl QuickFixData { + fn of(action: &ty_ide::QuickFix) -> Option { + if !action.unambiguous_import { + return None; + } + serde_json::to_value(QuickFixData { + unambiguous_import: true, + }) + .ok() + } +} + /// Whether a client that asked only for `only` wants actions of `kind`: a /// requested kind covers itself and every kind nested under it. fn is_requested(only: Option<&[CodeActionKind]>, kind: &CodeActionKind) -> bool { diff --git a/crates/ty_server/src/server/api/requests/copied_imports.rs b/crates/ty_server/src/server/api/requests/copied_imports.rs new file mode 100644 index 0000000000..9e20ed1c43 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/copied_imports.rs @@ -0,0 +1,146 @@ +//! `by/copiedImports`: the imports some code copied out of a document depends on +//! +//! what an editor asks once copied text has been pasted, so that the paste can import what the code +//! needs where it landed: `by/pastedImports` takes the answer and says which of the imports the +//! paste is missing. the request carries the copied text and the place in the document it was copied +//! from, rather than a range of the document: a cut takes the text out of the document, and the +//! names in it meant what the scope at that place binds, which the cut leaves where it was. the +//! answer describes each import — a module and a member — rather than pointing into this document, +//! so it means the same thing in another one +//! +//! ask with `textHash` naming the text the positions are in, the document as it is when asked + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Request, TextDocumentIdentifier, Uri, +}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::PositionExt; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum CopiedImportsRequest {} + +impl Request for CopiedImportsRequest { + type Params = CopiedImportsParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/copiedImports"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the document text was copied from, and what was copied: one piece, or one per caret +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct CopiedImportsParams { + text_document: TextDocumentIdentifier, + copies: Vec, +} + +/// a piece of copied text, and the position in the document it was copied from: where its first +/// character was +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct Copied { + text: String, + position: Position, +} + +/// a name the copied code uses, and the import that binds it +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CopiedImport { + /// the name the copied code uses + name: String, + /// the module to import, by its absolute name + module: String, + /// what to import from the module, when the name is bound to a member of it rather than to the + /// module itself: the member's own name, which differs from `name` when it was imported under + /// an alias. absent for `import module` and `import module as name` + #[serde(default, skip_serializing_if = "Option::is_none")] + member: Option, +} + +impl From for CopiedImport { + fn from(import: ty_ide::CopiedImport) -> Self { + Self { + name: import.name, + module: import.module, + member: import.member, + } + } +} + +impl From for ty_ide::CopiedImport { + fn from(import: CopiedImport) -> Self { + Self { + name: import.name, + module: import.module, + member: import.member, + } + } +} + +pub(crate) struct CopiedImportsRequestHandler; + +impl RequestHandler for CopiedImportsRequestHandler { + type RequestType = CopiedImportsRequest; +} + +impl BackgroundDocumentRequestHandler for CopiedImportsRequestHandler { + fn document_uri(params: &CopiedImportsParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, or for a document whose code cannot be pasted into a + /// module: a notebook, whose positions are per cell, or a template. otherwise each name the + /// copied text uses that the document's global scope binds where it was copied from — by an + /// import, or by defining it, when the document is a module others can import — once, in the + /// order the text first uses them, and an empty list when there are none. a piece whose + /// position is not in the document is left out + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: CopiedImportsParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let copies: Vec<_> = params + .copies + .iter() + .filter_map(|copy| { + let at = + copy.position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding())?; + Some(ty_ide::CopiedCode { + text: ©.text, + at, + }) + }) + .collect(); + + Ok(Some( + ty_ide::copied_imports(db, db.program_file(file), &copies) + .into_iter() + .map(CopiedImport::from) + .collect(), + )) + } +} + +impl RetriableRequestHandler for CopiedImportsRequestHandler {} diff --git a/crates/ty_server/src/server/api/requests/pasted_imports.rs b/crates/ty_server/src/server/api/requests/pasted_imports.rs new file mode 100644 index 0000000000..343301abfd --- /dev/null +++ b/crates/ty_server/src/server/api/requests/pasted_imports.rs @@ -0,0 +1,126 @@ +//! `by/pastedImports`: which of the imports copied code depends on it is missing where it was +//! pasted, and the edits that add them +//! +//! the second half of `by/copiedImports`: an editor asks it once the text has landed, naming the +//! range it landed in and the imports `by/copiedImports` answered when it was copied. an import is +//! missing when its name is not bound in the scope the paste landed in; a name that is bound there +//! is left alone even when it is bound to something else, since an import would change what the +//! code around the paste reads +//! +//! the edits add every missing import together, against the document's text as the request names +//! it. a client that lets its user choose among them asks again with only the ones chosen, rather +//! than applying some of the edits: two imports from one module can be added to the same +//! statement, and the edits of a subset are not a subset of the edits + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Range, Request, TextDocumentIdentifier, TextEdit, Uri, +}; +use ruff_text_size::Ranged; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{RangeExt, ToRangeExt}; +use crate::server::api::requests::copied_imports::CopiedImport; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum PastedImportsRequest {} + +impl Request for PastedImportsRequest { + type Params = PastedImportsParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/pastedImports"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the document text was pasted into, the range it now occupies there, and the imports +/// `by/copiedImports` answered for it +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct PastedImportsParams { + text_document: TextDocumentIdentifier, + range: Range, + imports: Vec, +} + +/// the imports the pasted code is missing, and the edits to the document that add all of them +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PastedImports { + /// the ones of the imports asked about that are missing, in the order asked + imports: Vec, + /// edits to the document, which add every one of `imports` + edits: Vec, +} + +pub(crate) struct PastedImportsRequestHandler; + +impl RequestHandler for PastedImportsRequestHandler { + type RequestType = PastedImportsRequest; +} + +impl BackgroundDocumentRequestHandler for PastedImportsRequestHandler { + fn document_uri(params: &PastedImportsParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, for a document no import can be added to — a + /// notebook or a template — or for a range that is not in the document. otherwise the missing + /// imports and their edits, both empty when nothing is missing + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: PastedImportsParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let encoding = snapshot.encoding(); + let Some(range) = params + .range + .to_text_range(db, file, snapshot.uri(), encoding) + else { + return Ok(None); + }; + let imports: Vec = + params.imports.into_iter().map(Into::into).collect(); + + let pasted = ty_ide::pasted_imports(db, db.program_file(file), range, &imports); + let edits = pasted + .edits + .iter() + .map(|edit| { + Some(TextEdit { + range: edit.range().to_lsp_range(db, file, encoding)?.local_range(), + new_text: edit.content().unwrap_or_default().to_string(), + }) + }) + .collect::>>(); + // an edit with no place in the document would leave its import out of what the edits + // add, and the answer would no longer say what they do + let Some(edits) = edits else { + return Ok(None); + }; + + Ok(Some(PastedImports { + imports: pasted.missing.into_iter().map(CopiedImport::from).collect(), + edits, + })) + } +} + +impl RetriableRequestHandler for PastedImportsRequestHandler {} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 5f9c8cb17f..412b2f062e 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -50,6 +50,7 @@ mod inlay_hints; mod named_text; mod notebook; mod override_members; +mod pasted_imports; mod program_model; mod project_server; mod publish_diagnostics; diff --git a/crates/ty_server/tests/e2e/pasted_imports.rs b/crates/ty_server/tests/e2e/pasted_imports.rs new file mode 100644 index 0000000000..22c33ecd1f --- /dev/null +++ b/crates/ty_server/tests/e2e/pasted_imports.rs @@ -0,0 +1,168 @@ +//! `by/copiedImports` and `by/pastedImports` over the wire, and the quick fix a client may add +//! without asking +//! +//! which names copied code needs and which a paste is missing is `ty_ide`'s and its unit tests cover +//! it. what these cover is the contract a client reads: the copied text and where it came from for +//! the copy, the imports out as names, modules and members, the same imports back in for the paste with +//! the edits that add the missing ones — and the `unambiguousImport` mark in a quick fix's `data`, +//! on the one import that resolves a name and on nothing else + +use anyhow::Result; +use lsp_types::{ + CodeActionRequest, DocumentDiagnosticReport, LspRequestMethod, MessageDirection, Position, + Range, Request, +}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder, text_hash}; + +/// `by/copiedImports`, in json throughout +enum CopiedImports {} + +impl Request for CopiedImports { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/copiedImports"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// `by/pastedImports`, in json throughout +enum PastedImports {} + +impl Request for PastedImports { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/pastedImports"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const SHAPES: &str = "\ +import math +from pathlib import Path + +SCALE = 2 + +def area(r: float) -> float: + return math.pi * r * r * SCALE +"; + +const MAIN: &str = "\ +from pathlib import PurePath + +def f() -> float: + return math.pi * SCALE +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes.py"), SHAPES)? + .with_file(SystemPath::new("src/main.py"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/shapes.py"), SHAPES, 1); + server.open_text_document(SystemPath::new("src/main.py"), MAIN, 1); + Ok(server) +} + +#[test] +fn a_copy_answers_its_imports_and_a_paste_the_ones_it_is_missing() -> Result<()> { + let mut server = server()?; + + // the body of `area`, as it was copied from where it starts + let copied = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/shapes.py")) }, + "copies": [{ + "text": "return math.pi * r * r * SCALE", + "position": { "line": 6, "character": 4 }, + }], + "textHash": text_hash(SHAPES), + })) + .expect("a python document is answered"); + assert_eq!( + copied, + serde_json::json!([ + { "name": "math", "module": "math" }, + { "name": "SCALE", "module": "shapes", "member": "SCALE" }, + ]), + ); + + // pasted as the body of `f`, which has neither + let pasted = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/main.py")) }, + "range": { "start": { "line": 3, "character": 4 }, "end": { "line": 3, "character": 26 } }, + "imports": copied, + "textHash": text_hash(MAIN), + })) + .expect("a python document is answered"); + assert_eq!(pasted["imports"], copied, "{pasted:#}"); + let added: Vec<_> = pasted["edits"] + .as_array() + .expect("a list of edits") + .iter() + .map(|edit| edit["newText"].as_str().unwrap_or_default().to_string()) + .collect(); + assert_eq!( + added, + ["import math\n", "from shapes import SCALE\n"], + "{pasted:#}" + ); + Ok(()) +} + +#[test] +fn only_the_one_import_that_resolves_a_name_is_marked() -> Result<()> { + let main = "\ +x = defaultdict(list) +y = deprecated +"; + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("ty.toml"), "")? + .with_file(SystemPath::new("src/main.py"), main)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.py"), main, 1); + + let diagnostics = match server.document_diagnostic_request(SystemPath::new("src/main.py"), None) + { + DocumentDiagnosticReport::RelatedFullDocumentDiagnosticReport(report) => { + report.full_document_diagnostic_report.items + } + DocumentDiagnosticReport::RelatedUnchangedDocumentDiagnosticReport(_) => { + panic!("Expected full diagnostic report") + } + }; + let id = server.send_request::(lsp_types::CodeActionParams { + text_document: lsp_types::TextDocumentIdentifier { + uri: server.file_uri(SystemPath::new("src/main.py")), + }, + range: Range::new(Position::new(0, 0), Position::new(2, 0)), + context: lsp_types::CodeActionContext { + diagnostics, + only: Some(vec![lsp_types::CodeActionKind::QuickFix]), + trigger_kind: None, + }, + work_done_progress_params: lsp_types::WorkDoneProgressParams::default(), + partial_result_params: lsp_types::PartialResultParams::default(), + }); + let actions = server + .await_response::(&id) + .expect("quick fixes"); + + let actions = serde_json::to_value(actions)?; + let marked: Vec<_> = actions + .as_array() + .expect("a list of actions") + .iter() + .filter(|action| action["data"] == serde_json::json!({ "unambiguousImport": true })) + .map(|action| action["title"].as_str().unwrap_or_default()) + .collect(); + // `deprecated` is in both `warnings` and `typing_extensions`, so it is not marked + assert_eq!(marked, ["import collections.defaultdict"]); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap index 18f08b423c..67f77cb3ea 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap @@ -46,6 +46,9 @@ expression: code_actions } ] } + }, + "data": { + "unambiguousImport": true } }, { diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index dc66c12619..8a968505da 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -564,6 +564,33 @@ end of the body for a position in the header, and replace a body that is only range in the text the edits leave behind, which is where the user goes next. it is `null` when none of the names is a method the class could override +## imports + +an `unresolved-reference` is offered an import for every module with a member +of that name. when there is exactly one, its quick fix carries +`{"unambiguousImport": true}` in its `data`: nothing is left to choose, so an +editor may add it as the code is typed, the way IDEs add unambiguous imports on +the fly. `isPreferred` does not say this — every import that resolves a name is +preferred, since each properly fixes the diagnostic — and the mark changes +nothing for a client that does not read it + +code copied from one module and pasted into another needs the imports it was +written against. `by/copiedImports` takes the copied text and the position in +the document it was copied from, one piece per caret, and answers each name the +text uses that the document's global scope binds there: `name`, the absolute +`module`, and the `member` imported from it, absent when the name is the module. +a name the text binds itself, a local of the scope it came from, and a builtin +are not answered. it takes the text rather than a range because a cut takes the +text out of the document, while the scope it was cut from still binds what its +names meant. `by/pastedImports` takes those imports, the document they were +pasted into and the range the paste now occupies, and answers the ones whose name +is not bound in the scope the paste landed in, with edits that add all of them. +a name bound there to something else is left alone, and so is an import of the +document's own module or of a module it cannot resolve. the edits of some of the +imports are not some of the edits — two imports from one module go into one +statement — so a client that lets its user choose asks again with the ones +chosen. both take `textHash` + ## the whole project's diagnostics with `diagnosticMode` set to `workspace`, the server checks every file in the From b2d2396bfe3ae08f65c023ffd719c9658a520276 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:05:57 +1000 Subject: [PATCH 09/15] move a clause that continues a compound statement out to it when its colon is typed, over textDocument/onTypeFormatting, and complete the statement at a position over by/completeStatement --- crates/ty_ide/src/complete_statement.rs | 722 ++++++++++++++++++ crates/ty_ide/src/lib.rs | 4 + crates/ty_ide/src/on_type_formatting.rs | 557 ++++++++++++++ crates/ty_ide/src/syntax_outline.rs | 20 +- crates/ty_server/src/capabilities.rs | 16 +- crates/ty_server/src/document.rs | 2 +- crates/ty_server/src/document/range.rs | 2 +- crates/ty_server/src/server/api.rs | 14 + crates/ty_server/src/server/api/requests.rs | 4 + .../server/api/requests/complete_statement.rs | 150 ++++ .../server/api/requests/on_type_formatting.rs | 75 ++ .../ty_server/tests/e2e/complete_statement.rs | 136 ++++ crates/ty_server/tests/e2e/main.rs | 2 + .../ty_server/tests/e2e/on_type_formatting.rs | 78 ++ .../e2e__initialize__initialization.snap | 3 + ...ialize__initialization_with_workspace.snap | 3 + docs/basedpython/features/editor.md | 72 ++ 17 files changed, 1847 insertions(+), 13 deletions(-) create mode 100644 crates/ty_ide/src/complete_statement.rs create mode 100644 crates/ty_ide/src/on_type_formatting.rs create mode 100644 crates/ty_server/src/server/api/requests/complete_statement.rs create mode 100644 crates/ty_server/src/server/api/requests/on_type_formatting.rs create mode 100644 crates/ty_server/tests/e2e/complete_statement.rs create mode 100644 crates/ty_server/tests/e2e/on_type_formatting.rs diff --git a/crates/ty_ide/src/complete_statement.rs b/crates/ty_ide/src/complete_statement.rs new file mode 100644 index 0000000000..a973971fad --- /dev/null +++ b/crates/ty_ide/src/complete_statement.rs @@ -0,0 +1,722 @@ +//! Completing the statement at the caret, and starting the line after it. +//! +//! What an editor's *Complete Current Statement* does: the statement being typed is finished off — +//! a string literal closed, the brackets left open on the line closed, the `:` a clause header +//! needs added, a clause that continues a compound statement moved out to it — and a new line is +//! started where the next statement goes: inside the suite a header opens, or after the statement +//! at its own indentation. +//! +//! ```python +//! if handle(item, flags["x" # completed to: +//! +//! if handle(item, flags["x"]): +//! | +//! ``` +//! +//! Which brackets are open, whether a header is missing its `:`, and where a statement ends are +//! all read off the tokens and the parse, so that a `(` in a string or a comment is not a bracket, +//! a `match = 1` is not a `match` statement, and a call spread over several lines is one statement. + +use ruff_db::PythonFile; +use ruff_db::parsed::parsed_module; +use ruff_db::source::source_text; +use ruff_python_ast::token::{Token, TokenKind, Tokens}; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_stmt}; +use ruff_python_ast::{Stmt, StringFlags}; +use ruff_python_codegen::Stylist; +use ruff_python_trivia::leading_indentation; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; + +use crate::Db; +use crate::on_type_formatting::{continuing_clause_reindent, header_colon, indentation_width}; +use crate::syntax_outline::OutlineBuilder; + +/// The statement at the caret, completed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompletedStatement { + /// What to change, in source order and none overlapping another, against the text as it is. + pub edits: Vec, + /// Where the caret goes, in the text the edits produce: the start of the new line, after its + /// indentation. + pub caret: TextSize, +} + +/// One change: `range` replaced with `content`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StatementEdit { + pub range: TextRange, + pub content: String, +} + +/// Completes the statement on the line `offset` is on, and starts the line after it. +/// +/// `indent_unit` is one level of indentation as the file writes it, for the line a header's suite +/// starts on when there is no statement in the suite yet to copy the indentation of. +pub fn complete_statement( + db: &dyn Db, + file: PythonFile<'_>, + offset: TextSize, + indent_unit: &str, +) -> CompletedStatement { + let parsed = parsed_module(db, file).load(db); + let tokens = parsed.tokens(); + let source = source_text(db, file.file(db)); + let source = source.as_str(); + let line_ending = Stylist::from_tokens(tokens, source).line_ending(); + let caret_line = source.line_range(offset); + + let Some(statement) = Statement::at(tokens, caret_line) else { + // Nothing but whitespace or a comment on the line: a new line at its indentation. + let indentation = leading_indentation(&source[caret_line]); + return new_line( + source, + caret_line.end(), + "", + line_ending.as_str(), + indentation, + Vec::new(), + ); + }; + + // The statement ends on the caret's line, unless what is open there — a bracket, a + // triple-quoted string, a backslash — is carried on to a later line that parses. A bracket the + // following lines do not close is not carried on: `foo(a, b` followed by `x = 1` leaves the + // parser reading `x` as a missing comma, and it is the `(` that is missing its `)`. + // The errors that count are the ones inside what is carried on: one where the logical line + // ends — the `:` a header is missing — is the statement's to have completed, not a sign that + // the lines below were never part of it. + let carried_on = statement.logical_end > caret_line.end() + && closes_what_is_open(tokens, statement.first, caret_line, statement.logical_end) + && !parsed.errors().iter().any(|error| { + error.location.start() >= caret_line.end() + && error.location.start() < statement.logical_end + }); + let last_line = if carried_on { + source.line_range(statement.logical_end) + } else { + caret_line + }; + let line = LineTokens::read(tokens, statement.first, last_line); + + let mut fixes = String::new(); + if !carried_on { + for opener in line.open.iter().rev() { + fixes.push_str(opener.closer()); + } + } + + let mut edits = Vec::new(); + let statement_line = source.line_start(statement.start); + let header = Header::on_line( + db, + file, + tokens, + source, + TextRange::new(statement.start, line.code_end), + ); + let indentation = match &header { + Some(header) => { + if header.needs_parentheses { + fixes.push_str("()"); + } + if header.needs_colon { + fixes.push(':'); + } + // A clause that continues a compound statement goes where that statement is. + let reindent = (source.line_start(header.anchor) == statement_line) + .then(|| continuing_clause_reindent(db, file, tokens, statement_line)) + .flatten(); + let header_indentation = match reindent { + Some(reindent) => { + edits.push(StatementEdit { + range: reindent.range, + content: reindent.indentation.clone(), + }); + reindent.indentation + } + None => leading_indentation(&source[source.line_range(header.anchor)]).to_string(), + }; + match header.suite { + // `if x: return` — the suite is done, and the next line is the statement's. + Suite::OnTheLine => header_indentation, + // A suite already written below is indented the way it is; a statement only + // counts as one if it is deeper than the header, rather than a recovery's reading + // of whatever came next. + Suite::Below(first) + if indentation_width(leading_indentation( + &source[source.line_range(first)], + )) > indentation_width(&header_indentation) => + { + leading_indentation(&source[source.line_range(first)]).to_string() + } + Suite::Below(_) | Suite::Empty => header_indentation + indent_unit, + } + } + None => leading_indentation(&source[source.line_range(statement.start)]).to_string(), + }; + + // `foo(a # note` — the fixes go before the comment, and the new line after it. + let new_line_at = match line.comment { + Some(comment) => { + if !fixes.is_empty() { + edits.push(StatementEdit { + range: TextRange::empty(line.code_end), + content: std::mem::take(&mut fixes), + }); + } + comment.end() + } + None => line.code_end, + }; + new_line( + source, + new_line_at, + &fixes, + line_ending.as_str(), + &indentation, + edits, + ) +} + +/// Starts a line at `indentation` at `at`, after `prefix` and `line_ending`, dropping the +/// whitespace that ends the line there; `edits` are the other changes, all before `at`. +fn new_line( + source: &str, + at: TextSize, + prefix: &str, + line_ending: &str, + indentation: &str, + mut edits: Vec, +) -> CompletedStatement { + // only whitespace follows `at` on its line: it is the end of the line's last token + let range = TextRange::new(at, source.line_end(at)); + let content = format!("{prefix}{line_ending}{indentation}"); + let mut caret = range.start() + content.text_len(); + for edit in &edits { + caret = caret + edit.content.text_len() - edit.range.len(); + } + edits.push(StatementEdit { range, content }); + edits.sort_by_key(|edit| edit.range.start()); + CompletedStatement { edits, caret } +} + +/// Whether the brackets left open at the end of `line` are closed before `logical_end`. A logical +/// line can end with brackets still open: at the end of the file, the parser ends it where the +/// brackets ran out of text. +fn closes_what_is_open( + tokens: &Tokens, + first: usize, + line: TextRange, + logical_end: TextSize, +) -> bool { + let open = LineTokens::read(tokens, first, line) + .open + .iter() + .filter(|opener| matches!(opener, Opener::Bracket(_))) + .count(); + let after = tokens.partition_point(|token| token.start() < line.end()); + let depth = tokens[after..] + .iter() + .take_while(|token| token.start() < logical_end) + .fold(open, |depth, token| match token.kind() { + TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace => depth + 1, + TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace => depth.saturating_sub(1), + _ => depth, + }); + depth == 0 +} + +/// The logical line the caret's line belongs to. +struct Statement { + /// The index of its first token. + first: usize, + /// Where its first token starts. + start: TextSize, + /// Where the logical line ends: the start of the `Newline` token that ends it. + logical_end: TextSize, +} + +impl Statement { + /// The statement the line `line` is part of; `None` when there is nothing on it but whitespace + /// and comments. + fn at(tokens: &Tokens, line: TextRange) -> Option { + let before = tokens.partition_point(|token| token.end() <= line.start()); + let after_newline = tokens[..before] + .iter() + .rposition(|token| token.kind() == TokenKind::Newline) + .map_or(0, |index| index + 1); + let first = after_newline + + tokens[after_newline..] + .iter() + .position(|token| !is_layout(token.kind()))?; + let start = tokens[first].start(); + if start > line.end() { + return None; + } + let logical_end = tokens[first..] + .iter() + .find(|token| token.kind() == TokenKind::Newline) + .map_or_else( + || tokens.last().map_or(line.end(), Ranged::end), + Ranged::start, + ); + Some(Self { + first, + start, + logical_end, + }) + } +} + +/// What a statement's tokens say up to the end of the line the statement is completed on. +struct LineTokens { + /// The brackets and single-quoted strings still open at the end of that line, outermost + /// first. + open: Vec, + /// The end of the line's last token that is not a comment. + code_end: TextSize, + /// The comment ending that line. + comment: Option, +} + +/// Something open that a statement being completed closes. +#[derive(Debug, Clone, Copy)] +enum Opener { + Bracket(TokenKind), + /// A string literal, with the quotes that close it; an f-string's or t-string's start, closed + /// by its end, can have brackets open inside it. + String(&'static str), +} + +impl Opener { + fn closer(self) -> &'static str { + match self { + Opener::Bracket(TokenKind::Lpar) => ")", + Opener::Bracket(TokenKind::Lsqb) => "]", + Opener::Bracket(_) => "}", + Opener::String(quote) => quote, + } + } +} + +impl LineTokens { + fn read(tokens: &Tokens, first: usize, line: TextRange) -> Self { + let mut open = Vec::new(); + let mut code_end = tokens[first].end(); + let mut comment = None; + for token in tokens[first..] + .iter() + .take_while(|token| token.start() < line.end()) + { + match token.kind() { + TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace => { + open.push(Opener::Bracket(token.kind())); + } + TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace => { + if matches!(open.last(), Some(Opener::Bracket(_))) { + open.pop(); + } + } + TokenKind::FStringStart | TokenKind::TStringStart => { + open.push(Opener::String(closing_quotes(token))); + } + TokenKind::FStringEnd | TokenKind::TStringEnd => { + // whatever was left open inside the interpolations ends with the string + if let Some(index) = open + .iter() + .rposition(|opener| matches!(opener, Opener::String(_))) + { + open.truncate(index); + } + } + TokenKind::String if token.start() >= line.start() => { + if token.string_flags().is_some_and(StringFlags::is_unclosed) { + open.push(Opener::String(closing_quotes(token))); + } + } + TokenKind::Comment => { + comment = Some(token.range()); + continue; + } + kind if is_layout(kind) => continue, + _ => {} + } + code_end = token.end(); + } + Self { + open, + code_end, + comment: comment.filter(|comment| comment.start() >= code_end), + } + } +} + +/// The quotes that close the string `token` starts. A triple-quoted string left open runs on to +/// the end of the file rather than to the end of the line, so where it was meant to end is not for +/// the line to say; it is left open. +fn closing_quotes(token: &Token) -> &'static str { + match token.string_flags() { + Some(flags) if !flags.is_triple_quoted() => flags.quote_str(), + _ => "", + } +} + +/// Tokens that are part of no statement: line breaks, indentation and comments. +fn is_layout(kind: TokenKind) -> bool { + matches!( + kind, + TokenKind::Newline + | TokenKind::NonLogicalNewline + | TokenKind::Indent + | TokenKind::Dedent + | TokenKind::Comment + ) +} + +/// A clause header on the logical line being completed. +struct Header { + /// Where the header's keyword is, or its start for one with none (a trailing lambda's). + anchor: TextSize, + needs_colon: bool, + /// A `def` written without its parameter list. + needs_parentheses: bool, + suite: Suite, +} + +/// Where a header's suite is. +enum Suite { + /// There is none yet. + Empty, + /// On the header's own line: `if x: return`. + OnTheLine, + /// Starting at this offset, on a later line. + Below(TextSize), +} + +impl Header { + /// The clause header the code of the logical line in `line` is, if it is one. + fn on_line( + db: &dyn Db, + file: PythonFile<'_>, + tokens: &Tokens, + source: &str, + line: TextRange, + ) -> Option { + let parsed = parsed_module(db, file).load(db); + let mut finder = HeaderFinder { + builder: OutlineBuilder::new(source, tokens), + tokens, + line, + found: None, + }; + finder.visit_body(parsed.suite()); + if let Some(header) = finder.found { + return Some(header); + } + + // A clause keyword the parser made no statement of — an `else` with no `if` above it — is + // still a header being written. + let first = tokens.partition_point(|token| token.start() < line.start()); + let keyword = tokens.get(first)?; + let is_compound_keyword = matches!( + keyword.kind(), + TokenKind::If + | TokenKind::Elif + | TokenKind::Else + | TokenKind::While + | TokenKind::For + | TokenKind::With + | TokenKind::Try + | TokenKind::Except + | TokenKind::Finally + | TokenKind::Def + | TokenKind::Class + | TokenKind::Async + // a soft keyword the parser kept as one + | TokenKind::Match + | TokenKind::Case + ); + if !is_compound_keyword { + return None; + } + let has_colon = + header_colon(&tokens[first + 1..]).is_some_and(|colon| colon.start() < line.end()); + Some(Self { + anchor: keyword.start(), + needs_colon: !has_colon, + needs_parentheses: false, + suite: Suite::Empty, + }) + } +} + +/// Finds the first clause header whose keyword is on the line being completed, statement +/// expressions' included. +struct HeaderFinder<'a> { + builder: OutlineBuilder<'a>, + tokens: &'a Tokens, + line: TextRange, + found: Option
, +} + +impl<'a> SourceOrderVisitor<'a> for HeaderFinder<'a> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if self.found.is_some() || stmt.range().intersect(self.line).is_none() { + return; + } + for (index, head) in self.builder.clause_heads(stmt).iter().enumerate() { + // the `let` half of a `let ... := ... else:` opens no suite of its own + if index == 0 && matches!(stmt, Stmt::Let(_)) { + continue; + } + let anchor = head.keyword.map_or(head.range.start(), TextRange::start); + if !self.line.contains_inclusive(anchor) { + continue; + } + let needs_parentheses = match stmt { + Stmt::FunctionDef(function) if !function.is_trailing_lambda && index == 0 => { + let from = self + .tokens + .partition_point(|token| token.start() < function.name.end()); + !self.tokens[from..] + .iter() + .take_while(|token| token.start() < self.line.end()) + .any(|token| token.kind() == TokenKind::Lpar) + } + _ => false, + }; + let suite = match head.body.first() { + None => Suite::Empty, + Some(first) if first.start() <= self.line.end() => Suite::OnTheLine, + Some(first) => Suite::Below(first.start()), + }; + self.found = Some(Header { + anchor, + needs_colon: head.colon.is_none(), + needs_parentheses, + suite, + }); + return; + } + walk_stmt(self, stmt); + } +} + +#[cfg(test)] +mod tests { + use ruff_source_file::LineEnding; + + use super::complete_statement; + use crate::tests::CursorTest; + + /// The source after completing the statement at ``, with `` where the caret + /// goes. + fn complete_in(path: &str, source: &str) -> String { + let test = CursorTest::builder().source(path, source).build(); + let file = test.program_file(test.cursor.file).python_file(&test.db); + let completed = complete_statement(&test.db, file, test.cursor.offset, " "); + let mut text = test.cursor.source.as_str().to_string(); + for edit in completed.edits.iter().rev() { + text.replace_range(edit.range.to_std_range(), &edit.content); + } + text.insert_str(completed.caret.to_usize(), ""); + text + } + + fn complete(source: &str) -> String { + complete_in("main.by", source) + } + + #[test] + fn a_header_gets_its_colon_and_a_suite() { + assert_eq!(complete("if x\n"), "if x:\n \n"); + assert_eq!( + complete("for item in items\n"), + "for item in items:\n \n" + ); + assert_eq!(complete("while x\n"), "while x:\n \n"); + assert_eq!(complete("class A\n"), "class A:\n \n"); + assert_eq!(complete("try\n"), "try:\n \n"); + assert_eq!( + complete("with open(p) as f\n"), + "with open(p) as f:\n \n" + ); + } + + #[test] + fn a_def_gets_its_parameters_too() { + assert_eq!(complete("def f\n"), "def f():\n \n"); + assert_eq!(complete("def f(x\n"), "def f(x):\n \n"); + assert_eq!( + complete("def f(x) -> int\n"), + "def f(x) -> int:\n \n" + ); + } + + #[test] + fn open_brackets_are_closed_innermost_first() { + assert_eq!( + complete("foo(a, [1, (2\n"), + "foo(a, [1, (2)])\n\n" + ); + assert_eq!( + complete("if handle(item, flags[\"x\"\n"), + "if handle(item, flags[\"x\"]):\n \n" + ); + } + + #[test] + fn a_bracket_the_next_line_does_not_close_is_closed() { + assert_eq!( + complete("foo(a, b\nx = 1\n"), + "foo(a, b)\n\nx = 1\n" + ); + } + + #[test] + fn a_bracket_open_at_the_end_of_the_file_is_closed() { + assert_eq!(complete("foo(a,\n"), "foo(a,)\n\n"); + } + + #[test] + fn a_statement_carried_on_below_is_completed_where_it_ends() { + assert_eq!( + complete("foo(a,\n b)\nx = 1\n"), + "foo(a,\n b)\n\nx = 1\n" + ); + assert_eq!( + complete("if (a and\n b)\n"), + "if (a and\n b):\n \n" + ); + } + + #[test] + fn a_string_left_open_is_closed() { + assert_eq!( + complete("print(\"hello\n"), + "print(\"hello\")\n\n" + ); + assert_eq!(complete("x = 'a\n"), "x = 'a'\n\n"); + assert_eq!( + complete("x = f\"a {y\n"), + "x = f\"a {y}\"\n\n" + ); + } + + #[test] + fn a_comment_stays_at_the_end() { + assert_eq!( + complete("foo(a # note\n"), + "foo(a) # note\n\n" + ); + assert_eq!( + complete("if x # why\n"), + "if x: # why\n \n" + ); + } + + #[test] + fn a_complete_statement_gets_a_new_line_at_its_indentation() { + assert_eq!( + complete("def f():\n x = g(1)\n return x\n"), + "def f():\n x = g(1)\n \n return x\n" + ); + } + + #[test] + fn a_header_with_a_suite_below_starts_a_line_at_the_suites_indentation() { + assert_eq!( + complete("if x:\n pass\n"), + "if x:\n \n pass\n" + ); + } + + #[test] + fn a_one_line_suite_is_left_done() { + assert_eq!( + complete("def f():\n if x: return 1\n"), + "def f():\n if x: return 1\n \n" + ); + } + + #[test] + fn a_continuing_clause_is_moved_out_to_its_statement() { + assert_eq!( + complete("if x:\n a()\n else\n"), + "if x:\n a()\nelse:\n \n" + ); + assert_eq!( + complete("try:\n a()\n except ValueError\n"), + "try:\n a()\nexcept ValueError:\n \n" + ); + } + + /// A clause with no statement above it to continue is read off its tokens, where a `:` in + /// brackets is not the one ending the header. + #[test] + fn a_colon_in_brackets_does_not_end_a_clause_the_parser_made_nothing_of() { + assert_eq!( + complete("if a:\n pass\n elif d[1:2]\n"), + "if a:\n pass\nelif d[1:2]:\n \n" + ); + assert_eq!( + complete("elif {1: 2}\n"), + "elif {1: 2}:\n \n" + ); + } + + /// The line a statement is completed with ends the way the file's lines do. + #[test] + fn a_new_line_ends_like_the_file_s_lines() { + assert_eq!(complete("x = 1\r\n"), "x = 1\r\n\r\n"); + assert_eq!( + complete("if x\r\ny = 2\r\n"), + "if x:\r\n \r\ny = 2\r\n" + ); + } + + #[test] + fn soft_keywords_are_the_parsers_call() { + assert_eq!(complete("match x\n"), "match x:\n \n"); + assert_eq!(complete("match = 1\n"), "match = 1\n\n"); + } + + #[test] + fn an_if_statement_expression_is_a_header() { + assert_eq!( + complete("label = if n == 0\n"), + "label = if n == 0:\n \n" + ); + } + + #[test] + fn a_blank_line_gets_a_new_line() { + assert_eq!( + complete("def f():\n \n"), + "def f():\n \n \n" + ); + } + + #[test] + fn trailing_whitespace_does_not_stay_behind() { + assert_eq!(complete("x = 1 \n"), "x = 1\n\n"); + } + + /// A file with no line ending to copy gets the platform's, as every other edit does. + #[test] + fn the_last_line_of_a_file_without_a_newline() { + assert_eq!( + complete("foo(1"), + format!("foo(1){}", LineEnding::default().as_str()) + ); + } + + #[test] + fn python_files_are_completed_too() { + assert_eq!( + complete_in("main.py", "elif_ = 1\nif x\n"), + "elif_ = 1\nif x:\n \n" + ); + } +} diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 9cd6e6a488..d7bd2be79b 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -8,6 +8,7 @@ mod all_symbols; mod call_hierarchy; mod code_action; mod common_aliases; +mod complete_statement; mod completion; mod data_flow; mod django_template; @@ -29,6 +30,7 @@ mod injection; mod inlay_hints; mod markup; mod module_rename; +mod on_type_formatting; mod override_members; mod pasted_imports; mod refactor; @@ -53,6 +55,7 @@ pub use call_hierarchy::incoming_calls::{IncomingCall, incoming_calls}; pub use call_hierarchy::outgoing_calls::{OutgoingCall, outgoing_calls}; pub use call_hierarchy::{CallHierarchyItem, prepare_call_hierarchy}; pub use code_action::{FileEdit, QuickFix, code_actions}; +pub use complete_statement::{CompletedStatement, StatementEdit, complete_statement}; pub use completion::{ Completion, CompletionCapabilities, CompletionCommand, CompletionInsertTextFormat, CompletionKind, CompletionSettings, completion, @@ -85,6 +88,7 @@ pub use markup::MarkupKind; pub use module_rename::{ FileMove, ModuleRenameEdits, SkipReason, SkippedImport, module_rename_edits, }; +pub use on_type_formatting::{Reindent, on_type_formatting}; pub use override_members::{ ClassMembersToOverride, MemberOverrides, MemberToOverride, members_to_override, override_members, diff --git a/crates/ty_ide/src/on_type_formatting.rs b/crates/ty_ide/src/on_type_formatting.rs new file mode 100644 index 0000000000..275c04bf13 --- /dev/null +++ b/crates/ty_ide/src/on_type_formatting.rs @@ -0,0 +1,557 @@ +//! What typing a character changes about the line it was typed on. +//! +//! A clause that continues a compound statement — `elif`, `else`, `except`, `finally`, a further +//! `case` — is written at the indentation of the clause it continues. An editor starts a new line +//! at the indentation of the one before, which is the body the clause closes, so the clause is +//! typed a level or more too deep. Typing its `:` is the moment to move it out: the header is +//! whole by then, and which statement it continues is decided by what is above it. +//! +//! ```python +//! for item in items: +//! if item: +//! handle(item) +//! else: # typed here; moved out to the `if` once the `:` is typed +//! ``` +//! +//! The statement it continues is the innermost one that is still open above the line, can take +//! that clause next, and is not indented deeper than the line already is: a clause is only ever +//! moved out, so one written at the indentation of an outer statement stays with that statement. + +use ruff_db::PythonFile; +use ruff_db::parsed::parsed_module; +use ruff_db::source::source_text; +use ruff_python_ast::token::{Token, TokenKind, Tokens}; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk_stmt}; +use ruff_python_ast::{self as ast, Expr, Stmt}; +use ruff_python_parser::{ParseOptions, parse_unchecked}; +use ruff_python_trivia::leading_indentation; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; + +use crate::Db; + +/// A line's indentation, replaced. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reindent { + /// The line's leading whitespace as it is. + pub range: TextRange, + /// What replaces it: the leading whitespace of the line the clause lines up with, as written + /// there, so a file indented with tabs stays indented with tabs. + pub indentation: String, +} + +/// What typing `typed`, which ends at `offset`, changes about its line: the indentation of a +/// clause whose header that character finished, when the clause is not yet lined up with the +/// statement it continues. +pub fn on_type_formatting( + db: &dyn Db, + file: PythonFile<'_>, + offset: TextSize, + typed: &str, +) -> Option { + if typed != ":" { + return None; + } + let parsed = parsed_module(db, file).load(db); + let tokens = parsed.tokens(); + let start = tokens.partition_point(|token| token.end() < offset); + let colon = tokens[start..] + .iter() + .take_while(|token| token.start() < offset) + .find(|token| token.kind() == TokenKind::Colon && token.end() == offset)?; + clause_reindent(db, file, tokens, colon.range()) +} + +/// The indentation the clause whose header ends at the `:` at `colon` belongs at, when that is +/// not where it is. +fn clause_reindent( + db: &dyn Db, + file: PythonFile<'_>, + tokens: &Tokens, + colon: TextRange, +) -> Option { + let source = source_text(db, file.file(db)); + let line_start = source.line_start(colon.start()); + let (index, keyword) = continuing_keyword(source.as_str(), tokens, line_start)?; + if !colon_ends_header(tokens, index, keyword, colon) { + return None; + } + line_reindent(db, file, line_start, keyword) +} + +/// The indentation the line starting at `line_start` belongs at, when it starts a clause that +/// continues a compound statement and is not lined up with that statement yet. +/// +/// Whether the clause's header is whole is the caller's to have made sure of. +pub(crate) fn continuing_clause_reindent( + db: &dyn Db, + file: PythonFile<'_>, + tokens: &Tokens, + line_start: TextSize, +) -> Option { + let source = source_text(db, file.file(db)); + let (_, keyword) = continuing_keyword(source.as_str(), tokens, line_start)?; + line_reindent(db, file, line_start, keyword) +} + +fn line_reindent( + db: &dyn Db, + file: PythonFile<'_>, + line_start: TextSize, + keyword: ContinuingKeyword, +) -> Option { + let source = source_text(db, file.file(db)); + let source = source.as_str(); + let indentation = leading_indentation(&source[line_start.to_usize()..]); + + // The statements above the line are read from the text above it alone: the clause itself, + // written where it does not belong yet, is an error in the full parse, and what the parser + // makes of the statements around an error is a recovery rather than the structure the clause + // is being lined up with. + let above = parse_unchecked(&source[..line_start.to_usize()], parse_options(db, file)); + let ast::Mod::Module(module) = above.syntax() else { + return None; + }; + let mut open = Vec::new(); + open_statements(&module.body, &mut open); + + let width = indentation_width(indentation); + let anchor = open.iter().rev().find_map(|statement| { + let anchor = continued_at(statement, keyword)?; + let anchor_line = source.line_start(anchor); + let anchor_indentation = leading_indentation(&source[TextRange::new(anchor_line, anchor)]); + (indentation_width(anchor_indentation) <= width).then_some(anchor_indentation) + })?; + (anchor != indentation).then(|| Reindent { + range: TextRange::at(line_start, indentation.text_len()), + indentation: anchor.to_string(), + }) +} + +/// The options `file` is parsed with, for parsing part of its text the way the whole of it is +/// parsed: basedpython syntax in a basedpython file, at the Python version the file is checked +/// against. +fn parse_options(db: &dyn Db, file: PythonFile<'_>) -> ParseOptions { + ParseOptions::from(file.file(db).source_type(db)).with_target_version(file.python_version(db)) +} + +/// A keyword that starts a clause continuing a compound statement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ContinuingKeyword { + Elif, + Else, + Except, + Finally, + Case, +} + +/// The token index and kind of the keyword the line starting at `line_start` opens with, when it +/// is one that continues a compound statement and it starts a logical line — rather than +/// continuing one inside brackets or after a backslash. +fn continuing_keyword( + source: &str, + tokens: &Tokens, + line_start: TextSize, +) -> Option<(usize, ContinuingKeyword)> { + let keyword_start = + line_start + leading_indentation(&source[line_start.to_usize()..]).text_len(); + // past the zero-width `Dedent`s that start where a dedented line's first token does + let first = tokens.partition_point(|token| token.start() < keyword_start); + let first = first + + tokens[first..] + .iter() + .take_while(|token| token.kind() == TokenKind::Dedent) + .count(); + let keyword_token = tokens.get(first)?; + if keyword_token.start() != keyword_start { + return None; + } + let keyword = match keyword_token.kind() { + TokenKind::Elif => ContinuingKeyword::Elif, + TokenKind::Else => ContinuingKeyword::Else, + TokenKind::Except => ContinuingKeyword::Except, + TokenKind::Finally => ContinuingKeyword::Finally, + // `case` is a soft keyword, and the parser only keeps a `case` token for one inside a + // `match`; written in a body where it does not belong yet, it reads as a name. + TokenKind::Case => ContinuingKeyword::Case, + TokenKind::Name if &source[keyword_token.range()] == "case" => ContinuingKeyword::Case, + _ => return None, + }; + let starts_line = tokens[..first] + .iter() + .rev() + .find(|token| { + !matches!( + token.kind(), + TokenKind::Indent + | TokenKind::Dedent + | TokenKind::Comment + | TokenKind::NonLogicalNewline + ) + }) + .is_none_or(|token| token.kind() == TokenKind::Newline); + starts_line.then_some((first, keyword)) +} + +/// Whether the `:` at `colon` ends the header that `keyword`, the token at `index`, starts — +/// rather than belonging to a lambda in it, or sitting in brackets. +fn colon_ends_header( + tokens: &Tokens, + index: usize, + keyword: ContinuingKeyword, + colon: TextRange, +) -> bool { + if header_colon(&tokens[index + 1..]) != Some(colon) { + return false; + } + let between = &tokens[index + 1..tokens.partition_point(|token| token.start() < colon.start())]; + let has_header = between.iter().any(|token| !token.kind().is_trivia()); + match keyword { + // nothing comes between these and their `:` + ContinuingKeyword::Else | ContinuingKeyword::Finally => !has_header, + // a `case` needs a pattern; `case: int = 1` annotates a variable called `case` + ContinuingKeyword::Case => has_header, + ContinuingKeyword::Elif | ContinuingKeyword::Except => true, + } +} + +/// The `:` that ends the clause header `tokens` start after its keyword: the first one outside +/// brackets and not a lambda's, before the logical line ends. +pub(crate) fn header_colon(tokens: &[Token]) -> Option { + let mut depth = 0u32; + let mut lambdas = 0u32; + for token in tokens { + match token.kind() { + TokenKind::Newline => return None, + TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace => depth += 1, + TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace => { + depth = depth.saturating_sub(1); + } + TokenKind::Lambda if depth == 0 => lambdas += 1, + TokenKind::Colon if depth == 0 => match lambdas.checked_sub(1) { + Some(rest) => lambdas = rest, + None => return Some(token.range()), + }, + _ => {} + } + } + None +} + +/// The statements a line after `suite` could still continue, outermost first: the last statement +/// of the suite, the last statement of that one's last clause, and so on down. +fn open_statements<'a>(suite: &'a [Stmt], open: &mut Vec<&'a Stmt>) { + let Some(last) = suite.last() else { + return; + }; + // A basedpython statement expression — `label = if count == 0:` — is a compound statement + // that ends where the simple statement holding it does, and continues like one. + let statement = trailing_statement_expression(last).unwrap_or(last); + open.push(statement); + if let Some(body) = last_suite(statement) { + open_statements(body, open); + } +} + +/// The suite of `statement`'s last clause, for a compound statement. +fn last_suite(statement: &Stmt) -> Option<&[Stmt]> { + let suite: &[Stmt] = match statement { + Stmt::If(if_stmt) => if_stmt + .elif_else_clauses + .last() + .map_or(&if_stmt.body, |clause| &clause.body), + Stmt::For(ast::StmtFor { body, orelse, .. }) + | Stmt::While(ast::StmtWhile { body, orelse, .. }) => { + if orelse.is_empty() { + body + } else { + orelse + } + } + Stmt::Try(try_stmt) => [&try_stmt.finalbody, &try_stmt.orelse] + .into_iter() + .find(|suite| !suite.is_empty()) + .map_or_else( + || { + try_stmt.handlers.last().map_or(&try_stmt.body, |handler| { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + &handler.body + }) + }, + |suite| suite, + ), + Stmt::With(with_stmt) => &with_stmt.body, + Stmt::FunctionDef(function) => &function.body, + Stmt::ClassDef(class) => &class.body, + Stmt::Match(match_stmt) => &match_stmt.cases.last()?.body, + Stmt::Let(let_stmt) => &let_stmt.orelse, + _ => return None, + }; + Some(suite) +} + +/// Where the line `keyword` would continue `statement` lines up, when `statement` can take that +/// clause next: the start of the statement's first keyword, or, for a `case`, of its last `case`. +fn continued_at(statement: &Stmt, keyword: ContinuingKeyword) -> Option { + match (statement, keyword) { + (Stmt::If(if_stmt), ContinuingKeyword::Elif | ContinuingKeyword::Else) => if_stmt + .elif_else_clauses + .last() + .is_none_or(|clause| clause.test.is_some()) + .then(|| if_stmt.start()), + ( + Stmt::For(ast::StmtFor { orelse, .. }) | Stmt::While(ast::StmtWhile { orelse, .. }), + ContinuingKeyword::Else, + ) => orelse.is_empty().then(|| statement.start()), + (Stmt::Try(try_stmt), ContinuingKeyword::Except) => { + (try_stmt.orelse.is_empty() && try_stmt.finalbody.is_empty()).then(|| try_stmt.start()) + } + (Stmt::Try(try_stmt), ContinuingKeyword::Else) => (!try_stmt.handlers.is_empty() + && try_stmt.orelse.is_empty() + && try_stmt.finalbody.is_empty()) + .then(|| try_stmt.start()), + (Stmt::Try(try_stmt), ContinuingKeyword::Finally) => { + try_stmt.finalbody.is_empty().then(|| try_stmt.start()) + } + (Stmt::Match(match_stmt), ContinuingKeyword::Case) => { + match_stmt.cases.last().map(Ranged::start) + } + _ => None, + } +} + +/// The compound statement a simple statement ends with, when it ends with a statement expression. +fn trailing_statement_expression(statement: &Stmt) -> Option<&Stmt> { + struct Finder<'a> { + end: TextSize, + found: Option<&'a Stmt>, + } + + impl<'a> SourceOrderVisitor<'a> for Finder<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + // Only an expression ending where the statement does can hold the one it ends with, + // and the outermost such statement expression is the one a clause would continue. + if self.found.is_some() || expr.end() != self.end { + return; + } + if let Expr::Statement(statement) = expr { + self.found = Some(statement.stmt.as_ref()); + } else { + walk_expr(self, expr); + } + } + } + + if !is_simple(statement) { + return None; + } + let mut finder = Finder { + end: statement.end(), + found: None, + }; + walk_stmt(&mut finder, statement); + finder.found +} + +fn is_simple(statement: &Stmt) -> bool { + !matches!( + statement, + Stmt::If(_) + | Stmt::For(_) + | Stmt::While(_) + | Stmt::Try(_) + | Stmt::With(_) + | Stmt::FunctionDef(_) + | Stmt::ClassDef(_) + | Stmt::Match(_) + | Stmt::Let(_) + ) +} + +/// How many columns `indentation` spans, a tab reaching the next multiple of eight as it does for +/// Python's tokenizer. +pub(crate) fn indentation_width(indentation: &str) -> usize { + indentation.chars().fold(0, |column, char| match char { + '\t' => (column / 8 + 1) * 8, + _ => column + 1, + }) +} + +#[cfg(test)] +mod tests { + use ruff_text_size::TextSize; + + use super::on_type_formatting; + use crate::tests::CursorTest; + + /// The source after typing a `:` at ``: the `:` is inserted, and the reindent applied. + fn typed_colon(path: &str, source: &str) -> String { + let source = source.replace("", ":"); + let test = CursorTest::builder().source(path, &source).build(); + let file = test.program_file(test.cursor.file).python_file(&test.db); + let text = test.cursor.source.as_str().to_string(); + match on_type_formatting(&test.db, file, test.cursor.offset, ":") { + Some(reindent) => { + let mut text = text; + text.replace_range(reindent.range.to_std_range(), &reindent.indentation); + text + } + None => text, + } + } + + fn by(source: &str) -> String { + typed_colon("main.by", source) + } + + #[test] + fn else_moves_out_to_its_if() { + assert_eq!( + by("if x:\n a = 1\n else\n"), + "if x:\n a = 1\nelse:\n" + ); + } + + #[test] + fn elif_moves_out_to_its_if() { + assert_eq!( + by("if x:\n a = 1\n elif y\n"), + "if x:\n a = 1\nelif y:\n" + ); + } + + #[test] + fn else_goes_to_the_innermost_if_not_deeper_than_it() { + let source = "for i in r:\n if x:\n pass\n else\n"; + assert_eq!( + by(source), + "for i in r:\n if x:\n pass\n else:\n" + ); + } + + #[test] + fn a_clause_already_at_an_outer_statement_stays_there() { + let source = "for i in r:\n if x:\n pass\nelse\n"; + assert_eq!(by(source), source.replace("", ":")); + } + + #[test] + fn else_skips_an_if_that_has_one() { + let source = + "for i in r:\n if x:\n pass\n else:\n pass\n else\n"; + // the `if` already has its `else`; the next one out that can take one is the loop + assert_eq!( + by(source), + "for i in r:\n if x:\n pass\n else:\n pass\nelse:\n" + ); + } + + #[test] + fn except_and_finally_move_out_to_their_try() { + assert_eq!( + by("try:\n a()\n except ValueError as e\n"), + "try:\n a()\nexcept ValueError as e:\n" + ); + assert_eq!( + by("try:\n a()\nexcept E:\n b()\n finally\n"), + "try:\n a()\nexcept E:\n b()\nfinally:\n" + ); + } + + #[test] + fn else_after_a_try_needs_an_except_first() { + let source = "try:\n a()\n else\n"; + assert_eq!(by(source), source.replace("", ":")); + } + + #[test] + fn case_lines_up_with_the_case_before() { + assert_eq!( + by("match x:\n case 1:\n a()\n case 2\n"), + "match x:\n case 1:\n a()\n case 2:\n" + ); + } + + #[test] + fn a_first_case_is_left_where_it_is() { + let source = "match x:\n case 1\n"; + assert_eq!(by(source), source.replace("", ":")); + } + + #[test] + fn a_variable_called_case_is_not_a_clause() { + let source = "match x:\n case 1:\n case int = 1\n"; + assert_eq!(by(source), source.replace("", ":")); + } + + #[test] + fn a_colon_that_does_not_end_the_header_changes_nothing() { + for source in [ + "if x:\n pass\n elif y[1]\n", + "if x:\n pass\n elif lambda\n", + "if x:\n pass\n elif {1 2}\n", + ] { + assert_eq!(by(source), source.replace("", ":"), "{source:?}"); + } + } + + #[test] + fn a_lambdas_colon_is_passed_over() { + assert_eq!( + by("if x:\n pass\n elif lambda: y\n"), + "if x:\n pass\nelif lambda: y:\n" + ); + } + + #[test] + fn a_keyword_inside_brackets_is_not_a_clause() { + let source = "if x:\n f(\n else\n"; + assert_eq!(by(source), source.replace("", ":")); + } + + #[test] + fn a_line_after_a_simple_statement_at_its_level_is_left_alone() { + let source = "if x:\n pass\ny = 1\n else\n"; + assert_eq!(by(source), source.replace("", ":")); + } + + #[test] + fn an_if_statement_expression_takes_its_else() { + assert_eq!( + by("label = if n == 0:\n \"none\"\n else\n"), + "label = if n == 0:\n \"none\"\nelse:\n" + ); + } + + #[test] + fn tabs_are_kept() { + assert_eq!( + by("def f():\n\tif x:\n\t\tpass\n\t\telse\n"), + "def f():\n\tif x:\n\t\tpass\n\telse:\n" + ); + } + + #[test] + fn python_files_are_reindented_too() { + assert_eq!( + typed_colon("main.py", "while x:\n pass\n else\n"), + "while x:\n pass\nelse:\n" + ); + } + + #[test] + fn only_a_colon_triggers() { + let test = CursorTest::builder() + .source("main.by", "if x:\n pass\n else:\n") + .build(); + let file = test.program_file(test.cursor.file).python_file(&test.db); + assert_eq!( + on_type_formatting(&test.db, file, test.cursor.offset, "\n"), + None + ); + assert_eq!( + on_type_formatting(&test.db, file, test.cursor.offset - TextSize::new(1), ":"), + None + ); + } +} diff --git a/crates/ty_ide/src/syntax_outline.rs b/crates/ty_ide/src/syntax_outline.rs index 018579c44c..f2c59e30cc 100644 --- a/crates/ty_ide/src/syntax_outline.rs +++ b/crates/ty_ide/src/syntax_outline.rs @@ -194,11 +194,17 @@ fn keyword_at(tokens: &Tokens, offset: TextSize) -> Option { .map(Ranged::range) } -struct OutlineBuilder<'a> { +pub(crate) struct OutlineBuilder<'a> { source: &'a str, tokens: &'a Tokens, } +impl<'a> OutlineBuilder<'a> { + pub(crate) fn new(source: &'a str, tokens: &'a Tokens) -> Self { + Self { source, tokens } + } +} + impl OutlineBuilder<'_> { fn statements(&self, suite: &[Stmt]) -> Vec { suite.iter().map(|stmt| self.statement(stmt)).collect() @@ -230,7 +236,7 @@ impl OutlineBuilder<'_> { } /// The clauses of `stmt` without their bodies' outlines, which is all a family needs. - fn clause_heads<'s>(&self, stmt: &'s Stmt) -> Vec> { + pub(crate) fn clause_heads<'s>(&self, stmt: &'s Stmt) -> Vec> { let mut heads = Vec::new(); match stmt { Stmt::If(if_stmt) => { @@ -529,11 +535,11 @@ impl OutlineBuilder<'_> { } } -struct ClauseHead<'s> { - keyword: Option, - colon: Option, - range: TextRange, - body: &'s [Stmt], +pub(crate) struct ClauseHead<'s> { + pub(crate) keyword: Option, + pub(crate) colon: Option, + pub(crate) range: TextRange, + pub(crate) body: &'s [Stmt], } /// A block enclosing the statement being looked at, as a `return` or a `break` sees it. diff --git a/crates/ty_server/src/capabilities.rs b/crates/ty_server/src/capabilities.rs index 396f691927..b227cc4e8c 100644 --- a/crates/ty_server/src/capabilities.rs +++ b/crates/ty_server/src/capabilities.rs @@ -1,9 +1,10 @@ use lsp_types::{ self as types, ClientCapabilities, CodeActionKind, CodeActionOptions, CompletionOptions, - DiagnosticOptions, DiagnosticProvider, InlayHintOptions, MarkupKind, NotebookCellLanguage, - NotebookDocumentFilterWithCells, NotebookSelector, RenameOptions, Save, SaveOptions, - SemanticTokensLegend, SemanticTokensOptions, ServerCapabilities, SignatureHelpOptions, - TextDocumentSyncKind, TextDocumentSyncOptions, WorkDoneProgressOptions, + DiagnosticOptions, DiagnosticProvider, DocumentOnTypeFormattingOptions, InlayHintOptions, + MarkupKind, NotebookCellLanguage, NotebookDocumentFilterWithCells, NotebookSelector, + RenameOptions, Save, SaveOptions, SemanticTokensLegend, SemanticTokensOptions, + ServerCapabilities, SignatureHelpOptions, TextDocumentSyncKind, TextDocumentSyncOptions, + WorkDoneProgressOptions, }; use std::str::FromStr; @@ -606,6 +607,13 @@ pub(crate) fn server_capabilities( resolve_provider: Some(false), work_done_progress_options: WorkDoneProgressOptions::default(), }), + // `:` ends a clause header, which is when a clause that continues a compound statement — + // `else`, `except`, a further `case` — is moved out to that statement. No other character + // finishes something whose layout the parser decides. + document_on_type_formatting_provider: Some(DocumentOnTypeFormattingOptions { + first_trigger_character: ":".to_string(), + more_trigger_character: None, + }), selection_range_provider: Some(true.into()), folding_range_provider: Some(true.into()), document_symbol_provider: Some(true.into()), diff --git a/crates/ty_server/src/document.rs b/crates/ty_server/src/document.rs index c0318cec2a..d9dde8d87a 100644 --- a/crates/ty_server/src/document.rs +++ b/crates/ty_server/src/document.rs @@ -15,7 +15,7 @@ pub(crate) use location::ToLink; pub use notebook::NotebookDocument; pub(crate) use range::{ FileRangeExt, LspRange, PositionExt, RangeExt, TextSizeExt, ToRangeExt, resolve_file_uri_range, - text_range_to_lsp_range, + text_range_to_lsp_range, text_size_to_lsp_position, }; pub use text_document::TextDocument; pub(crate) use text_document::{DocumentVersion, LanguageId}; diff --git a/crates/ty_server/src/document/range.rs b/crates/ty_server/src/document/range.rs index 05c1d8d6e2..dcfd1bc14f 100644 --- a/crates/ty_server/src/document/range.rs +++ b/crates/ty_server/src/document/range.rs @@ -228,7 +228,7 @@ fn u32_index_to_usize(index: u32) -> usize { usize::try_from(index).expect("u32 fits in usize") } -fn text_size_to_lsp_position( +pub(crate) fn text_size_to_lsp_position( offset: TextSize, text: &str, index: &LineIndex, diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 06398006c7..1abc4728b6 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -155,6 +155,20 @@ pub(super) fn request(req: server::Request) -> Task { BackgroundSchedule::LatencySensitive, ) } + // Both answer a key just pressed, and what they answer is applied to the text only if + // nothing has been typed since: a slow answer is a dropped one. Each is a parse or two. + requests::OnTypeFormattingRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } + requests::CompleteStatementRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } requests::ExplainTranspilationHandler::METHOD => background_document_request_task::< requests::ExplainTranspilationHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 5e2d4dc136..9eb900c018 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -20,6 +20,7 @@ mod check_workspace; mod code_action; mod code_action_resolve; mod code_lens; +mod complete_statement; mod completion; mod copied_imports; mod data_flow; @@ -43,6 +44,7 @@ mod hover; mod injections; mod inlay_hints; mod module_name; +mod on_type_formatting; mod overridable_members; mod override_members; mod pasted_imports; @@ -78,6 +80,7 @@ pub(crate) use check_workspace::SuspendedWorkspaceRequestKind; pub(super) use code_action::CodeActionRequestHandler; pub(super) use code_action_resolve::CodeActionResolveRequestHandler; pub(super) use code_lens::CodeLensRequestHandler; +pub(super) use complete_statement::CompleteStatementRequestHandler; pub(super) use completion::CompletionRequestHandler; pub(super) use copied_imports::CopiedImportsRequestHandler; pub(super) use data_flow::DataFlowRequestHandler; @@ -100,6 +103,7 @@ pub(super) use hover::HoverRequestHandler; pub(super) use injections::InjectionsRequestHandler; pub(super) use inlay_hints::InlayHintRequestHandler; pub(super) use module_name::ModuleNameHandler; +pub(super) use on_type_formatting::OnTypeFormattingRequestHandler; pub(super) use overridable_members::OverridableMembersRequestHandler; pub(super) use override_members::OverrideMembersRequestHandler; pub(super) use pasted_imports::PastedImportsRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/complete_statement.rs b/crates/ty_server/src/server/api/requests/complete_statement.rs new file mode 100644 index 0000000000..d6005ce2b8 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/complete_statement.rs @@ -0,0 +1,150 @@ +//! `by/completeStatement` — the statement at a position completed, and the line after it started: +//! what an editor's *Complete Current Statement* (Smart Enter) does. See +//! [`ty_ide::complete_statement`]. +//! +//! LSP has no request for it. The answer is the edits and where the caret goes afterwards, which +//! is a place in the text the edits produce rather than in the text that was asked about, so it +//! is not a code action's workspace edit either. +//! +//! One position in one document; `textHash` says which revision of the document it is in. + +use std::borrow::Cow; + +use lsp_types::{ + FormattingOptions, LspRequestMethod, MessageDirection, Position, Request, + TextDocumentIdentifier, TextEdit, Uri, +}; +use ruff_db::source::source_text; +use ruff_source_file::LineIndex; +use ty_ide::complete_statement; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{PositionExt, ToRangeExt, text_size_to_lsp_position}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum CompleteStatementRequest {} + +impl Request for CompleteStatementRequest { + type Params = CompleteStatementParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/completeStatement"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// Where the statement to complete is. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct CompleteStatementParams { + text_document: TextDocumentIdentifier, + /// The caret: the statement completed is the one on its line. + position: Position, + /// How the document is indented, as `textDocument/onTypeFormatting` says it: one level is + /// `tabSize` spaces when `insertSpaces`, and a tab otherwise. Used for the line a header's + /// suite starts on when the suite has no statement yet to copy the indentation of. + options: FormattingOptions, +} + +/// The statement completed. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CompletedStatement { + /// The edits, against the document as it was asked about. None overlaps another. + edits: Vec, + /// Where the caret goes, in the document as the edits leave it. + caret: Position, +} + +pub(crate) struct CompleteStatementRequestHandler; + +impl RequestHandler for CompleteStatementRequestHandler { + type RequestType = CompleteStatementRequest; +} + +impl BackgroundDocumentRequestHandler for CompleteStatementRequestHandler { + fn document_uri(params: &CompleteStatementParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: CompleteStatementParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + + let indent_unit = if params.options.insert_spaces { + " ".repeat(usize::try_from(params.options.tab_size.max(1)).unwrap_or(4)) + } else { + "\t".to_string() + }; + let python_file = db.program_file(file).python_file(db); + let completed = complete_statement(db, python_file, offset, &indent_unit); + + let mut edits = Vec::with_capacity(completed.edits.len()); + for edit in &completed.edits { + let Some(range) = edit + .range + .to_lsp_range(db, file, snapshot.encoding()) + .map(|range| range.local_range()) + else { + return Ok(None); + }; + edits.push(TextEdit { + range, + new_text: edit.content.clone(), + }); + } + + // The caret is a place in the edited text, so it is measured there. + let mut edited = source_text(db, file).as_str().to_string(); + for edit in completed.edits.iter().rev() { + edited.replace_range(edit.range.to_std_range(), &edit.content); + } + let index = LineIndex::from_source_text(&edited); + let caret = + text_size_to_lsp_position(completed.caret, &edited, &index, snapshot.encoding()); + + Ok(Some(CompletedStatement { edits, caret })) + } +} + +impl RetriableRequestHandler for CompleteStatementRequestHandler {} + +#[cfg(test)] +mod tests { + use super::*; + + /// The wire form, exactly as a client sends it. + #[test] + fn the_params_a_client_sends_parse() { + let parsed: CompleteStatementParams = serde_json::from_str( + r#"{"textDocument":{"uri":"file:///main.by"},"position":{"line":0,"character":4},"options":{"tabSize":4,"insertSpaces":true}}"#, + ) + .expect("a client sends a document, a position and the formatting options"); + assert_eq!(parsed.position.character, 4); + assert!(parsed.options.insert_spaces); + } +} diff --git a/crates/ty_server/src/server/api/requests/on_type_formatting.rs b/crates/ty_server/src/server/api/requests/on_type_formatting.rs new file mode 100644 index 0000000000..6dea6c5d1f --- /dev/null +++ b/crates/ty_server/src/server/api/requests/on_type_formatting.rs @@ -0,0 +1,75 @@ +//! `textDocument/onTypeFormatting` — a clause moved out to the statement it continues when its +//! `:` is typed. See [`ty_ide::on_type_formatting`]. + +use std::borrow::Cow; + +use lsp_types::{DocumentOnTypeFormattingParams, DocumentOnTypeFormattingRequest, TextEdit, Uri}; +use ty_ide::on_type_formatting; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{PositionExt, ToRangeExt}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) struct OnTypeFormattingRequestHandler; + +impl RequestHandler for OnTypeFormattingRequestHandler { + type RequestType = DocumentOnTypeFormattingRequest; +} + +impl BackgroundDocumentRequestHandler for OnTypeFormattingRequestHandler { + fn document_uri(params: &DocumentOnTypeFormattingParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: DocumentOnTypeFormattingParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + // A notebook cell is not a file whose lines the statements above can be read from. + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + + let python_file = db.program_file(file).python_file(db); + let Some(reindent) = on_type_formatting(db, python_file, offset, ¶ms.ch) else { + return Ok(None); + }; + let Some(range) = reindent + .range + .to_lsp_range(db, file, snapshot.encoding()) + .map(|range| range.local_range()) + else { + return Ok(None); + }; + Ok(Some(vec![TextEdit { + range, + new_text: reindent.indentation, + }])) + } +} + +// The edit is only right for the text it was worked out on: one overtaken by an edit is refused +// rather than asked again, since the client applies it to the text as it is when it arrives. +impl RetriableRequestHandler for OnTypeFormattingRequestHandler {} diff --git a/crates/ty_server/tests/e2e/complete_statement.rs b/crates/ty_server/tests/e2e/complete_statement.rs new file mode 100644 index 0000000000..aa60444b49 --- /dev/null +++ b/crates/ty_server/tests/e2e/complete_statement.rs @@ -0,0 +1,136 @@ +//! `by/completeStatement` over the wire +//! +//! the unit tests in `ty_ide` cover what completing a statement writes. what they cannot cover is +//! the contract a client reads: the edits in the document as it was asked about, and the caret in +//! the document as the edits leave it, both in line/character positions — a caret one line out +//! puts the user's next keystroke in the wrong suite + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder, text_hash}; + +/// the request as a client sends it, in json throughout +enum CompleteStatement {} + +impl Request for CompleteStatement { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/completeStatement"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +fn server_with(content: &str) -> Result { + let main = SystemPath::new("src/main.by"); + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(main, content)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(main, content, 1); + Ok(server) +} + +fn complete( + server: &mut TestServer, + content: &str, + line: u32, + character: u32, + options: &serde_json::Value, +) -> serde_json::Value { + let uri = server.file_uri(SystemPath::new("src/main.by")); + server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": line, "character": character }, + "options": options, + "textHash": text_hash(content), + })) + .expect("the server completes a statement in a file it has open") +} + +#[test] +fn a_header_is_completed_and_the_caret_goes_into_its_suite() -> Result<()> { + let content = "def f():\n if g(x\n"; + let mut server = server_with(content)?; + let completed = complete( + &mut server, + content, + 1, + 10, + &serde_json::json!({ "tabSize": 4, "insertSpaces": true }), + ); + assert_eq!( + completed, + serde_json::json!({ + "edits": [{ + "range": { + "start": { "line": 1, "character": 10 }, + "end": { "line": 1, "character": 10 }, + }, + "newText": "):\n ", + }], + "caret": { "line": 2, "character": 8 }, + }) + ); + Ok(()) +} + +#[test] +fn a_suite_started_in_a_tab_indented_file_is_indented_with_a_tab() -> Result<()> { + let content = "while x\n"; + let mut server = server_with(content)?; + let completed = complete( + &mut server, + content, + 0, + 7, + &serde_json::json!({ "tabSize": 8, "insertSpaces": false }), + ); + assert_eq!(completed["edits"][0]["newText"], ":\n\t"); + assert_eq!( + completed["caret"], + serde_json::json!({ "line": 1, "character": 1 }) + ); + Ok(()) +} + +#[test] +fn a_clause_moved_out_is_one_edit_and_the_completion_another() -> Result<()> { + let content = "if x:\n a()\n else\n"; + let mut server = server_with(content)?; + let completed = complete( + &mut server, + content, + 2, + 8, + &serde_json::json!({ "tabSize": 4, "insertSpaces": true }), + ); + assert_eq!( + completed, + serde_json::json!({ + "edits": [ + { + "range": { + "start": { "line": 2, "character": 0 }, + "end": { "line": 2, "character": 4 }, + }, + "newText": "", + }, + { + "range": { + "start": { "line": 2, "character": 8 }, + "end": { "line": 2, "character": 8 }, + }, + "newText": ":\n ", + }, + ], + // in the edited text: `else` has moved four characters left + "caret": { "line": 3, "character": 4 }, + }) + ); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 412b2f062e..579eb81a5e 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -34,6 +34,7 @@ mod call_hierarchy; mod check_workspace; mod code_actions; mod commands; +mod complete_statement; mod completions; mod configuration; mod data_flow; @@ -49,6 +50,7 @@ mod injections; mod inlay_hints; mod named_text; mod notebook; +mod on_type_formatting; mod override_members; mod pasted_imports; mod program_model; diff --git a/crates/ty_server/tests/e2e/on_type_formatting.rs b/crates/ty_server/tests/e2e/on_type_formatting.rs new file mode 100644 index 0000000000..62519cfbce --- /dev/null +++ b/crates/ty_server/tests/e2e/on_type_formatting.rs @@ -0,0 +1,78 @@ +//! `textDocument/onTypeFormatting` over the wire: the `:` trigger the server advertises, and the +//! edit that moves a clause out to the statement it continues, in a client's line/character +//! positions + +use anyhow::Result; +use lsp_types::{ + DocumentOnTypeFormattingParams, DocumentOnTypeFormattingRequest, FormattingOptions, Position, + TextDocumentIdentifier, +}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder}; + +fn server_with(content: &str) -> Result { + let main = SystemPath::new("src/main.by"); + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(main, content)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(main, content, 1); + Ok(server) +} + +fn typed(server: &mut TestServer, line: u32, character: u32, ch: &str) -> serde_json::Value { + let uri = server.file_uri(SystemPath::new("src/main.by")); + let answer = server.send_request_await::( + DocumentOnTypeFormattingParams::new( + TextDocumentIdentifier::new(uri), + Position::new(line, character), + ch.to_string(), + FormattingOptions::new(4, true, None, None, None), + ), + ); + serde_json::to_value(answer).expect("an answer serialises") +} + +#[test] +fn the_colon_is_the_trigger() -> Result<()> { + let server = server_with("x = 1\n")?; + let provider = &server + .initialization_result() + .expect("initialized") + .capabilities; + let provider = serde_json::to_value(&provider.document_on_type_formatting_provider)?; + assert_eq!( + provider, + serde_json::json!({ "firstTriggerCharacter": ":" }) + ); + Ok(()) +} + +#[test] +fn an_else_typed_in_the_body_is_moved_out_to_its_if() -> Result<()> { + let mut server = server_with("def f(x):\n if x:\n a = 1\n else:\n")?; + // the caret is after the `:` just typed + let edits = typed(&mut server, 3, 13, ":"); + assert_eq!( + edits, + serde_json::json!([{ + "range": { + "start": { "line": 3, "character": 0 }, + "end": { "line": 3, "character": 8 }, + }, + "newText": " ", + }]) + ); + Ok(()) +} + +#[test] +fn a_colon_that_ends_nothing_changes_nothing() -> Result<()> { + let mut server = server_with("if x:\n d = {1: 2}\n")?; + assert_eq!(typed(&mut server, 1, 11, ":"), serde_json::Value::Null); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap index 5a7fced022..8b81e65593 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap @@ -69,6 +69,9 @@ expression: initialization_result "resolveProvider": false }, "workspaceSymbolProvider": true, + "documentOnTypeFormattingProvider": { + "firstTriggerCharacter": ":" + }, "renameProvider": { "prepareProvider": true }, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap index 5a7fced022..8b81e65593 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap @@ -69,6 +69,9 @@ expression: initialization_result "resolveProvider": false }, "workspaceSymbolProvider": true, + "documentOnTypeFormattingProvider": { + "firstTriggerCharacter": ":" + }, "renameProvider": { "prepareProvider": true }, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 8a968505da..3e9668f32d 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -389,6 +389,78 @@ the pairing is read off the parse tree, so `match = 1` is an assignment and nothing lights up on it. a position that is not a keyword is answered the way it always was +## typing a clause + +a clause that continues a compound statement — `elif`, `else`, `except`, +`finally`, a further `case` — belongs at the indentation of the clause it +continues, and an editor starts a new line at the indentation of the body it +closes. the server advertises `textDocument/onTypeFormatting` with `:` as its +trigger, and answers the `:` that finishes such a header with the edit that +moves the clause out: + +```by +for item in items: + if item: + handle(item) + else: # typed here, moved out to the `if` when the `:` goes in +``` + +the statement it moves to is the innermost one still open above the line that +can take that clause next and is not indented deeper than the line already is. +a clause is only ever moved out, so an `else:` typed at the loop's column stays +with the loop, and an `if` that already has its `else` is passed over. which +statements are open is read from a parse of the text above the line alone, +since the clause itself is a syntax error until it is where it belongs. a `:` +that does not end the header — in a slice, a dict or a lambda — and +`case: int = 1`, an annotated variable called `case`, are answered with no edit + +## completing a statement + +`by/completeStatement` finishes the statement at a position and starts the line +after it, which is what an editor's *complete current statement* does: + +```json +{ + "textDocument": { "uri": "file:///src/main.by" }, + "position": { "line": 1, "character": 10 }, + "options": { "tabSize": 4, "insertSpaces": true }, + "textHash": "…" +} +``` + +a single-quoted string left open on the line is closed, the brackets left open +on it are closed innermost first, a clause header gets its `:` — and a `def` +its `()` — and a clause that continues a compound statement is moved out the +way typing its `:` moves it. a statement carried on to lines below that parse, +such as a call spread over several lines, is completed where it ends rather +than on the line asked about; a bracket the following lines do not close is +closed where it was left open. the new line goes inside the suite a header +opens, at the indentation of the suite's statements when it has some already +and one level deeper than the header otherwise, and after any other statement +at the statement's own indentation. `options` says what one level is, the way +`textDocument/onTypeFormatting` does: `tabSize` spaces, or a tab + +the reply is the edits, against the document as it was asked about, and where +the caret goes in the document as the edits leave it: + +```json +{ + "edits": [ + { + "range": { + "start": { "line": 1, "character": 10 }, + "end": { "line": 1, "character": 10 } + }, + "newText": "):\n " + } + ], + "caret": { "line": 2, "character": 8 } +} +``` + +like the other document requests it takes `textHash`, and it is `null` when +language services are off + ## go to super `textDocument/implementation` on a method goes down, to the methods that From f2f30a1013b03e768399ed2e66cf478213ab9cff Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:35:33 +1000 Subject: [PATCH 10/15] report every comma-separated list and each definition's declared names in by/syntaxOutline, and offer putting a list's elements on separate lines or on one line as the refactor.rewrite.splitList and joinList code actions --- crates/ty_ide/src/comma_list.rs | 299 +++++++++++++ crates/ty_ide/src/lib.rs | 5 +- crates/ty_ide/src/refactor.rs | 11 + crates/ty_ide/src/refactor/split_join.rs | 394 ++++++++++++++++++ crates/ty_ide/src/syntax_outline.rs | 169 +++++++- .../src/server/api/requests/syntax_outline.rs | 44 +- docs/basedpython/features/editor.md | 17 +- 7 files changed, 932 insertions(+), 7 deletions(-) create mode 100644 crates/ty_ide/src/comma_list.rs create mode 100644 crates/ty_ide/src/refactor/split_join.rs diff --git a/crates/ty_ide/src/comma_list.rs b/crates/ty_ide/src/comma_list.rs new file mode 100644 index 0000000000..9021a473c5 --- /dev/null +++ b/crates/ty_ide/src/comma_list.rs @@ -0,0 +1,299 @@ +//! Comma-separated lists — a call's arguments, a definition's parameters, the elements of a +//! display, the names of an import — and the elements of each, as the parser read them. +//! +//! An element is read off the parse tree, not off the text between two commas: a lambda among the +//! arguments has commas of its own (`f(lambda a, b: a, c)` has two arguments), and so does a nested +//! call, and only the tree tells those apart from the list's. What the tree has no node for is +//! found in the gaps between the nodes, where the only tokens are the list's own: the `/` and bare +//! `*` that separate parameters, which are elements of their own, and the `*`, `**` that unpack an +//! element or mark a variadic parameter, which belong to the element after them. + +use ruff_python_ast::token::{Token, TokenKind, Tokens, parenthesized_range}; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal}; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr}; +use ruff_text_size::{Ranged, TextRange}; + +/// What a list holds, which is what a client calls its elements. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ListKind { + /// A call's arguments, or a class's bases and keywords. + Arguments, + /// A `def`'s or a `lambda`'s parameters. + Parameters, + /// A type parameter list. + TypeParameters, + /// The elements of a list, tuple, set or dict display, or of a subscript's tuple. + Elements, + /// The names an `import`, `global` or `nonlocal` statement lists. + Names, + /// The targets of a `del`. + Targets, + /// The context managers of a `with`. + ContextManagers, +} + +/// One comma-separated list. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CommaList { + pub(crate) kind: ListKind, + /// The bracket tokens around the list, when it has brackets of its own: a call's parentheses, + /// a display's brackets, a parenthesised tuple's parentheses. `None` for a list that runs + /// without them, like a lambda's parameters or an import's names. + pub(crate) brackets: Option<(TextRange, TextRange)>, + /// The elements, in source order, each from its first token to its last: parentheses around + /// an element, and a `*` or `**` before one, included. + pub(crate) elements: Vec, +} + +impl CommaList { + /// Between the brackets, both excluded; `None` for a list without brackets of its own. + pub(crate) fn inside(&self) -> Option { + let (open, close) = self.brackets?; + Some(TextRange::new(open.end(), close.start())) + } +} + +/// The list `node` is, if it is one. +/// +/// `None` for anything else, and for a list whose elements could not all be placed — which a +/// parse that recovered from an error can leave behind. +pub(crate) fn comma_list(node: AnyNodeRef, tokens: &Tokens) -> Option { + let expressions = |elements: &[Expr], parent: AnyNodeRef| -> Vec { + elements + .iter() + .map(|element| { + parenthesized_range(element.into(), parent, tokens).unwrap_or(element.range()) + }) + .collect() + }; + + let (kind, brackets, bounds, nodes) = match node { + AnyNodeRef::Arguments(arguments) => { + let nodes = arguments + .iter_source_order() + .map(|argument| match argument { + ast::ArgOrKeyword::Arg(expr) => { + parenthesized_range(expr.into(), node, tokens).unwrap_or(expr.range()) + } + ast::ArgOrKeyword::Keyword(keyword) => keyword.range(), + }) + .collect(); + ( + ListKind::Arguments, + bracketed(arguments.range(), tokens, TokenKind::Lpar, TokenKind::Rpar), + arguments.range(), + nodes, + ) + } + AnyNodeRef::Parameters(parameters) => { + let nodes = parameters + .iter() + .map(|parameter| parameter.range()) + .collect(); + ( + ListKind::Parameters, + bracketed(parameters.range(), tokens, TokenKind::Lpar, TokenKind::Rpar), + parameters.range(), + nodes, + ) + } + AnyNodeRef::TypeParams(type_params) => ( + ListKind::TypeParameters, + bracketed( + type_params.range(), + tokens, + TokenKind::Lsqb, + TokenKind::Rsqb, + ), + type_params.range(), + type_params.type_params.iter().map(Ranged::range).collect(), + ), + AnyNodeRef::ExprList(list) => ( + ListKind::Elements, + bracketed(list.range(), tokens, TokenKind::Lsqb, TokenKind::Rsqb), + list.range(), + expressions(&list.elts, node), + ), + AnyNodeRef::ExprSet(set) => ( + ListKind::Elements, + bracketed(set.range(), tokens, TokenKind::Lbrace, TokenKind::Rbrace), + set.range(), + expressions(&set.elts, node), + ), + AnyNodeRef::ExprTuple(tuple) => ( + ListKind::Elements, + if tuple.parenthesized { + bracketed(tuple.range(), tokens, TokenKind::Lpar, TokenKind::Rpar) + } else { + None + }, + tuple.range(), + expressions(&tuple.elts, node), + ), + AnyNodeRef::ExprDict(dict) => { + let nodes = dict + .items + .iter() + .map(|item| { + let value = parenthesized_range((&item.value).into(), node, tokens) + .unwrap_or(item.value.range()); + match &item.key { + Some(key) => parenthesized_range(key.into(), node, tokens) + .unwrap_or(key.range()) + .cover(value), + // the `**` before it is found in the gap + None => value, + } + }) + .collect(); + ( + ListKind::Elements, + bracketed(dict.range(), tokens, TokenKind::Lbrace, TokenKind::Rbrace), + dict.range(), + nodes, + ) + } + AnyNodeRef::StmtImport(import) => names( + ListKind::Names, + import.names.iter().map(Ranged::range).collect(), + )?, + AnyNodeRef::StmtImportFrom(import) => names( + ListKind::Names, + import.names.iter().map(Ranged::range).collect(), + )?, + AnyNodeRef::StmtGlobal(global) => names( + ListKind::Names, + global.names.iter().map(Ranged::range).collect(), + )?, + AnyNodeRef::StmtNonlocal(nonlocal) => names( + ListKind::Names, + nonlocal.names.iter().map(Ranged::range).collect(), + )?, + AnyNodeRef::StmtDelete(delete) => { + names(ListKind::Targets, expressions(&delete.targets, node))? + } + AnyNodeRef::StmtWith(with) => names( + ListKind::ContextManagers, + with.items.iter().map(Ranged::range).collect(), + )?, + _ => return None, + }; + + let bounds = match brackets { + Some((open, close)) => TextRange::new(open.end(), close.start()), + None => bounds, + }; + let elements = with_gaps(bounds, &nodes, tokens)?; + Some(CommaList { + kind, + brackets, + elements, + }) +} + +/// A list of `nodes` with no brackets of its own, bounded by its first and last node. +#[expect(clippy::type_complexity)] +fn names( + kind: ListKind, + nodes: Vec, +) -> Option<( + ListKind, + Option<(TextRange, TextRange)>, + TextRange, + Vec, +)> { + let bounds = nodes.first()?.cover(*nodes.last()?); + Some((kind, None, bounds, nodes)) +} + +/// The `open` and `close` tokens at either end of `range`, when it starts and ends with them. +fn bracketed( + range: TextRange, + tokens: &Tokens, + open: TokenKind, + close: TokenKind, +) -> Option<(TextRange, TextRange)> { + let inner = tokens_within(tokens, range); + let first = inner.iter().find(|token| !token.kind().is_trivia())?; + let last = inner.iter().rev().find(|token| !token.kind().is_trivia())?; + (first.kind() == open + && last.kind() == close + && first.start() == range.start() + && last.end() == range.end()) + .then(|| (first.range(), last.range())) +} + +/// The elements of a list whose nodes are `nodes`, reading what lies between them inside +/// `bounds`: a run of tokens between two commas is an element of its own (a `/` or bare `*` +/// among parameters), and a run that ends at a node is part of it (the `**` of `**kwargs`). +/// +/// `None` when the nodes are not in order inside the bounds, or a run between them is not one the +/// list can hold, either of which is a parse recovered from an error. +fn with_gaps(bounds: TextRange, nodes: &[TextRange], tokens: &Tokens) -> Option> { + let mut elements = Vec::with_capacity(nodes.len()); + let mut position = bounds.start(); + for index in 0..=nodes.len() { + let next = nodes.get(index).copied(); + let gap_end = next.map_or(bounds.end(), TextRange::start); + if gap_end < position { + return None; + } + let mut run: Option = None; + for token in tokens_within(tokens, TextRange::new(position, gap_end)) { + match token.kind() { + kind if kind.is_trivia() => {} + TokenKind::Comma => { + elements.extend(run.take()); + } + TokenKind::Star | TokenKind::DoubleStar | TokenKind::Slash => { + run = Some(run.map_or(token.range(), |run| run.cover(token.range()))); + } + _ => return None, + } + } + match next { + Some(node) => { + if node.is_empty() { + return None; + } + elements.push(run.map_or(node, |run| run.cover(node))); + position = node.end(); + } + None => elements.extend(run), + } + } + Some(elements) +} + +/// The tokens wholly inside `range`. +/// +/// Unlike [`Tokens::in_range`], this does not panic on a range whose ends fall inside a token, +/// which the nodes of a parse recovered from an error can have: the outline that reads these is +/// asked about every keystroke's text, the broken ones included. +pub(crate) fn tokens_within(tokens: &Tokens, range: TextRange) -> &[Token] { + let start = tokens.partition_point(|token| token.start() < range.start()); + let end = tokens.partition_point(|token| token.end() <= range.end()); + tokens.get(start..end.max(start)).unwrap_or_default() +} + +/// Every list in `suite` and everything nested in it, in the order their nodes start. +pub(crate) fn comma_lists(suite: &[ast::Stmt], tokens: &Tokens) -> Vec { + struct Collector<'t> { + tokens: &'t Tokens, + lists: Vec, + } + + impl<'a> SourceOrderVisitor<'a> for Collector<'_> { + fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal { + self.lists.extend(comma_list(node, self.tokens)); + TraversalSignal::Traverse + } + } + + let mut collector = Collector { + tokens, + lists: Vec::new(), + }; + collector.visit_body(suite); + collector.lists +} diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index d7bd2be79b..c3d8d717ab 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -7,6 +7,7 @@ mod alignment; mod all_symbols; mod call_hierarchy; mod code_action; +mod comma_list; mod common_aliases; mod complete_statement; mod completion; @@ -106,8 +107,8 @@ pub use signature_help::{ParameterDetails, SignatureDetails, SignatureHelpInfo, pub use super_members::{OverridingMember, SuperMember, document_super_members, super_members}; pub use symbols::{FlatSymbols, HierarchicalSymbols, SymbolId, SymbolInfo, SymbolKind}; pub use syntax_outline::{ - OutlineCall, OutlineClause, OutlineModifier, OutlineStatement, OutlineString, SyntaxOutline, - syntax_outline, + OutlineCall, OutlineClause, OutlineList, OutlineModifier, OutlineStatement, OutlineString, + SyntaxOutline, syntax_outline, }; pub use test_links::{NewTest, SubjectTests, TestLink, subject_tests, test_subjects}; pub use ty_python_semantic::types::ide_support::InheritedMethodKind; diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 03090e621c..950dfe4fe0 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -24,6 +24,7 @@ mod hazards; mod inline_variable; mod names; mod return_annotation; +mod split_join; mod text; use ruff_db::files::File; @@ -48,6 +49,8 @@ pub enum RefactorKind { AddReturnAnnotation, ConvertToDataClass, ConvertFromDataClass, + SplitList, + JoinList, } impl RefactorKind { @@ -59,6 +62,8 @@ impl RefactorKind { RefactorKind::AddReturnAnnotation, RefactorKind::ConvertToDataClass, RefactorKind::ConvertFromDataClass, + RefactorKind::SplitList, + RefactorKind::JoinList, ]; /// The LSP code action kind the refactoring is offered as. @@ -71,6 +76,8 @@ impl RefactorKind { RefactorKind::AddReturnAnnotation => "refactor.rewrite.returnAnnotation", RefactorKind::ConvertToDataClass => "refactor.rewrite.toDataClass", RefactorKind::ConvertFromDataClass => "refactor.rewrite.fromDataClass", + RefactorKind::SplitList => "refactor.rewrite.splitList", + RefactorKind::JoinList => "refactor.rewrite.joinList", } } @@ -85,6 +92,8 @@ impl RefactorKind { RefactorKind::AddReturnAnnotation => "add-return-annotation", RefactorKind::ConvertToDataClass => "convert-to-data-class", RefactorKind::ConvertFromDataClass => "convert-from-data-class", + RefactorKind::SplitList => "split-list", + RefactorKind::JoinList => "join-list", } } @@ -200,6 +209,8 @@ impl<'db> RefactorContext<'db> { RefactorKind::ConvertFromDataClass => { data_class::plan(self, range, data_class::Direction::FromDataClass) } + RefactorKind::SplitList => split_join::plan(self, range, split_join::Direction::Split), + RefactorKind::JoinList => split_join::plan(self, range, split_join::Direction::Join), } } } diff --git a/crates/ty_ide/src/refactor/split_join.rs b/crates/ty_ide/src/refactor/split_join.rs new file mode 100644 index 0000000000..8dd635bedf --- /dev/null +++ b/crates/ty_ide/src/refactor/split_join.rs @@ -0,0 +1,394 @@ +//! Put a list's elements each on a line of its own, or all on one line: a call's arguments, a +//! definition's parameters, the elements of a display. +//! +//! The list is the innermost one with brackets of its own around the range that the rewrite +//! applies to, so a range inside a list that is already split splits nothing and is offered the +//! list around it instead. Only the text between the brackets changes, and only the text between +//! the elements: each element is kept as it is written. +//! +//! A split list ends with a comma, which a formatter reads as the list being meant to stay split; +//! a joined one loses it, except for a one-element tuple, which needs it to be a tuple. + +use ruff_diagnostics::Edit; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::token::TokenKind; +use ruff_python_ast::{AnyNodeRef, StringFlags}; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextRange}; + +use super::{Plan, RefactorContext, Refusal}; +use crate::comma_list::{CommaList, ListKind, comma_list, tokens_within}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Direction { + /// Each element on a line of its own. + Split, + /// Every element on the line the list starts on. + Join, +} + +pub(super) fn plan( + context: &RefactorContext<'_>, + range: TextRange, + direction: Direction, +) -> Result { + let module = context.parsed.syntax(); + if !module.range().contains_range(range) { + return Err(Refusal::NotApplicable); + } + let tokens = context.parsed.tokens(); + let covering = covering_node(module.into(), range); + let ancestors: Vec = covering.ancestors().collect(); + + for (index, node) in ancestors.iter().enumerate() { + let Some(list) = comma_list(*node, tokens) else { + continue; + }; + let (Some((open, close)), Some(inside)) = (list.brackets, list.inside()) else { + continue; + }; + if !TextRange::new(open.start(), close.end()).contains_range(range) { + continue; + } + let gaps = gaps(&list, inside); + let broken = gaps + .iter() + .filter(|gap| context.is_multiline(**gap)) + .count(); + let applies = match direction { + Direction::Split => list.elements.len() >= 2 && broken < gaps.len(), + Direction::Join => !list.elements.is_empty() && broken > 0, + }; + if !applies { + continue; + } + + let title = title(direction, list.kind); + if tokens_within(tokens, inside) + .iter() + .any(|token| token.kind() == TokenKind::Comment) + { + return Err(Refusal::refused( + title, + "the list has comments in it, which would have to move with the text around them", + )); + } + let in_single_line_string = ancestors[index + 1..] + .iter() + .any(|ancestor| match ancestor { + AnyNodeRef::FString(string) => !string.flags.is_triple_quoted(), + AnyNodeRef::TString(string) => !string.flags.is_triple_quoted(), + _ => false, + }); + if direction == Direction::Split && in_single_line_string { + return Err(Refusal::refused( + title, + "the list is inside an interpolation of a string that is written on one line", + )); + } + + let source = context.source(); + let text = match direction { + Direction::Split => { + let line_ending = context.line_ending(); + // the indentation of the line the list opens on, which its closing bracket goes + // back to + let line = source.line_start(open.start()); + let line_text = &source[TextRange::new(line, source.line_end(line))]; + let base = &line_text[..line_text.len() - line_text.trim_start().len()]; + let indent = format!("{base}{}", context.indent_unit()); + let mut text = String::from(line_ending); + for element in &list.elements { + text.push_str(&indent); + text.push_str(&reindented(context, *element, &indent)); + text.push(','); + text.push_str(line_ending); + } + text.push_str(base); + text + } + Direction::Join => { + let mut text = list + .elements + .iter() + .map(|element| &source[*element]) + .collect::>() + .join(", "); + let lone_tuple_element = + matches!(node, AnyNodeRef::ExprTuple(tuple) if tuple.elts.len() == 1); + if lone_tuple_element { + text.push(','); + } + text + } + }; + return Ok(Plan { + title: title.to_string(), + edits: vec![Edit::range_replacement(text, inside)], + }); + } + Err(Refusal::NotApplicable) +} + +/// The text of `element`, an element that is moving to a line of its own indented by `indent`, +/// with each of its lines after the first moved as far as the line it started on moves: what is +/// nested in it stays indented under it. A line that starts inside a string literal is the +/// string's, and is left as it is. +fn reindented(context: &RefactorContext<'_>, element: TextRange, indent: &str) -> String { + let source = context.source(); + let text = &source[element]; + if !context.is_multiline(element) { + return text.to_string(); + } + let line = source.line_start(element.start()); + let line_text = &source[TextRange::new(line, source.line_end(line))]; + let old = line_text.len() - line_text.trim_start().len(); + let new = indent.len(); + + let tokens = context.parsed.tokens(); + let mut out = String::with_capacity(text.len()); + let mut start = element.start(); + for piece in text.split_inclusive('\n') { + let piece_start = start; + start += ruff_text_size::TextSize::of(piece); + let index = tokens.partition_point(|token| token.end() <= piece_start); + let in_string = tokens.get(index).is_some_and(|token| { + token.start() < piece_start + && matches!( + token.kind(), + TokenKind::String | TokenKind::FStringMiddle | TokenKind::TStringMiddle + ) + }); + if piece_start == element.start() || in_string || piece.trim().is_empty() { + out.push_str(piece); + continue; + } + let leading = piece.len() - piece.trim_start().len(); + let kept = if new >= old { + format!("{}{}", &piece[..leading], " ".repeat(new - old)) + } else { + piece[..leading.saturating_sub(old - new)].to_string() + }; + out.push_str(&kept); + out.push_str(&piece[leading..]); + } + out +} + +/// The stretches of `list` between the brackets that are not its elements: before the first, +/// between each two, and after the last. +fn gaps(list: &CommaList, inside: TextRange) -> Vec { + let mut gaps = Vec::with_capacity(list.elements.len() + 1); + let mut start = inside.start(); + for element in &list.elements { + gaps.push(TextRange::new(start, element.start())); + start = element.end(); + } + gaps.push(TextRange::new(start, inside.end())); + gaps +} + +fn title(direction: Direction, kind: ListKind) -> &'static str { + match (direction, kind) { + (Direction::Split, ListKind::Arguments) => "Put arguments on separate lines", + (Direction::Split, ListKind::Parameters) => "Put parameters on separate lines", + (Direction::Split, ListKind::TypeParameters) => "Put type parameters on separate lines", + (Direction::Split, _) => "Put elements on separate lines", + (Direction::Join, ListKind::Arguments) => "Put arguments on one line", + (Direction::Join, ListKind::Parameters) => "Put parameters on one line", + (Direction::Join, ListKind::TypeParameters) => "Put type parameters on one line", + (Direction::Join, _) => "Put elements on one line", + } +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + + use crate::refactor::RefactorKind; + use crate::refactor::test_support::RefactorTest; + + fn split(source: &str) -> String { + RefactorTest::python(source).apply(RefactorKind::SplitList) + } + + fn join(source: &str) -> String { + RefactorTest::python(source).apply(RefactorKind::JoinList) + } + + #[test] + fn splits_a_call_s_arguments_one_per_line_with_a_trailing_comma() { + assert_snapshot!(split( + " + def g(): + f(a, b=1, *c, **d) + ", + ), @" + Put arguments on separate lines + --- + + def g(): + f( + a, + b=1, + *c, + **d, + ) + "); + } + + #[test] + fn joins_a_split_call_and_drops_the_trailing_comma() { + assert_snapshot!(join( + " + f( + a, + b, + ) + ", + ), @" + Put arguments on one line + --- + + f(a, b) + "); + } + + /// A lambda's commas are its own: the call has two arguments, not three. + #[test] + fn an_element_s_own_commas_do_not_split_it() { + assert_snapshot!(split( + " + f(lambda a, b: a, (c, d)) + ", + ), @" + Put arguments on separate lines + --- + + f( + lambda a, b: a, + (c, d), + ) + "); + } + + /// The `/` and bare `*` separators are parameters of their own as far as the layout goes. + #[test] + fn splits_parameters_with_their_separators() { + assert_snapshot!(split( + " + def f(a, /, b, *, c=1, **d): ... + ", + ), @" + Put parameters on separate lines + --- + + def f( + a, + /, + b, + *, + c=1, + **d, + ): ... + "); + } + + #[test] + fn a_dict_s_entries_and_unpackings_are_its_elements() { + assert_snapshot!(split( + " + x = {'a': 1, **b} + ", + ), @" + Put elements on separate lines + --- + + x = { + 'a': 1, + **b, + } + "); + } + + /// A tuple of one element needs its comma to stay a tuple. + #[test] + fn joining_a_one_element_tuple_keeps_its_comma() { + assert_snapshot!(join( + " + x = ( + a, + ) + ", + ), @" + Put elements on one line + --- + + x = (a,) + "); + } + + /// Inside a list that is split already there is nothing to split, so the list around it is + /// the one offered. + #[test] + fn a_list_that_is_split_already_offers_the_list_around_it() { + assert_snapshot!(split( + " + f(x, [ + 1, + 2, + ]) + ", + ), @" + Put arguments on separate lines + --- + + f( + x, + [ + 1, + 2, + ], + ) + "); + } + + #[test] + fn a_list_on_one_line_is_not_joined() { + assert_snapshot!(join("f(a, b)"), @"not offered"); + } + + #[test] + fn a_single_element_is_not_split() { + assert_snapshot!(split("f(a)"), @"not offered"); + } + + /// Joining would put what follows a comment inside it. + #[test] + fn a_list_with_a_comment_is_refused() { + assert_snapshot!(join( + " + f( + a, # the first + b, + ) + ", + ), @"refused: Put arguments on one line (the list has comments in it, which would have to move with the text around them)"); + } + + /// A line break inside an interpolation of a string written on one line is only valid from + /// Python 3.12. + #[test] + fn a_list_in_a_one_line_f_string_is_not_split() { + assert_snapshot!(split( + r#" + x = f"{g(a, b)}" + "#, + ), @"refused: Put arguments on separate lines (the list is inside an interpolation of a string that is written on one line)"); + } + + #[test] + fn the_caret_on_the_callee_is_not_in_the_list() { + assert_snapshot!(split("f(a, b)"), @"not offered"); + } +} diff --git a/crates/ty_ide/src/syntax_outline.rs b/crates/ty_ide/src/syntax_outline.rs index f2c59e30cc..eedc83cd50 100644 --- a/crates/ty_ide/src/syntax_outline.rs +++ b/crates/ty_ide/src/syntax_outline.rs @@ -23,6 +23,7 @@ use ruff_python_trivia::basedpython::{TripleQuotedDedent, dedent_triple_quoted_b use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::Db; +use crate::comma_list::comma_lists; /// The block structure and string literals of one file. #[derive(Debug, Clone, PartialEq, Eq)] @@ -31,6 +32,18 @@ pub struct SyntaxOutline { pub statements: Vec, /// Every string literal part in the file, in source order. pub strings: Vec, + /// Every comma-separated list of two elements or more, in the order the lists start: a list + /// nested in an element of another comes after it. + pub lists: Vec, +} + +/// A comma-separated list: a call's arguments, a definition's parameters, a display's elements, +/// the names of an `import`, `global` or `del`, the context managers of a `with`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutlineList { + /// Each element from its first token to its last: parentheses around it, and a `*` or `**` + /// before it, included. The `/` and bare `*` among parameters are elements of their own. + pub elements: Vec, } /// One statement. @@ -46,6 +59,10 @@ pub struct OutlineStatement { pub modifiers: Vec, /// For an expression statement that is a call, the call. pub call: Option, + /// For a `def` or a `class`, the name it declares. + pub name: Option, + /// For a `def`, the names of its parameters, in source order. + pub parameters: Vec, } /// One clause of a compound statement: `if`, `elif`, `else`, `case`, the `def` of a function. @@ -116,6 +133,13 @@ pub fn syntax_outline(db: &dyn Db, file: PythonFile<'_>) -> SyntaxOutline { }; let statements = builder.statements(parsed.suite()); + let lists = comma_lists(parsed.suite(), parsed.tokens()) + .into_iter() + .filter(|list| list.elements.len() >= 2) + .map(|list| OutlineList { + elements: list.elements, + }) + .collect(); let mut strings = StringCollector { source: source.as_str(), @@ -128,6 +152,7 @@ pub fn syntax_outline(db: &dyn Db, file: PythonFile<'_>) -> SyntaxOutline { SyntaxOutline { statements, strings: strings.strings, + lists, } } @@ -195,8 +220,8 @@ fn keyword_at(tokens: &Tokens, offset: TextSize) -> Option { } pub(crate) struct OutlineBuilder<'a> { - source: &'a str, - tokens: &'a Tokens, + pub(crate) source: &'a str, + pub(crate) tokens: &'a Tokens, } impl<'a> OutlineBuilder<'a> { @@ -216,6 +241,8 @@ impl OutlineBuilder<'_> { clauses: self.clauses(stmt), modifiers: self.modifiers(stmt), call: self.call(stmt), + name: Self::name(stmt), + parameters: Self::parameters(stmt), } } @@ -235,6 +262,30 @@ impl OutlineBuilder<'_> { .collect() } + /// The name a `def` or `class` declares. A trailing lambda declares none, and neither does a + /// definition the parser recovered without one. + fn name(stmt: &Stmt) -> Option { + let name = match stmt { + Stmt::FunctionDef(function) if !function.is_trailing_lambda => &function.name, + Stmt::ClassDef(class) => &class.name, + _ => return None, + }; + (!name.range().is_empty()).then(|| name.range()) + } + + /// The names of a `def`'s parameters. + fn parameters(stmt: &Stmt) -> Vec { + match stmt { + Stmt::FunctionDef(function) if !function.is_trailing_lambda => function + .parameters + .iter() + .map(|parameter| parameter.name().range()) + .filter(|name| !name.is_empty()) + .collect(), + _ => Vec::new(), + } + } + /// The clauses of `stmt` without their bodies' outlines, which is all a family needs. pub(crate) fn clause_heads<'s>(&self, stmt: &'s Stmt) -> Vec> { let mut heads = Vec::new(); @@ -535,9 +586,13 @@ impl OutlineBuilder<'_> { } } +/// One clause of a compound statement, as [`OutlineBuilder::clause_heads`] finds it. pub(crate) struct ClauseHead<'s> { + /// The clause keyword; `None` on a trailing lambda's clause. pub(crate) keyword: Option, + /// The `:` opening the suite, when there is one to find. pub(crate) colon: Option, + /// From the start of the header to the end of the body. pub(crate) range: TextRange, pub(crate) body: &'s [Stmt], } @@ -1272,4 +1327,114 @@ else: "while@2 break@3" ); } + + /// Every list, one per line, as its elements between bars. + fn lists(source: &str) -> String { + let test = test("main.by", source); + let file = test.program_file(test.cursor.file).python_file(&test.db); + let outline = syntax_outline(&test.db, file); + let text = test.cursor.source.as_str(); + let mut out = String::new(); + for list in &outline.lists { + let elements: Vec<&str> = list + .elements + .iter() + .map(|range| &text[range.to_std_range()]) + .collect(); + writeln!(out, "|{}|", elements.join("|")).unwrap(); + } + out + } + + /// The names the module's definitions declare, and their parameters', depth first. + fn names(source: &str) -> String { + fn walk(statements: &[OutlineStatement], text: &str, out: &mut String) { + for statement in statements { + if let Some(name) = statement.name { + write!(out, "{}", &text[name.to_std_range()]).unwrap(); + let parameters: Vec<&str> = statement + .parameters + .iter() + .map(|range| &text[range.to_std_range()]) + .collect(); + if !parameters.is_empty() { + write!(out, "({})", parameters.join(", ")).unwrap(); + } + out.push('\n'); + } + for clause in &statement.clauses { + walk(&clause.body, text, out); + } + } + } + + let test = test("main.by", source); + let file = test.program_file(test.cursor.file).python_file(&test.db); + let outline = syntax_outline(&test.db, file); + let mut out = String::new(); + walk(&outline.statements, test.cursor.source.as_str(), &mut out); + out + } + + /// An element is a node, so the commas of a lambda, a nested call or a tuple among the + /// arguments are not the call's own; each list of two elements or more is reported, nested + /// ones after the one they are in. + #[test] + fn a_list_s_elements_are_its_nodes_not_the_text_between_commas() { + assert_eq!( + lists("f(lambda a, b: a, g(c, d), (e, h), *i, k=1, **m)\n"), + "\ +|lambda a, b: a|g(c, d)|(e, h)|*i|k=1|**m| +|a|b| +|c|d| +|e|h| +" + ); + } + + /// The `/` and bare `*` among parameters are elements of their own, and the `*` and `**` of + /// a variadic parameter belong to it. + #[test] + fn parameters_keep_their_separators_and_stars() { + assert_eq!( + lists("def f(a, /, b: int = 1, *args, c, **kw): pass\n"), + "|a|/|b: int = 1|*args|c|**kw|\n" + ); + assert_eq!(lists("def f(a, *, b): pass\n"), "|a|*|b|\n"); + } + + #[test] + fn displays_imports_and_statements_list_their_elements() { + assert_eq!( + lists( + "from m import a, b as c\nx = {1: 2, **d}\ny, z = [1, 2], {3, 4}\ndel x, y\nwith a as b, c:\n pass\n" + ), + "\ +|a|b as c| +|1: 2|**d| +|y|z| +|[1, 2]|{3, 4}| +|1|2| +|3|4| +|x|y| +|a as b|c| +" + ); + } + + /// A list of one element has nothing to move among, and is not reported. + #[test] + fn a_list_of_one_element_is_left_out() { + assert_eq!(lists("f(a)\nx = [1]\n"), ""); + } + + #[test] + fn definitions_report_their_names_and_parameters() { + assert_eq!( + names( + "class C:\n def m(self, x, *rest, **kw): pass\ndef f(): pass\nitems.each:\n pass\n" + ), + "C\nm(self, x, rest, kw)\nf\n" + ); + } } diff --git a/crates/ty_server/src/server/api/requests/syntax_outline.rs b/crates/ty_server/src/server/api/requests/syntax_outline.rs index a198736cf8..b14817d3d3 100644 --- a/crates/ty_server/src/server/api/requests/syntax_outline.rs +++ b/crates/ty_server/src/server/api/requests/syntax_outline.rs @@ -20,7 +20,7 @@ use std::borrow::Cow; use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, TextDocumentIdentifier, Uri}; use ruff_db::files::File; use ruff_text_size::TextRange; -use ty_ide::{OutlineStatement, OutlineString, syntax_outline}; +use ty_ide::{OutlineList, OutlineStatement, OutlineString, syntax_outline}; use ty_project::{ProjectDatabase, SemanticDb as _}; use crate::PositionEncoding; @@ -56,6 +56,19 @@ pub(crate) struct SyntaxOutlineResponse { statements: Vec, /// Every string literal part, in source order. strings: Vec, + /// Every comma-separated list of two elements or more — a call's arguments, a definition's + /// parameters, a display's elements, an import's names — in the order the lists start. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + lists: Vec, +} + +/// A comma-separated list. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct List { + /// Each element from its first token to its last: parentheses around it, and a `*` or `**` + /// before it, included. The `/` and bare `*` among parameters are elements of their own. + elements: Vec, } /// One statement. @@ -74,6 +87,12 @@ pub(crate) struct Statement { /// On an expression statement that is a call: the call. #[serde(default, skip_serializing_if = "Option::is_none")] call: Option, + /// On a `def` or a `class`: the name it declares. + #[serde(default, skip_serializing_if = "Option::is_none")] + name: Option, + /// On a `def`: the names of its parameters, in source order. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + parameters: Vec, } /// One clause of a compound statement. @@ -189,6 +208,11 @@ impl BackgroundDocumentRequestHandler for SyntaxOutlineRequestHandler { .iter() .filter_map(|string| converter.string(string)) .collect(), + lists: outline + .lists + .iter() + .filter_map(|list| converter.list(list)) + .collect(), })) } } @@ -243,11 +267,29 @@ impl Converter<'_> { positional_only: call.positional_only, }) }), + name: statement.name.and_then(|range| self.range(range)), + parameters: statement + .parameters + .iter() + .filter_map(|range| self.range(*range)) + .collect(), }) }) .collect() } + /// The list, or `None` when an element of it does not map: a list missing an element is + /// not the same list. + fn list(&self, list: &OutlineList) -> Option { + Some(List { + elements: list + .elements + .iter() + .map(|range| self.range(*range)) + .collect::>()?, + }) + } + fn string(&self, string: &OutlineString) -> Option { Some(StringPart { range: self.range(string.range)?, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 3e9668f32d..a0bfbf4e6e 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -348,6 +348,8 @@ one that asks while the caret moves is only offered what applies | add return annotation | `refactor.rewrite.returnAnnotation` | a `def` header with no `->` | | convert to data class | `refactor.rewrite.toDataClass` | a class header | | convert from data class | `refactor.rewrite.fromDataClass` | a `data class` header | +| put on separate lines | `refactor.rewrite.splitList` | a list not yet one element a line | +| put on one line | `refactor.rewrite.joinList` | a list spread over several lines | moving an expression can change when it runs, so that is what most refusals are about. a value with an effect is only inlined into a single read in the very next @@ -380,6 +382,14 @@ decided by working out the rewrite, so the actions a `textDocument/codeAction` reply offers have each been computed to answer it. send `only` to narrow that — a request that does not ask for `refactor` kinds does no refactoring work at all +putting a list on separate lines, or on one, rewrites the innermost list with +brackets of its own around the caret — a call's arguments, a definition's +parameters, a display's elements — and only the text between its elements. an +element is a node of the parse, so a lambda among the arguments keeps its commas. +a split list ends with a comma, which tells a formatter it is meant to stay split, +and a joined one loses it unless it is a tuple of one. a list with a comment in it +is refused, since joining would put what follows the comment inside it + ## keyword highlighting `textDocument/documentHighlight` on a keyword answers the keywords that belong @@ -714,7 +724,7 @@ instead, from the same reading the transpiler and the checker use: | `by/testItems` | the tests pytest collects from a document, with their node ids | | `by/runModules` | the modules `by run` can be given, and the one `run.main` names | | `by/buildOutput` | where a project's build writes, and which output came from which source | -| `by/syntaxOutline` | a document's suites, clause keywords, call statements and string parts | +| `by/syntaxOutline` | a document's suites, clause keywords, declared names, lists and strings | `by/buildOutput` maps both ways: given a `.by` it names the file the build writes it to, given a file in the build directory it names the source. the path inside @@ -733,7 +743,10 @@ the build wrote to serve features from: it is what an editor needs to type in an indentation-delimited language — which line opens a suite, where a compound statement ends, what a string literal's escapes and interpolations are, and how -much indentation a triple-quoted string has stripped +much indentation a triple-quoted string has stripped. it also carries the name +each `def` and `class` declares and a `def`'s parameter names, and the elements of +every comma-separated list of two or more — what an editor moves an argument among, +or tracks a declaration being renamed by `.by` test files are collected under the same names as python ones — `test_*.by` and `*_test.by` — because a `.by` transpiles to the `.py` of the same stem, and From 9388c64090b126d72f98d88f7c476cac4f4f4c42 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:35:36 +1000 Subject: [PATCH 11/15] answer by/unwrap with the statements around a range that unwrap into one of their suites and the clauses that can be removed, each with the ranges it rewrites and keeps and edits that dedent the kept suites line by line --- crates/ty_ide/src/lib.rs | 2 + ...sts__a_comment_ending_a_suite_is_kept.snap | 9 + ...a_file_that_does_not_parse_is_refused.snap | 5 + ...__a_loop_whose_body_breaks_is_refused.snap | 9 + ...ti_line_string_s_lines_are_left_alone.snap | 9 + ...a_nested_function_s_return_is_its_own.snap | 9 + ...sts__a_nested_loop_s_break_is_its_own.snap | 9 + ...ap__tests__a_suite_on_the_header_line.snap | 6 + ...hat_can_leave_early_keeps_its_finally.snap | 5 + ...aps_into_the_path_that_raises_nothing.snap | 8 + ..._tests__an_else_unwraps_or_is_removed.snap | 14 + ..._a_try_with_an_else_cannot_be_removed.snap | 5 + ...eps_its_suite_at_the_if_s_indentation.snap | 12 + crates/ty_ide/src/unwrap.rs | 645 ++++++++++++++++++ crates/ty_server/src/server/api.rs | 6 + crates/ty_server/src/server/api/requests.rs | 2 + .../src/server/api/requests/unwrap.rs | 201 ++++++ docs/basedpython/features/editor.md | 22 + 18 files changed, 978 insertions(+) create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_comment_ending_a_suite_is_kept.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_file_that_does_not_parse_is_refused.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_loop_whose_body_breaks_is_refused.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_multi_line_string_s_lines_are_left_alone.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_nested_function_s_return_is_its_own.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_nested_loop_s_break_is_its_own.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_suite_on_the_header_line.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_try_that_can_leave_early_keeps_its_finally.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_try_unwraps_into_the_path_that_raises_nothing.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__an_else_unwraps_or_is_removed.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__the_last_except_of_a_try_with_an_else_cannot_be_removed.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__unwrapping_an_if_keeps_its_suite_at_the_if_s_indentation.snap create mode 100644 crates/ty_ide/src/unwrap.rs create mode 100644 crates/ty_server/src/server/api/requests/unwrap.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index c3d8d717ab..408ec00a33 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -46,6 +46,7 @@ mod symbols; mod syntax_outline; mod test_links; mod type_hierarchy; +mod unwrap; mod usage_counts; mod workspace_symbols; @@ -115,6 +116,7 @@ pub use ty_python_semantic::types::ide_support::InheritedMethodKind; pub use type_hierarchy::{ TypeHierarchyItem, prepare_type_hierarchy, type_hierarchy_subtypes, type_hierarchy_supertypes, }; +pub use unwrap::{UnwrapOption, unwrap_options}; pub use usage_counts::{ DeclarationKind, DeclarationUsages, UsageCountKinds, document_usage_counts, }; diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_comment_ending_a_suite_is_kept.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_comment_ending_a_suite_is_kept.snap new file mode 100644 index 0000000000..3fb67b397f --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_comment_ending_a_suite_is_kept.snap @@ -0,0 +1,9 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\nif x:\n a()\n\n # keep this\nelse:\n b()\nc()\n\")" +--- +Unwrap 'if...' keeps ["a()\n\n # keep this"] +a() + +# keep this +c() diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_file_that_does_not_parse_is_refused.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_file_that_does_not_parse_is_refused.snap new file mode 100644 index 0000000000..cc96bb33c0 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_file_that_does_not_parse_is_refused.snap @@ -0,0 +1,5 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"if x:\\n a(\\n\")" +--- +refused: the file has syntax errors diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_loop_whose_body_breaks_is_refused.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_loop_whose_body_breaks_is_refused.snap new file mode 100644 index 0000000000..5549cab105 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_loop_whose_body_breaks_is_refused.snap @@ -0,0 +1,9 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\nfor a in b:\n if a:\n break\n\")" +--- +Unwrap 'if...' keeps ["break"] +for a in b: + break +--- +Unwrap 'for...': refused (`break` in the loop's body would no longer leave this loop) diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_multi_line_string_s_lines_are_left_alone.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_multi_line_string_s_lines_are_left_alone.snap new file mode 100644 index 0000000000..db07ac5a19 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_multi_line_string_s_lines_are_left_alone.snap @@ -0,0 +1,9 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\nwith open(p) as f:\n text = \\\"\\\"\\\"\n keep\n\\\"\\\"\\\"\n g(text)\n\")" +--- +Unwrap 'with...' keeps ["text = \"\"\"\n keep\n\"\"\"\n g(text)"] +text = """ + keep +""" +g(text) diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_nested_function_s_return_is_its_own.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_nested_function_s_return_is_its_own.snap new file mode 100644 index 0000000000..d2b178ac75 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_nested_function_s_return_is_its_own.snap @@ -0,0 +1,9 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\ntry:\n def g():\n return 1\n a()\nfinally:\n d()\n\")" +--- +Unwrap 'try...' keeps ["def g():\n return 1\n a()", "d()"] +def g(): + return 1 +a() +d() diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_nested_loop_s_break_is_its_own.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_nested_loop_s_break_is_its_own.snap new file mode 100644 index 0000000000..6826bfdf1d --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_nested_loop_s_break_is_its_own.snap @@ -0,0 +1,9 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\nfor a in b:\n for c in d:\n break\n\")" +--- +Unwrap 'for...': refused (`break` in the loop's body would no longer leave this loop) +--- +Unwrap 'for...' keeps ["for c in d:\n break"] +for c in d: + break diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_suite_on_the_header_line.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_suite_on_the_header_line.snap new file mode 100644 index 0000000000..e7d56d59df --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_suite_on_the_header_line.snap @@ -0,0 +1,6 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"if x: a(); b()\\n\")" +--- +Unwrap 'if...' keeps ["a(); b()"] +a(); b() diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_try_that_can_leave_early_keeps_its_finally.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_try_that_can_leave_early_keeps_its_finally.snap new file mode 100644 index 0000000000..eceab7b80d --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_try_that_can_leave_early_keeps_its_finally.snap @@ -0,0 +1,5 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\ndef f():\n try:\n if x:\n return 1\n a()\n finally:\n d()\n\")" +--- +Unwrap 'try...': refused (`return` would no longer run the `finally` suite) diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_try_unwraps_into_the_path_that_raises_nothing.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_try_unwraps_into_the_path_that_raises_nothing.snap new file mode 100644 index 0000000000..93cb8b40c8 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__a_try_unwraps_into_the_path_that_raises_nothing.snap @@ -0,0 +1,8 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\ntry:\n a()\nexcept E:\n b()\nelse:\n c()\nfinally:\n d()\n\")" +--- +Unwrap 'try...' keeps ["a()", "c()", "d()"] +a() +c() +d() diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__an_else_unwraps_or_is_removed.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__an_else_unwraps_or_is_removed.snap new file mode 100644 index 0000000000..ac298fe909 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__an_else_unwraps_or_is_removed.snap @@ -0,0 +1,14 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\nif x:\n a()\nelif y:\n b()\nelse:\n c()\nd()\n\")" +--- +Unwrap 'else...' keeps ["c()"] +c() +d() +--- +Remove 'else...' keeps [] +if x: + a() +elif y: + b() +d() diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__the_last_except_of_a_try_with_an_else_cannot_be_removed.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__the_last_except_of_a_try_with_an_else_cannot_be_removed.snap new file mode 100644 index 0000000000..c23885b718 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__the_last_except_of_a_try_with_an_else_cannot_be_removed.snap @@ -0,0 +1,5 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\ntry:\n a()\nexcept E:\n b()\nelse:\n c()\n\")" +--- +Remove 'except...': refused (a `try` with an `else` needs an `except`) diff --git a/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__unwrapping_an_if_keeps_its_suite_at_the_if_s_indentation.snap b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__unwrapping_an_if_keeps_its_suite_at_the_if_s_indentation.snap new file mode 100644 index 0000000000..3cc667388c --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__unwrap__tests__unwrapping_an_if_keeps_its_suite_at_the_if_s_indentation.snap @@ -0,0 +1,12 @@ +--- +source: crates/ty_ide/src/unwrap.rs +expression: "unwrap(\"\\\ndef f():\n if x:\n # why\n a = 1\n b(\n 2,\n )\n else:\n c()\n d()\n\")" +--- +Unwrap 'if...' keeps ["# why\n a = 1\n b(\n 2,\n )"] +def f(): + # why + a = 1 + b( + 2, + ) + d() diff --git a/crates/ty_ide/src/unwrap.rs b/crates/ty_ide/src/unwrap.rs new file mode 100644 index 0000000000..64052bfaca --- /dev/null +++ b/crates/ty_ide/src/unwrap.rs @@ -0,0 +1,645 @@ +//! Unwrapping a compound statement, or removing one of its clauses. +//! +//! Unwrapping an `if` replaces the whole statement with the suite of the clause the range is in, +//! dedented to where the statement stood; removing its `else` deletes that clause and leaves the +//! rest. These are edits the user asks for to change what a program does, so unlike the +//! refactorings they are not refused for changing it. They are refused when the result would not +//! say what the user can see it saying: a `break` in an unwrapped loop's body would leave some +//! other loop, or none, a `try` cannot be left without an `except` or a `finally`, and a +//! `finally` suite would no longer run after a `return` or a `raise` in the suites before it. +//! +//! Every statement around the range is offered, innermost first, since that is the order in +//! which they enclose it. +//! +//! The rewrite is spelled as many small edits rather than one replacement of the statement: the +//! header deleted, then each line of the suite losing one level of indentation. An editor then +//! keeps the caret, and anything else it has anchored in the suite, where it was in the text. +//! The lines of a string literal that spans several lines are left alone, because their +//! indentation is part of the string. + +use ruff_db::PythonFile; +use ruff_db::parsed::parsed_module; +use ruff_db::source::source_text; +use ruff_diagnostics::Edit; +use ruff_python_ast::Stmt; +use ruff_python_ast::token::{TokenKind, Tokens}; +use ruff_python_codegen::Stylist; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; + +use crate::Db; +use crate::syntax_outline::{ClauseHead, OutlineBuilder}; + +/// One way to unwrap or remove around a range. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnwrapOption { + /// What the option does, as an editor lists it: `Unwrap 'if...'`, `Remove 'else...'`. + pub title: String, + /// What the option rewrites: the whole statement for an unwrap, the clause for a removal. + pub range: TextRange, + /// The parts of [`Self::range`] that stay, which are the suites an unwrap keeps. Empty for a + /// removal. + pub kept: Vec, + /// The edits, or why the option cannot be applied. + pub edits: Result, String>, +} + +/// What can be unwrapped or removed around `range` in `file`, innermost first. +/// +/// `Err` with the reason when the file cannot be rewritten at all, which is when it does not +/// parse: the statements a recovered parse makes up are not ones the user wrote. +pub fn unwrap_options( + db: &dyn Db, + file: PythonFile<'_>, + range: TextRange, +) -> Result, String> { + let parsed = parsed_module(db, file).load(db); + if parsed.has_invalid_syntax() { + return Err("the file has syntax errors".to_string()); + } + let source = source_text(db, file.file(db)); + let stylist = Stylist::from_tokens(parsed.tokens(), source.as_str()); + let unwrapper = Unwrapper { + source: source.as_str(), + line_ending: stylist.line_ending().as_str(), + tokens: parsed.tokens(), + builder: OutlineBuilder { + source: source.as_str(), + tokens: parsed.tokens(), + }, + }; + + let mut enclosing = Vec::new(); + unwrapper.enclosing(parsed.suite(), range, &mut enclosing); + + let mut options = Vec::new(); + for stmt in enclosing.into_iter().rev() { + unwrapper.options(stmt, range, &mut options); + } + Ok(options) +} + +struct Unwrapper<'a> { + source: &'a str, + /// What the file ends its lines with, for the line breaks an unwrap writes. + line_ending: &'static str, + tokens: &'a Tokens, + builder: OutlineBuilder<'a>, +} + +impl Unwrapper<'_> { + /// The statements around `range`, outermost first. + fn enclosing<'s>(&self, suite: &'s [Stmt], range: TextRange, out: &mut Vec<&'s Stmt>) { + let Some(stmt) = suite.iter().find(|stmt| stmt.range().contains_range(range)) else { + return; + }; + out.push(stmt); + for head in self.builder.clause_heads(stmt) { + if head.range.contains_range(range) { + self.enclosing(head.body, range, out); + } + } + } + + /// The options `stmt` offers for a range in one of its clauses. + fn options(&self, stmt: &Stmt, range: TextRange, out: &mut Vec) { + let heads = self.builder.clause_heads(stmt); + let Some(index) = heads + .iter() + .position(|head| head.range.contains_range(range)) + else { + return; + }; + let kinds: Vec> = heads.iter().map(|head| self.keyword(head)).collect(); + let keyword = |index: usize| -> &str { + heads[index] + .keyword + .map_or("", |keyword| &self.source[keyword]) + }; + + match stmt { + Stmt::If(_) => { + out.push(self.unwrap(stmt, &heads, &[index], keyword(index), None)); + if index > 0 { + out.push(self.remove(&heads[index], keyword(index), None)); + } + } + Stmt::For(_) | Stmt::While(_) => { + if index == 0 { + let refusal = self.exit(heads[0].body, true, false).map(|exit| { + format!("`{exit}` in the loop's body would no longer leave this loop") + }); + out.push(self.unwrap(stmt, &heads, &[0], keyword(0), refusal)); + } else { + out.push(self.unwrap(stmt, &heads, &[index], keyword(index), None)); + out.push(self.remove(&heads[index], keyword(index), None)); + } + } + Stmt::Try(try_stmt) => match kinds[index] { + Some(TokenKind::Try) => { + // the path through the statement that raises nothing: the `try` suite, then + // the `else` suite, then the `finally` suite + let kept: Vec = (0..heads.len()) + .filter(|&clause| { + matches!( + kinds[clause], + Some(TokenKind::Try | TokenKind::Else | TokenKind::Finally) + ) + }) + .collect(); + // a `finally` also runs when the suites before it are left early, which the + // unwrapped statements would skip it on + let refusal = if try_stmt.finalbody.is_empty() { + None + } else { + [&try_stmt.body, &try_stmt.orelse] + .into_iter() + .find_map(|body| self.exit(body, true, true)) + .map(|exit| format!("`{exit}` would no longer run the `finally` suite")) + }; + out.push(self.unwrap(stmt, &heads, &kept, keyword(0), refusal)); + } + Some(TokenKind::Except) => { + let handlers = try_stmt.handlers.len(); + let refusal = if handlers > 1 { + None + } else if !try_stmt.orelse.is_empty() { + Some("a `try` with an `else` needs an `except`".to_string()) + } else if try_stmt.finalbody.is_empty() { + Some("a `try` needs an `except` or a `finally`".to_string()) + } else { + None + }; + out.push(self.remove(&heads[index], keyword(index), refusal)); + } + Some(TokenKind::Else) => { + out.push(self.remove(&heads[index], keyword(index), None)); + } + Some(TokenKind::Finally) => { + let refusal = try_stmt + .handlers + .is_empty() + .then(|| "a `try` needs an `except` or a `finally`".to_string()); + out.push(self.remove(&heads[index], keyword(index), refusal)); + } + _ => {} + }, + Stmt::With(_) if index == 0 => { + out.push(self.unwrap(stmt, &heads, &[0], keyword(0), None)); + } + _ => {} + } + } + + /// The kind of `head`'s keyword token, passing over the zero-width `Dedent`s that start where + /// a dedented clause keyword does. + fn keyword(&self, head: &ClauseHead) -> Option { + let keyword = head.keyword?; + let index = self + .tokens + .partition_point(|token| token.start() < keyword.start()); + self.tokens[index..] + .iter() + .find(|token| !token.range().is_empty()) + .map(ruff_python_ast::token::Token::kind) + } + + /// The first statement in `body` that leaves it other than by running to its end, outside any + /// function or class `body` defines: with `loop_jumps`, a `break` or `continue` not inside a + /// nested loop's own suite, and with `returns`, a `return` or a `raise`. + fn exit(&self, body: &[Stmt], loop_jumps: bool, returns: bool) -> Option<&'static str> { + for stmt in body { + match stmt { + Stmt::Break(_) if loop_jumps => return Some("break"), + Stmt::Continue(_) if loop_jumps => return Some("continue"), + Stmt::Return(_) if returns => return Some("return"), + Stmt::Raise(_) if returns => return Some("raise"), + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} + // a nested loop's own suite has its own `break`s; its `else` does not + Stmt::For(_) | Stmt::While(_) => { + for (index, head) in self.builder.clause_heads(stmt).iter().enumerate() { + if let Some(exit) = self.exit(head.body, loop_jumps && index > 0, returns) { + return Some(exit); + } + } + } + _ => { + for head in self.builder.clause_heads(stmt) { + if let Some(exit) = self.exit(head.body, loop_jumps, returns) { + return Some(exit); + } + } + } + } + } + None + } + + /// Replacing `stmt` with the suites of the clauses `kept` (in source order), dedented to where + /// `stmt` starts. + fn unwrap( + &self, + stmt: &Stmt, + heads: &[ClauseHead], + kept: &[usize], + keyword: &str, + refusal: Option, + ) -> UnwrapOption { + let title = format!("Unwrap '{keyword}...'"); + let suites: Option> = kept + .iter() + .map(|&index| self.suite(&heads[index])) + .collect(); + let Some(suites) = suites else { + return UnwrapOption { + title, + range: stmt.range(), + kept: Vec::new(), + edits: Err("a clause has no suite to keep".to_string()), + }; + }; + let kept_ranges = suites.iter().map(|suite| suite.kept).collect(); + let edits = match refusal { + Some(reason) => Err(reason), + None => self.unwrap_edits(stmt, &suites), + }; + UnwrapOption { + title, + range: stmt.range(), + kept: kept_ranges, + edits, + } + } + + fn unwrap_edits(&self, stmt: &Stmt, suites: &[Suite]) -> Result, String> { + let source = self.source; + let line_start = source.line_start(stmt.start()); + let target = &source[TextRange::new(line_start, stmt.start())]; + if !target.chars().all(char::is_whitespace) { + return Err("the statement does not start its line".to_string()); + } + + let mut edits = Vec::new(); + let mut previous_end: Option = None; + for suite in suites { + match previous_end { + // the header and everything before the first suite kept + None => edits.push(Edit::deletion(stmt.start(), suite.kept.start())), + // the clauses between two suites kept, and the header of the second + Some(end) => edits.push(Edit::replacement( + format!("{}{target}", self.line_ending), + source.line_end(end), + suite.kept.start(), + )), + } + if let Some(indent) = &suite.indent { + edits.extend(self.dedent(suite, indent, target)); + } + previous_end = Some(suite.kept.end()); + } + // the clauses after the last suite kept + if let Some(end) = previous_end { + let from = source.line_end(end); + let to = source.line_end(stmt.end()); + if from < to { + edits.push(Edit::deletion(from, to)); + } + } + Ok(edits) + } + + /// The edits taking each line of `suite` after its first from `indent` to `target`. + fn dedent(&self, suite: &Suite, indent: &str, target: &str) -> Vec { + let source = self.source; + let mut edits = Vec::new(); + let mut line = source.full_line_end(suite.kept.start()); + while line < suite.kept.end() { + let next = source.full_line_end(line); + let text = &source[TextRange::new(line, next)]; + let in_string = self.inside_string(line); + if !in_string && text.starts_with(indent) && !text.trim().is_empty() { + let indent_end = line + indent.text_len(); + if let Some(extra) = indent.strip_prefix(target) { + if !extra.is_empty() { + edits.push(Edit::deletion(line + target.text_len(), indent_end)); + } + } else { + edits.push(Edit::replacement(target.to_string(), line, indent_end)); + } + } + line = next; + } + edits + } + + /// Whether `offset` is inside a string literal's text, rather than between tokens. + fn inside_string(&self, offset: TextSize) -> bool { + let index = self.tokens.partition_point(|token| token.end() <= offset); + self.tokens.get(index).is_some_and(|token| { + token.start() < offset + && matches!( + token.kind(), + TokenKind::String | TokenKind::FStringMiddle | TokenKind::TStringMiddle + ) + }) + } + + /// The suite of `head` as an unwrap keeps it. + fn suite(&self, head: &ClauseHead) -> Option { + let source = self.source; + let first = head.body.first()?; + let last = head.body.last()?; + let colon = head.colon?; + let on_header_line = + !source[TextRange::new(colon.end(), first.start())].contains(['\n', '\r']); + if on_header_line { + return Some(Suite { + kept: TextRange::new(first.start(), last.end()), + indent: None, + }); + } + let indent = + source[TextRange::new(source.line_start(first.start()), first.start())].to_string(); + // the first line with anything on it after the header: a comment above the first + // statement is kept with it + let mut line = source.full_line_end(colon.end()); + while line < first.start() + && source[TextRange::new(line, source.line_end(line))] + .trim() + .is_empty() + { + line = source.full_line_end(line); + } + let content = line + + source[TextRange::new(line, source.line_end(line))] + .chars() + .take_while(|c| c.is_whitespace()) + .map(TextLen::text_len) + .sum::(); + // the comments after the last statement, at the suite's indentation or deeper, are the + // suite's too + let mut end = last.end(); + let mut line = source.full_line_end(last.end()); + while line < source.text_len() { + let text = &source[TextRange::new(line, source.line_end(line))]; + if text + .strip_prefix(indent.as_str()) + .is_some_and(|rest| rest.trim_start().starts_with('#')) + && !self.inside_string(line) + { + end = source.line_end(line); + } else if !text.trim().is_empty() { + break; + } + line = source.full_line_end(line); + } + Some(Suite { + kept: TextRange::new(content.min(first.start()), end), + indent: Some(indent), + }) + } + + /// Deleting the clause `head`, from the start of its line to the end of its suite's last. + fn remove(&self, head: &ClauseHead, keyword: &str, refusal: Option) -> UnwrapOption { + let source = self.source; + let title = format!("Remove '{keyword}...'"); + let start = source.line_start(head.range.start()); + let edits = match refusal { + Some(reason) => Err(reason), + None if !source[TextRange::new(start, head.range.start())] + .chars() + .all(char::is_whitespace) => + { + Err("the clause does not start its line".to_string()) + } + None => Ok(vec![Edit::deletion( + start, + source.full_line_end(head.range.end()), + )]), + }; + UnwrapOption { + title, + range: head.range, + kept: Vec::new(), + edits, + } + } +} + +/// A suite an unwrap keeps. +struct Suite { + /// From its first comment or statement to the end of its last statement, or of the comments + /// after it. + kept: TextRange, + /// The indentation of its lines; `None` for a suite on its header's line. + indent: Option, +} + +#[cfg(test)] +mod tests { + use ruff_text_size::{Ranged, TextRange}; + + use super::unwrap_options; + use crate::tests::CursorTest; + + /// Every option at ``, innermost first, each with the text it keeps and the file it + /// leaves, or the reason it is refused. + fn unwrap(source: &str) -> String { + let test = CursorTest::builder().source("main.by", source).build(); + let file = test.program_file(test.cursor.file).python_file(&test.db); + let text = test.cursor.source.as_str(); + let options = match unwrap_options(&test.db, file, TextRange::empty(test.cursor.offset)) { + Ok(options) => options, + Err(reason) => return format!("refused: {reason}"), + }; + options + .iter() + .map(|option| { + let kept: Vec<&str> = option.kept.iter().map(|range| &text[*range]).collect(); + match &option.edits { + Ok(edits) => { + let mut result = text.to_string(); + let mut edits = edits.clone(); + edits.sort_by_key(|edit| std::cmp::Reverse(edit.start())); + for edit in edits { + result.replace_range( + edit.range().to_std_range(), + edit.content().unwrap_or_default(), + ); + } + format!("{} keeps {kept:?}\n{result}", option.title) + } + Err(reason) => format!("{}: refused ({reason})\n", option.title), + } + }) + .collect::>() + .join("---\n") + } + + /// The suite takes the `if`'s place, one level shallower, with the comment above its first + /// statement and the lines inside its brackets; the `else` goes with the `if`. + #[test] + fn unwrapping_an_if_keeps_its_suite_at_the_if_s_indentation() { + insta::assert_snapshot!(unwrap( + "\ +def f(): + if x: + # why + a = 1 + b( + 2, + ) + else: + c() + d() +" + )); + } + + /// In an `else` both are offered: the statement replaced by that suite, or the clause gone. + #[test] + fn an_else_unwraps_or_is_removed() { + insta::assert_snapshot!(unwrap( + "\ +if x: + a() +elif y: + b() +else: + c() +d() +" + )); + } + + /// The lines of a string that spans several lines are the string's, and stay as they are. + #[test] + fn a_multi_line_string_s_lines_are_left_alone() { + insta::assert_snapshot!(unwrap( + "\ +with open(p) as f: + text = \"\"\" + keep +\"\"\" + g(text) +" + )); + } + + /// Without its loop, a `break` would leave the loop around it, or be an error. + #[test] + fn a_loop_whose_body_breaks_is_refused() { + insta::assert_snapshot!(unwrap( + "\ +for a in b: + if a: + break +" + )); + } + + /// A nested loop's own `break` does not stop the loop around it from unwrapping. + #[test] + fn a_nested_loop_s_break_is_its_own() { + insta::assert_snapshot!(unwrap( + "\ +for a in b: + for c in d: + break +" + )); + } + + /// Unwrapping a `try` keeps the path that raises nothing: the `try` suite, the `else` suite, + /// then the `finally` suite. + #[test] + fn a_try_unwraps_into_the_path_that_raises_nothing() { + insta::assert_snapshot!(unwrap( + "\ +try: + a() +except E: + b() +else: + c() +finally: + d() +" + )); + } + + /// A `finally` runs however its `try` is left, so a `try` that can leave other than by + /// running to its end does not unwrap: the `finally` suite would no longer run after a + /// `return`, a `raise`, or a `break` out of the loop around it. + #[test] + fn a_try_that_can_leave_early_keeps_its_finally() { + insta::assert_snapshot!(unwrap( + "\ +def f(): + try: + if x: + return 1 + a() + finally: + d() +" + )); + } + + /// A `return` in a function the `try` suite defines does not leave the `try`. + #[test] + fn a_nested_function_s_return_is_its_own() { + insta::assert_snapshot!(unwrap( + "\ +try: + def g(): + return 1 + a() +finally: + d() +" + )); + } + + /// The comments after a suite's last statement, at its indentation or deeper, are the + /// suite's, and stay with it. + #[test] + fn a_comment_ending_a_suite_is_kept() { + insta::assert_snapshot!(unwrap( + "\ +if x: + a() + + # keep this +else: + b() +c() +" + )); + } + + #[test] + fn the_last_except_of_a_try_with_an_else_cannot_be_removed() { + insta::assert_snapshot!(unwrap( + "\ +try: + a() +except E: + b() +else: + c() +" + )); + } + + /// A suite written on its header's line is kept as it is written. + #[test] + fn a_suite_on_the_header_line() { + insta::assert_snapshot!(unwrap("if x: a(); b()\n")); + } + + #[test] + fn a_file_that_does_not_parse_is_refused() { + insta::assert_snapshot!(unwrap("if x:\n a(\n")); + } +} diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 1abc4728b6..a3e3b1e637 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -231,6 +231,12 @@ pub(super) fn request(req: server::Request) -> Task { requests::ModuleNameHandler::METHOD => { background_request_task::(req, BackgroundSchedule::Worker) } + // The user invoked Unwrap/Remove and waits on a progress for the answer. + requests::UnwrapRequestHandler::METHOD => background_document_request_task::< + requests::UnwrapRequestHandler, + >( + req, BackgroundSchedule::LatencySensitive + ), requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 9eb900c018..b58876d9ef 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -67,6 +67,7 @@ mod transpile; mod transpile_for_build; mod type_hierarchy_subtypes; mod type_hierarchy_supertypes; +mod unwrap; mod will_rename_files; mod workspace_diagnostic; mod workspace_symbols; @@ -126,6 +127,7 @@ pub(super) use transpile::TranspileRequestHandler; pub(super) use transpile_for_build::TranspileForBuildRequestHandler; pub(super) use type_hierarchy_subtypes::TypeHierarchySubtypesRequestHandler; pub(super) use type_hierarchy_supertypes::TypeHierarchySupertypesRequestHandler; +pub(super) use unwrap::UnwrapRequestHandler; pub(super) use will_rename_files::WillRenameFilesRequestHandler; pub(super) use workspace_diagnostic::WorkspaceDiagnosticRequestHandler; pub(super) use workspace_symbols::WorkspaceSymbolRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/unwrap.rs b/crates/ty_server/src/server/api/requests/unwrap.rs new file mode 100644 index 0000000000..f3ba209230 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/unwrap.rs @@ -0,0 +1,201 @@ +//! `by/unwrap` — what an editor's *Unwrap/Remove* offers around a range: each compound statement +//! that can be unwrapped into one of its suites, and each clause that can be removed, innermost +//! first. +//! +//! LSP has a request for rewrites, `textDocument/codeAction`, but not one an editor can show these +//! with: it lists the options in a popup and, while one is selected, marks what it deletes apart +//! from what it keeps, which a code action carries nothing to say. So each option comes with the +//! range it rewrites, the ranges of it that stay, and the edits. + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Range, Request, TextDocumentIdentifier, TextEdit, Uri, +}; +use ruff_db::files::File; +use ruff_text_size::{Ranged, TextRange}; +use ty_ide::unwrap_options; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::PositionEncoding; +use crate::document::{RangeExt, ToRangeExt}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum UnwrapRequest {} + +impl Request for UnwrapRequest { + type Params = UnwrapParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/unwrap"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// Where to unwrap: the document, and the caret or selection. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct UnwrapParams { + text_document: TextDocumentIdentifier, + range: Range, +} + +/// The options around the range. +#[derive(Debug, Default, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct UnwrapResponse { + /// Innermost first. Empty when nothing around the range is a statement that unwraps. + options: Vec, + /// Why there are no options at all, when that is not for want of a statement around the + /// range: the file does not parse. + #[serde(default, skip_serializing_if = "Option::is_none")] + refused: Option, +} + +/// One way to unwrap or remove. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct UnwrapOption { + /// What to list it as: `Unwrap 'if...'`, `Remove 'else...'`. + title: String, + /// What it rewrites: the whole statement for an unwrap, the clause for a removal. + range: Range, + /// The parts of `range` that stay. Empty for a removal. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + kept: Vec, + /// The edits to the document, many small ones rather than one: the header deleted, and each + /// line of a kept suite dedented, so that what an editor anchors in the suite stays put. + /// Absent when the option cannot be applied, and `refused` says why. + #[serde(default, skip_serializing_if = "Option::is_none")] + edits: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + refused: Option, +} + +pub(crate) struct UnwrapRequestHandler; + +impl RequestHandler for UnwrapRequestHandler { + type RequestType = UnwrapRequest; +} + +impl BackgroundDocumentRequestHandler for UnwrapRequestHandler { + fn document_uri(params: &UnwrapParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, and for a notebook or a template, which this does + /// not rewrite. + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: UnwrapParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + // a notebook's ranges are per cell, and a cell's suites are not rewritten across cells + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(range) = params + .range + .to_text_range(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + + let options = match unwrap_options(db, db.program_file(file).python_file(db), range) { + Ok(options) => options, + Err(reason) => { + return Ok(Some(UnwrapResponse { + options: Vec::new(), + refused: Some(reason), + })); + } + }; + let converter = Converter { + db, + file, + encoding: snapshot.encoding(), + }; + Ok(Some(UnwrapResponse { + options: options + .iter() + .filter_map(|option| converter.option(option)) + .collect(), + refused: None, + })) + } +} + +impl RetriableRequestHandler for UnwrapRequestHandler {} + +struct Converter<'a> { + db: &'a ProjectDatabase, + file: File, + encoding: PositionEncoding, +} + +impl Converter<'_> { + fn range(&self, range: TextRange) -> Option { + range + .to_lsp_range(self.db, self.file, self.encoding) + .map(|range| range.local_range()) + } + + /// The option as the client reads it, or `None` when one of its ranges cannot be said in the + /// client's terms — an option with an edit missing would apply only part of itself. + fn option(&self, option: &ty_ide::UnwrapOption) -> Option { + let (edits, refused) = match &option.edits { + Ok(edits) => { + let edits = edits + .iter() + .map(|edit| { + Some(TextEdit { + range: self.range(edit.range())?, + new_text: edit.content().unwrap_or_default().to_string(), + }) + }) + .collect::>>()?; + (Some(edits), None) + } + Err(reason) => (None, Some(reason.clone())), + }; + Some(UnwrapOption { + title: option.title.clone(), + range: self.range(option.range)?, + kept: option + .kept + .iter() + .map(|range| self.range(*range)) + .collect::>>()?, + edits, + refused, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The wire form, exactly as a client sends it. + #[test] + fn the_params_a_client_sends_parse() { + let parsed: UnwrapParams = serde_json::from_str( + r#"{"textDocument":{"uri":"file:///main.by"},"range":{"start":{"line":1,"character":4},"end":{"line":1,"character":4}}}"#, + ) + .expect("a client sends a document and a range"); + assert_eq!(parsed.text_document.uri.as_str(), "file:///main.by"); + assert_eq!(parsed.range.start.line, 1); + } +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index a0bfbf4e6e..28fa8f8856 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -390,6 +390,28 @@ a split list ends with a comma, which tells a formatter it is meant to stay spli and a joined one loses it unless it is a tuple of one. a list with a comment in it is refused, since joining would put what follows the comment inside it +## unwrap and remove + +`by/unwrap` answers what an editor's *unwrap/remove* offers around a range, innermost +first: each `if`, loop, `try` or `with` around it can be replaced by one of its +suites, dedented to where the statement stood, and each `elif`, `else`, `except` or +`finally` the range is in can be removed. a `try` unwraps into the path that raises +nothing — its suite, then its `else`, then its `finally` + +these change what a program does, which is what they are asked for, so they are +not refused for that. they are refused when the result would not say what it seems +to: a loop whose body `break`s or `continue`s does not unwrap, since the jump would +leave some other loop or none, a `try` keeps an `except` or a `finally`, and a +`try` with a `finally` does not unwrap when its suite can `return`, `raise`, +`break` or `continue`, since the `finally` would no longer run after it. comments +after a kept suite's last statement go with the suite + +each option comes with the range it rewrites and the ranges of it that stay, for an +editor to mark the one selected, and its edits are many small ones — the header +deleted, each line of a kept suite dedented — so the caret stays where it was in +the text. the lines of a string literal that spans several lines are left as they +are, because their indentation is part of the string + ## keyword highlighting `textDocument/documentHighlight` on a keyword answers the keywords that belong From c61bf1d609e89730912eb4d46c99db0047b307dc Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:07:43 +1000 Subject: [PATCH 12/15] mark names bound in a function's scope with a functionScope semantic token modifier, report each string part's text runs in by/syntaxOutline, and answer by/transpile for a document the client has not opened --- crates/ty_ide/src/injection.rs | 2 +- crates/ty_ide/src/semantic_tokens.rs | 285 +++++++++++++++--- crates/ty_ide/src/syntax_outline.rs | 116 +++++++ .../src/server/api/requests/syntax_outline.rs | 11 + .../src/server/api/requests/transpile.rs | 5 + docs/basedpython/features/editor.md | 25 +- 6 files changed, 399 insertions(+), 45 deletions(-) diff --git a/crates/ty_ide/src/injection.rs b/crates/ty_ide/src/injection.rs index 7c143b675d..4e4998d09a 100644 --- a/crates/ty_ide/src/injection.rs +++ b/crates/ty_ide/src/injection.rs @@ -397,7 +397,7 @@ fn content_ranges( /// here exactly as it will at runtime. Empty runs are dropped: a blank line /// contributes only the newline that follows it, and the range before that /// newline is nothing at all. -fn text_ranges(body: TextRange, source: &str) -> Box<[TextRange]> { +pub(crate) fn text_ranges(body: TextRange, source: &str) -> Box<[TextRange]> { let Some(text) = source.get(body.start().to_usize()..body.end().to_usize()) else { return Box::from([body]); }; diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index 80968a995e..61426d3323 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -54,6 +54,7 @@ use toml_parser::parser::{Event, EventKind}; use ty_project::script_tag; use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; +use ty_python_core::scope::ScopeKind; use ty_python_semantic::{ HasType, ImportAliasResolution, ResolvedDefinition, SemanticModel, definitions_for_attribute, definitions_for_imported_symbol, @@ -163,6 +164,12 @@ bitflags! { /// produced for django templates, where it separates a builtin tag or /// filter from one the project registered. const DEFAULT_LIBRARY = 1 << 4; + /// The name is bound in the scope of a function, a lambda or a + /// comprehension: a parameter or a local, as against a module global or + /// a class attribute. An editor colours these per name — "rainbow" + /// highlighting — which only reads well for names that live and die with + /// one call. + const FUNCTION_SCOPE = 1 << 5; } } @@ -183,6 +190,7 @@ impl SemanticTokenModifier { "async", "documentation", "defaultLibrary", + "functionScope", ] } } @@ -689,16 +697,19 @@ impl<'db> SemanticTokenVisitor<'db> { /// A name a pattern binds. basedpython's match types match on *types*, so a capture /// there declares a type variable the case body is written in terms of. - fn add_capture_token(&mut self, range: TextRange) { + fn add_capture_token(&mut self, name: &ast::Identifier) { let (token_type, modifiers) = if self.in_match_type { ( SemanticTokenType::TypeParameter, SemanticTokenModifier::DEFINITION, ) } else { - (SemanticTokenType::Variable, SemanticTokenModifier::empty()) + ( + SemanticTokenType::Variable, + self.binding_scope_modifiers(name), + ) }; - self.add_token(range, token_type, modifiers); + self.add_token(name.range(), token_type, modifiers); } /// basedpython: highlight the `let` of an `if let := :` @@ -819,17 +830,84 @@ impl<'db> SemanticTokenVisitor<'db> { ty_python_semantic::ImportAliasResolution::ResolveAliases, ); - if let Some(definition) = definition { - let name_str = name.id.as_str(); - if let Some(classification) = self.classify_from_definition(definition, name_str) { - return Some(classification); + let name_str = name.id.as_str(); + let (token_type, mut modifiers) = definition + .and_then(|definition| self.classify_from_definition(definition, name_str)) + .or_else(|| { + // Fall back to type-based classification. + let ty = name.inferred_type(self.model).unwrap_or(Type::unknown()); + self.classify_from_type_and_name_str(ty, name_str) + })?; + + if self.binds_in_function_scope(name, definition) { + modifiers |= SemanticTokenModifier::FUNCTION_SCOPE; + } + Some((token_type, modifiers)) + } + + /// Whether `name` is bound in a function's, a lambda's or a comprehension's + /// scope, given the definition it resolves to. See + /// [`SemanticTokenModifier::FUNCTION_SCOPE`]. + fn binds_in_function_scope( + &self, + name: &ast::ExprName, + resolved: Option>, + ) -> bool { + let db = self.model.db(); + let binding = match resolved { + // resolved through an import to where another module defines it: the + // name here is bound by the import, which is the scope it lives in + Some(definition) if definition.program_file(db) != self.model.program_file() => { + definition_for_name( + self.model, + name, + ty_python_semantic::ImportAliasResolution::PreserveAliases, + ) + } + resolved => resolved, + }; + binding.is_some_and(|definition| self.is_function_scoped(definition)) + } + + /// Whether `definition` binds its name in a function's, a lambda's or a + /// comprehension's scope. + fn is_function_scoped(&self, definition: Definition<'db>) -> bool { + let db = self.model.db(); + match definition.scope(db).scope(db).kind() { + ScopeKind::Function | ScopeKind::Lambda => true, + // an assignment expression in a comprehension binds in the scope around + // the comprehension, which at the top of a module is the module's + ScopeKind::Comprehension => { + !matches!(definition.kind(db), DefinitionKind::NamedExpression(_)) + || !definition.place(db).as_symbol().is_some_and(|symbol| { + ty_python_core::semantic_index(db, definition.program_file(db)) + .symbol_resolves_to_global_scope(symbol, definition.file_scope(db)) + }) + } + ScopeKind::Module | ScopeKind::Class | ScopeKind::TypeParams | ScopeKind::TypeAlias => { + false } } + } - // Fall back to type-based classification. - let ty = name.inferred_type(self.model).unwrap_or(Type::unknown()); - let name_str = name.id.as_str(); - self.classify_from_type_and_name_str(ty, name_str) + /// The modifiers of the name a binding statement introduces without an + /// expression of its own — an `except ... as` name, a pattern's capture — + /// read off the definitions the semantic index keys on `node`. + fn binding_scope_modifiers<'a>( + &self, + node: impl Into>, + ) -> SemanticTokenModifier { + let db = self.model.db(); + let index = ty_python_core::semantic_index(db, self.model.program_file()); + let in_function = index + .try_definitions(node.into()) + .and_then(|definitions| definitions.first().copied()) + .is_some_and(|definition| self.is_function_scoped(definition)); + if in_function { + SemanticTokenModifier::FUNCTION_SCOPE + } else { + SemanticTokenModifier::empty() + } } fn classify_from_definition( @@ -1280,10 +1358,12 @@ impl<'db> SemanticTokenVisitor<'db> { ); } + // a parameter is always bound in the scope of the function or lambda + // that declares it self.add_token( parameter.name.range(), token_type, - SemanticTokenModifier::DEFINITION, + SemanticTokenModifier::DEFINITION | SemanticTokenModifier::FUNCTION_SCOPE, ); // Handle parameter type annotations @@ -1466,7 +1546,7 @@ impl<'db> SemanticTokenVisitor<'db> { } else if binds_parameter { Some(( SemanticTokenType::Parameter, - SemanticTokenModifier::DEFINITION, + SemanticTokenModifier::DEFINITION | SemanticTokenModifier::FUNCTION_SCOPE, )) } else { None @@ -1891,12 +1971,13 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { } } ast::Stmt::Nonlocal(nonlocal_stmt) => { - // Handle nonlocal statements - classify identifiers as variables + // Handle nonlocal statements - classify identifiers as variables, + // which a `nonlocal` always finds in an enclosing function for identifier in &nonlocal_stmt.names { self.add_token( identifier.range(), SemanticTokenType::Variable, - SemanticTokenModifier::empty(), + SemanticTokenModifier::FUNCTION_SCOPE, ); } } @@ -2002,7 +2083,8 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { self.add_token( name.range(), SemanticTokenType::Variable, - SemanticTokenModifier::DEFINITION, + SemanticTokenModifier::DEFINITION + | self.binding_scope_modifiers(except_handler), ); } self.visit_body(&except_handler.body); @@ -2086,7 +2168,7 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { // a place named by a narrowing return annotation classifies as the // parameter it refers to, which no binding in this scope resolves if let Some(token_type) = self.guard_place_classification(name) { - self.add_token(name, token_type, SemanticTokenModifier::empty()); + self.add_token(name, token_type, SemanticTokenModifier::FUNCTION_SCOPE); } else if let Some((token_type, mut modifiers)) = self.classify_name(name) { if self.in_target_creating_definition && name.ctx.is_store() { modifiers |= SemanticTokenModifier::DEFINITION; @@ -2513,7 +2595,7 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { self.add_token( name.range(), SemanticTokenType::Variable, - SemanticTokenModifier::empty(), + self.binding_scope_modifiers(handler), ); } @@ -2533,7 +2615,7 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { // Now add the "as" variable name token if let Some(name) = &pattern_as.name { - self.add_capture_token(name.range()); + self.add_capture_token(name); } } ast::Pattern::MatchMapping(pattern_mapping) => { @@ -2554,7 +2636,7 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { }); if let Some(rest_name) = rest_before_keys { - self.add_capture_token(rest_name.range()); + self.add_capture_token(rest_name); } for (key, nested_pattern) in @@ -2565,13 +2647,13 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { } if let Some(rest_name) = rest_after_keys { - self.add_capture_token(rest_name.range()); + self.add_capture_token(rest_name); } } ast::Pattern::MatchStar(pattern_star) => { // Just the one ident here if let Some(rest_name) = &pattern_star.name { - self.add_capture_token(rest_name.range()); + self.add_capture_token(rest_name); } } _ => { @@ -6188,6 +6270,115 @@ class C: "#); } + /// A parameter, and a name a function binds, are marked as living in the function; + /// a module global is not, even where a function reads it, and neither is a class + /// attribute. + #[test] + fn function_scope_marks_parameters_and_locals() { + let test = SemanticTokenTest::new( + r#" +limit = 10 + +class Box: + size = 1 + +def clamp(value: int) -> int: + result = min(value, limit) + return result +"#, + ); + + let tokens = test.highlight_file(); + + assert_eq!( + test.function_scoped_names(&tokens), + ["value@46", "result@67", "value@80", "result@101"] + ); + } + + /// A lambda's parameters and a comprehension's variables are the lambda's and the + /// comprehension's own, wherever they are written. + #[test] + fn function_scope_marks_lambda_parameters_and_comprehension_variables() { + let test = SemanticTokenTest::new( + r#" +double = lambda n: n * 2 +squares = [k * k for k in range(3)] +"#, + ); + + let tokens = test.highlight_file(); + + assert_eq!( + test.function_scoped_names(&tokens), + ["n@17", "n@20", "k@37", "k@41", "k@47"] + ); + } + + /// The names a statement binds without an expression of its own — an `except ... + /// as` and a pattern's capture — are local where the statement is in a function and + /// global where it is not. + #[test] + fn function_scope_follows_where_a_handler_or_pattern_binds() { + let test = SemanticTokenTest::new( + r#" +try: + pass +except ValueError as outer: + pass + +def f(subject: object): + try: + pass + except ValueError as inner: + pass + match subject: + case [first, *rest]: + pass +"#, + ); + + let tokens = test.highlight_file(); + + assert_eq!( + test.function_scoped_names(&tokens), + [ + "subject@59", + "inner@124", + "subject@151", + "first@175", + "rest@183" + ] + ); + } + + /// An assignment expression in a comprehension binds in the scope around the + /// comprehension (PEP 572), which at the top of a module is the module's. A + /// `nonlocal` name is the enclosing function's, and a `global` one is not a local. + #[test] + fn function_scope_follows_assignment_expressions_global_and_nonlocal() { + let test = SemanticTokenTest::new( + r#" +total = 0 +found = [last := x for x in range(3)] + +def outer(): + count = 0 + def inner(): + nonlocal count + global total + count = total +"#, + ); + + let tokens = test.highlight_file(); + + assert_eq!( + test.function_scoped_names(&tokens), + ["x@27", "x@32", "count@71", "count@114", "count@143"] + ); + } + #[test] fn augmented_assignment() { let test = SemanticTokenTest::new( @@ -8085,6 +8276,20 @@ x = cast(int, "") ) } + /// The tokens marked `functionScope`, each as its text and start offset. + fn function_scoped_names(&self, tokens: &SemanticTokens) -> Vec { + let source = ruff_db::source::source_text(&self.db, self.file); + tokens + .iter() + .filter(|token| { + token + .modifiers + .contains(SemanticTokenModifier::FUNCTION_SCOPE) + }) + .map(|token| format!("{}@{}", &source[token.range()], u32::from(token.start()))) + .collect() + } + /// Helper function to convert semantic tokens to a snapshot-friendly text format fn to_snapshot(&self, tokens: &SemanticTokens) -> String { use std::fmt::Write; @@ -8093,25 +8298,27 @@ x = cast(int, "") for token in tokens.iter() { let token_text = &source[token.range()]; - let modifiers_text = if token.modifiers.is_empty() { + // `functionScope` is left out: it is on every parameter and local, and the + // tests that are about it say so with `function_scoped_names` + let mut mods = Vec::new(); + if token.modifiers.contains(SemanticTokenModifier::DEFINITION) { + mods.push("definition"); + } + if token.modifiers.contains(SemanticTokenModifier::READONLY) { + mods.push("readonly"); + } + if token.modifiers.contains(SemanticTokenModifier::ASYNC) { + mods.push("async"); + } + if token + .modifiers + .contains(SemanticTokenModifier::DOCUMENTATION) + { + mods.push("documentation"); + } + let modifiers_text = if mods.is_empty() { String::new() } else { - let mut mods = Vec::new(); - if token.modifiers.contains(SemanticTokenModifier::DEFINITION) { - mods.push("definition"); - } - if token.modifiers.contains(SemanticTokenModifier::READONLY) { - mods.push("readonly"); - } - if token.modifiers.contains(SemanticTokenModifier::ASYNC) { - mods.push("async"); - } - if token - .modifiers - .contains(SemanticTokenModifier::DOCUMENTATION) - { - mods.push("documentation"); - } format!(" [{}]", mods.join(", ")) }; diff --git a/crates/ty_ide/src/syntax_outline.rs b/crates/ty_ide/src/syntax_outline.rs index eedc83cd50..100e2bb834 100644 --- a/crates/ty_ide/src/syntax_outline.rs +++ b/crates/ty_ide/src/syntax_outline.rs @@ -24,6 +24,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::Db; use crate::comma_list::comma_lists; +use crate::injection::text_ranges; /// The block structure and string literals of one file. #[derive(Debug, Clone, PartialEq, Eq)] @@ -121,6 +122,14 @@ pub struct OutlineString { /// The escape sequences in the literal text. A backslash Python does not recognise is still /// reported, as two characters: Python keeps it verbatim, and that is worth seeing. pub escapes: Vec, + /// Where the literal's text is written, in source order: the runs whose text, joined, is what + /// the literal holds — what survives the indentation basedpython strips, without the + /// interpolations, escapes still as written. + /// + /// The body between the quotes when nothing is stripped and nothing is interpolated. An editor + /// that puts another language into a string it was not told about needs exactly this: the + /// characters that make up the string's text, and a place in the file for each. + pub text: Vec, } /// The outline of `file`. @@ -763,14 +772,67 @@ impl StringCollector<'_> { } } + let text = Self::text( + range, + flags, + stripped_indent.is_some(), + &interpolations, + self.source, + ); + self.strings.push(OutlineString { range, stripped_indent, interpolations, escapes, + text, }); } + /// See [`OutlineString::text`]. `stripped` says whether basedpython dedents this literal, + /// whose runs are then the ones the dedent leaves, by [`text_ranges`] — the rule the + /// language-injection fragments are reported by, so a string reads the same either way. + fn text( + range: TextRange, + flags: AnyStringFlags, + stripped: bool, + interpolations: &[TextRange], + source: &str, + ) -> Vec { + let start = range.start() + flags.opener_len(); + let closer = if flags.is_unclosed() { + TextSize::default() + } else { + flags.closer_len() + }; + let end = range.end().checked_sub(closer).unwrap_or(start).max(start); + let body = TextRange::new(start, end); + let runs = if stripped { + text_ranges(body, source).into_vec() + } else { + vec![body] + }; + + // An interpolation is an expression rather than text, so each run is cut around it. + let mut text = Vec::with_capacity(runs.len()); + for run in runs { + let mut from = run.start(); + for interpolation in interpolations { + if interpolation.end() <= from || interpolation.start() >= run.end() { + continue; + } + if interpolation.start() > from { + text.push(TextRange::new(from, interpolation.start())); + } + from = from.max(interpolation.end()); + } + if from < run.end() { + text.push(TextRange::new(from, run.end())); + } + } + text + } + /// The literal text of `elements`, and the top-level interpolations among them. A format /// spec's literal text is literal text too, and is where its escapes are. fn elements( @@ -1009,6 +1071,22 @@ mod tests { out } + /// Every string part's text runs, each in brackets. + fn texts(path: &str, source: &str) -> String { + let test = test(path, source); + let file = test.program_file(test.cursor.file).python_file(&test.db); + let outline = syntax_outline(&test.db, file); + let text = test.cursor.source.as_str(); + let mut out = String::new(); + for string in &outline.strings { + for run in &string.text { + write!(out, "[{}]", text[run.to_std_range()].replace('\n', "⏎")).unwrap(); + } + out.push('\n'); + } + out + } + /// The keywords that light up with the one under ``, or `none`. fn family(source: &str) -> String { let test = test("main.by", source); @@ -1224,6 +1302,44 @@ def f(): \"b\" b\"\"\"⏎ a⏎ \"\"\" f\"\"\"⏎ {x}⏎ \"\"\" strips 8 interpolation `{x}` +" + ); + } + + /// A string's text is what the program holds, so it is the body less what basedpython + /// strips and less the interpolations, which are expressions. Each literal of an + /// implicit concatenation is its own part with its own text, and is not dedented. + #[test] + fn the_text_is_what_survives_the_dedent_and_the_interpolations() { + assert_eq!( + texts( + "main.by", + "\ +def f(): + plain = \"a\\tb\" + page = \"\"\" +
+ +

hi

+
+ \"\"\" + query = f\"\"\" + SELECT {cols} + FROM t{n}x + \"\"\" + joined = \"\"\" + a + \"\"\" \"b\" + empty = \"\" +" + ), + "\ +[a\\tb] +[
⏎][⏎][

hi

⏎][
] +[SELECT ][⏎][FROM t][x] +[⏎ a⏎ ] +[b] + " ); } diff --git a/crates/ty_server/src/server/api/requests/syntax_outline.rs b/crates/ty_server/src/server/api/requests/syntax_outline.rs index b14817d3d3..e6cad07f09 100644 --- a/crates/ty_server/src/server/api/requests/syntax_outline.rs +++ b/crates/ty_server/src/server/api/requests/syntax_outline.rs @@ -155,6 +155,11 @@ pub(crate) struct StringPart { /// The escape sequences in the literal text. #[serde(default, skip_serializing_if = "Vec::is_empty")] escapes: Vec, + /// Where the literal's text is written: the runs whose text, joined, is what the literal + /// holds — what survives the stripped indentation, without the interpolations. Absent for an + /// empty literal. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + text: Vec, } pub(crate) struct SyntaxOutlineRequestHandler; @@ -304,6 +309,12 @@ impl Converter<'_> { .iter() .filter_map(|range| self.range(*range)) .collect(), + // all or nothing: the runs are joined to get the text, and one missing shifts the rest + text: string + .text + .iter() + .map(|range| self.range(*range)) + .collect::>()?, }) } } diff --git a/crates/ty_server/src/server/api/requests/transpile.rs b/crates/ty_server/src/server/api/requests/transpile.rs index 0f611f3981..a153f5934d 100644 --- a/crates/ty_server/src/server/api/requests/transpile.rs +++ b/crates/ty_server/src/server/api/requests/transpile.rs @@ -94,6 +94,11 @@ impl RequestHandler for TranspileRequestHandler { } impl BackgroundDocumentRequestHandler for TranspileRequestHandler { + // the file's text and the project around it, which the server has for a file no editor holds + // as much as for a buffer: a client asking about a file nobody opened — an agent through the + // editor's MCP server — gets the python it lowers to + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + fn document_uri(params: &TranspileParams) -> Cow<'_, Uri> { Cow::Borrowed(¶ms.text_document.uri) } diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 28fa8f8856..49f70d2847 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -412,6 +412,16 @@ deleted, each line of a kept suite dedented — so the caret stays where it was the text. the lines of a string literal that spans several lines are left as they are, because their indentation is part of the string +## semantic tokens + +besides the standard modifiers, a token can carry `functionScope`: the name is +bound in the scope of a function, a lambda or a comprehension — a parameter or a +local, as against a module global or a class attribute. it is the binding that +decides, not where the name is written: a global read inside a function is not +marked, a `nonlocal` name is, and an assignment expression in a comprehension at +the top of a module binds a global. an editor that colours each local by its name +("rainbow" highlighting) reads it off this modifier + ## keyword highlighting `textDocument/documentHighlight` on a keyword answers the keywords that belong @@ -765,10 +775,15 @@ the build wrote to serve features from: it is what an editor needs to type in an indentation-delimited language — which line opens a suite, where a compound statement ends, what a string literal's escapes and interpolations are, and how -much indentation a triple-quoted string has stripped. it also carries the name -each `def` and `class` declares and a `def`'s parameter names, and the elements of -every comma-separated list of two or more — what an editor moves an argument among, -or tracks a declaration being renamed by +much indentation a triple-quoted string has stripped. each string part also says +where its *text* is: the runs of the file whose text, joined, is what the literal +holds — one per line of a dedented string, past the indentation, and around the +interpolations of an f-string. that is what an editor needs to put another +language into a string it was not told about, with a place in the file for every +character of the fragment. the outline also carries the name each `def` and +`class` declares and a `def`'s parameter names, and the elements of every +comma-separated list of two or more — what an editor moves an argument among, or +tracks a declaration being renamed by `.by` test files are collected under the same names as python ones — `test_*.by` and `*_test.by` — because a `.by` transpiles to the `.py` of the same stem, and @@ -783,7 +798,7 @@ counted as one `\n`, written as sixteen lowercase hex digits. for a notebook cel that is the cell's own text. the request is then answered about that text and no other. when the server already holds it — the open buffer, or for `by/syntaxOutline`, `by/injections`, `by/superMembers`, -`by/documentSuperMembers`, `textDocument/semanticTokens/full` and +`by/documentSuperMembers`, `by/transpile`, `textDocument/semanticTokens/full` and `textDocument/documentSymbol` the file on disk of a document the client has not opened — it answers at once; otherwise the request waits for the `didOpen`, `didChange` or file write that brings the text, From be8472937c1e9a8456b6eac23005f956cac41e71 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:54:30 +1000 Subject: [PATCH 13/15] correct the functionScope tests' offsets and the new-test name snapshot, bound unused_name's search for clippy, and update the initialize and semantic token snapshots for the code actions and token modifier the combined branches add --- crates/ty_ide/src/semantic_tokens.rs | 20 +++++++--- crates/ty_ide/src/test_links.rs | 9 ++++- .../e2e__initialize__initialization.snap | 7 +++- ...ialize__initialization_with_workspace.snap | 7 +++- .../e2e__notebook__semantic_tokens.snap | 40 +++++++++---------- ...lient_not_supporting_multiline_tokens.snap | 4 +- ...en_client_supporting_multiline_tokens.snap | 4 +- 7 files changed, 56 insertions(+), 35 deletions(-) diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index 61426d3323..ef32d3be7b 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -6292,7 +6292,7 @@ def clamp(value: int) -> int: assert_eq!( test.function_scoped_names(&tokens), - ["value@46", "result@67", "value@80", "result@101"] + ["value@48", "result@72", "value@85", "result@110"] ); } @@ -6345,9 +6345,9 @@ def f(subject: object): [ "subject@59", "inner@124", - "subject@151", - "first@175", - "rest@183" + "subject@154", + "first@177", + "rest@185" ] ); } @@ -6368,6 +6368,8 @@ def outer(): nonlocal count global total count = total + total = count + count = total "#, ); @@ -6375,7 +6377,15 @@ def outer(): assert_eq!( test.function_scoped_names(&tokens), - ["x@27", "x@32", "count@71", "count@114", "count@143"] + [ + "x@28", + "x@34", + "count@67", + "count@111", + "count@146", + "count@176", + "count@190", + ] ); } diff --git a/crates/ty_ide/src/test_links.rs b/crates/ty_ide/src/test_links.rs index e8132613c2..7089a78e06 100644 --- a/crates/ty_ide/src/test_links.rs +++ b/crates/ty_ide/src/test_links.rs @@ -461,7 +461,9 @@ fn unused_name(db: &dyn Db, file: ProgramFile<'_>, name: String) -> String { if !bound.contains(name.as_str()) { return name; } - (2..) + // the module binds `bound.len()` names, so one of the `bound.len() + 1` suffixes + // from 2 is free + (2..=bound.len() + 2) .map(|suffix| format!("{name}_{suffix}")) .find(|candidate| !bound.contains(candidate.as_str())) .unwrap_or(name) @@ -811,7 +813,10 @@ class TestShapes: "from geometry import perimeter\n\ndef test_perimeter() -> None: ...\n\ndef test_perimeter_2() -> None: ...\n", ) .build(); - assert_snapshot!(test.subject_tests(), @""); + assert_snapshot!(test.subject_tests(), @" + + new test test_perimeter_3 added in /tests/test_geometry.py importing perimeter from geometry + "); } #[test] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap index 8b81e65593..72e6f8cd14 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap @@ -61,7 +61,9 @@ expression: initialization_result "refactor.extract.function", "refactor.rewrite.returnAnnotation", "refactor.rewrite.toDataClass", - "refactor.rewrite.fromDataClass" + "refactor.rewrite.fromDataClass", + "refactor.rewrite.splitList", + "refactor.rewrite.joinList" ], "resolveProvider": true }, @@ -113,7 +115,8 @@ expression: initialization_result "readonly", "async", "documentation", - "defaultLibrary" + "defaultLibrary", + "functionScope" ] }, "range": true, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap index 8b81e65593..72e6f8cd14 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap @@ -61,7 +61,9 @@ expression: initialization_result "refactor.extract.function", "refactor.rewrite.returnAnnotation", "refactor.rewrite.toDataClass", - "refactor.rewrite.fromDataClass" + "refactor.rewrite.fromDataClass", + "refactor.rewrite.splitList", + "refactor.rewrite.joinList" ], "resolveProvider": true }, @@ -113,7 +115,8 @@ expression: initialization_result "readonly", "async", "documentation", - "defaultLibrary" + "defaultLibrary", + "functionScope" ] }, "range": true, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__semantic_tokens.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__semantic_tokens.snap index 7173897828..93ed2f4c9b 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__semantic_tokens.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__semantic_tokens.snap @@ -53,7 +53,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 11, 4, 2, - 1, + 33, 0, 6, 3, @@ -63,12 +63,12 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 5, 4, 2, - 1, + 33, 0, 6, 5, 2, - 1, + 33, 0, 7, 5, @@ -83,7 +83,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 7, 5, 2, - 0, + 32, 0, 9, 8, @@ -93,7 +93,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 15, 4, 2, - 0, + 32, 0, 5, 7, @@ -103,7 +103,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 8, 4, 2, - 0, + 32, 0, 8, 1, @@ -113,7 +113,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 2, 4, 2, - 0, + 32, 0, 5, 1, @@ -123,7 +123,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 9, 5, 2, - 0, + 32, 0, 9, 6, @@ -133,7 +133,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 15, 4, 2, - 0, + 32, 0, 5, 7, @@ -143,7 +143,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 8, 4, 2, - 0, + 32, 0, 8, 2, @@ -153,7 +153,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 3, 4, 2, - 0, + 32, 0, 5, 2, @@ -163,12 +163,12 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 4, 8, 5, - 1, + 33, 0, 11, 4, 2, - 0, + 32, 0, 5, 4, @@ -178,17 +178,17 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 5, 4, 2, - 0, + 32, 1, 4, 6, 5, - 1, + 33, 0, 9, 4, 2, - 0, + 32, 0, 7, 4, @@ -198,7 +198,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 4, 6, 5, - 0, + 32, 0, 10, 3, @@ -208,12 +208,12 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 6, 8, 5, - 0, + 32, 1, 4, 6, 5, - 0, + 32, 0, 10, 3, @@ -228,7 +228,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 4, 4, 2, - 0 + 32 ] }, { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_not_supporting_multiline_tokens.snap b/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_not_supporting_multiline_tokens.snap index 695e4a3a7c..596706f8c3 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_not_supporting_multiline_tokens.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_not_supporting_multiline_tokens.snap @@ -13,7 +13,7 @@ expression: tokens 12, 6, 2, - 1, + 33, 0, 8, 3, @@ -23,7 +23,7 @@ expression: tokens 5, 6, 2, - 1, + 33, 0, 8, 3, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_supporting_multiline_tokens.snap b/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_supporting_multiline_tokens.snap index 4b2debb7ce..c59148eb59 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_supporting_multiline_tokens.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_supporting_multiline_tokens.snap @@ -13,7 +13,7 @@ expression: tokens 12, 6, 2, - 1, + 33, 0, 8, 3, @@ -23,7 +23,7 @@ expression: tokens 5, 6, 2, - 1, + 33, 0, 8, 3, From 82c48950dae99fdd8c8c40a7ef87cbacefff1cb0 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 09:34:25 +1000 Subject: [PATCH 14/15] run a reverse by/transpile on a thread with no database attached, as the forward one is, since its semantic model builds a database of its own and the request panicked with cannot change database mid-query --- .../src/server/api/requests/transpile.rs | 26 +++++++------ crates/ty_server/tests/e2e/transpile.rs | 38 +++++++++++++++++++ 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/crates/ty_server/src/server/api/requests/transpile.rs b/crates/ty_server/src/server/api/requests/transpile.rs index a153f5934d..edf845e259 100644 --- a/crates/ty_server/src/server/api/requests/transpile.rs +++ b/crates/ty_server/src/server/api/requests/transpile.rs @@ -117,20 +117,22 @@ impl BackgroundDocumentRequestHandler for TranspileRequestHandler { let document = ruff_db::source::source_text(db, file); let source = params.source.as_deref().unwrap_or(document.as_str()); - // Reverse always goes through the text form: the rewrite is syntactic, and there is no - // python project db to infer against. Forward uses the project db when it is the whole - // document that was asked about — that is the reason for answering here, since cross-module - // types resolve — and the single-file path for a fragment, which has no module to resolve - // against however it is transpiled. - // Both forward paths run on a thread that has attached no database — see - // [`super::detached_transpile`]. Every forward transpile builds a database of its - // own, and doing that under this handler's attached one panics; the reverse - // direction is purely syntactic and needs no database at all, so it stays here. + // Reverse always goes through the text form: there is no python project db to infer + // against. Forward uses the project db when it is the whole document that was asked + // about — that is the reason for answering here, since cross-module types resolve — and + // the single-file path for a fragment, which has no module to resolve against however it + // is transpiled. + // Every path runs on a thread that has attached no database — see + // [`super::detached_transpile`]. Each builds a database of its own, the reverse + // direction included (it reads a semantic model of the source for its literal, + // subscript and callable rewrites), and doing that under this handler's attached one + // panics. let owned = db.dyn_clone(); let response = if params.reverse { - match by_transforms::reverse_transpile(source, &config) { - Ok(out) => TranspileResponse::generated(out), - Err(error) => TranspileResponse::failed(error), + match transpile_detached(move || by_transforms::reverse_transpile(source, &config)) { + Ok(Ok(out)) => TranspileResponse::generated(out), + Ok(Err(error)) => TranspileResponse::failed(error), + Err(()) => TranspileResponse::failed("the transpiler panicked".to_owned()), } } else { let fragment = params.source.is_some(); diff --git a/crates/ty_server/tests/e2e/transpile.rs b/crates/ty_server/tests/e2e/transpile.rs index 2c5a28ff19..28400dcef6 100644 --- a/crates/ty_server/tests/e2e/transpile.rs +++ b/crates/ty_server/tests/e2e/transpile.rs @@ -123,6 +123,44 @@ fn transpiling_a_document_answers_rather_than_panicking() -> Result<()> { Ok(()) } +/// The reverse direction, python in and basedpython out, which builds an in-memory +/// database of its own to read a semantic model of the source — so it needs the +/// detached thread as much as the forward one does. +#[test] +fn reverse_transpiling_a_document_answers_rather_than_panicking() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let main = SystemPath::new("src/legacy.py"); + let content = "def f(x: int | None) -> int:\n return x if x is not None else 0\n"; + + let mut server = TestServerBuilder::new()? + .with_workspace(workspace_root, None)? + .with_file(main, content)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(main, content, 1); + let uri = server.file_uri(main); + + let response = server + .send_request_await::(TranspileParams { + reverse: true, + ..params(&uri, None) + }) + .expect("the server answered nothing at all"); + + assert!( + response.error.is_none(), + "reverse transpile failed: {:?}", + response.error + ); + let generated = response.source.expect("no generated source"); + assert!( + generated.contains("return x ?? 0"), + "unexpected output:\n{generated}" + ); + Ok(()) +} + #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase")] struct TranspileForBuildParams { From 18fa185ecc6a339aa4a074707bb32dd601cd9679 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:15:39 +1000 Subject: [PATCH 15/15] count a document's usages and implementations in one pass over the workspace instead of one per declaration, run the counts one at a time on a search lane of their own so they never fill the workers other requests wait on, stop a running request and its parallel work when the client cancels it, and read the vendored root only when a stub is mapped, since every file a search visits read it under the environment lock --- crates/ty_ide/src/find_references.rs | 42 ++- crates/ty_ide/src/goto_implementation.rs | 74 +++++ crates/ty_ide/src/references.rs | 208 +++++++++++--- crates/ty_ide/src/stub_mapping.rs | 15 +- crates/ty_ide/src/usage_counts.rs | 77 +++--- crates/ty_project/src/parallel.rs | 257 +++++++++++++++++- crates/ty_server/src/server/api.rs | 15 +- .../api/requests/document_usage_counts.rs | 11 +- crates/ty_server/src/server/schedule.rs | 39 +++ crates/ty_server/src/server/schedule/task.rs | 6 + .../src/server/schedule/thread/pool.rs | 21 +- crates/ty_server/src/session/request_queue.rs | 22 +- crates/ty_server/tests/e2e/usage_counts.rs | 63 ++++- docs/basedpython/features/editor.md | 21 +- 14 files changed, 769 insertions(+), 102 deletions(-) diff --git a/crates/ty_ide/src/find_references.rs b/crates/ty_ide/src/find_references.rs index ef3ba082de..4eec244660 100644 --- a/crates/ty_ide/src/find_references.rs +++ b/crates/ty_ide/src/find_references.rs @@ -1,5 +1,5 @@ use crate::goto::find_goto_target; -use crate::references::{FixtureReferenceTarget, ReferencesMode, references}; +use crate::references::{FixtureReferenceTarget, ReferencesMode, references, references_to_each}; use crate::{Db, ReferenceTarget}; use ruff_db::parsed::ParsedModuleRef; use ruff_python_ast::AnyNodeRef; @@ -38,6 +38,46 @@ pub fn find_references( references(db, file, &goto_target, mode) } +/// What [`find_references`] answers at each of `offsets` in `file`, found with one pass over the +/// project's files for all of them rather than one pass each (see [`references_to_each`]). +pub(crate) fn find_references_to_each( + db: &dyn Db, + file: ProgramFile<'_>, + offsets: &[TextSize], + include_declaration: bool, +) -> Vec>> { + let parsed = ruff_db::parsed::parsed_module(db, file.python_file(db)); + let module = parsed.load(db); + let model = SemanticModel::new(db, file); + + let mode = if include_declaration { + ReferencesMode::References + } else { + ReferencesMode::ReferencesSkipDeclaration + }; + + // an explicit fixture name is searched for on its own, as `find_references` does + let mut found: Vec>> = offsets.iter().map(|_| None).collect(); + let mut goto_targets = Vec::with_capacity(offsets.len()); + for (index, &offset) in offsets.iter().enumerate() { + if let Some(target) = explicit_fixture_name_at_offset(&model, &module, offset) { + found[index] = target.references(db, file, mode); + goto_targets.push(None); + } else { + goto_targets.push(find_goto_target(&model, &module, offset)); + } + } + for (index, references) in references_to_each(db, file, &goto_targets, mode) + .into_iter() + .enumerate() + { + if goto_targets[index].is_some() { + found[index] = references; + } + } + found +} + /// Returns a target for the explicit fixture name at `offset`. /// Quotes, prefixes, and token boundaries select the name's contents. /// diff --git a/crates/ty_ide/src/goto_implementation.rs b/crates/ty_ide/src/goto_implementation.rs index 460f135c81..7fae8b400c 100644 --- a/crates/ty_ide/src/goto_implementation.rs +++ b/crates/ty_ide/src/goto_implementation.rs @@ -116,6 +116,80 @@ pub fn goto_implementation( }) } +/// What [`goto_implementation`] answers at each of `offsets` in `file`, found with one pass over +/// the project's files for all of them rather than one pass each. +/// +/// Each file is scanned once, for every target in turn; the same scan of the same files as +/// [`goto_implementation`] makes for each target, so each finds the same implementations, in the +/// same order. +pub(crate) fn implementations_at_each( + db: &dyn Db, + file: ProgramFile<'_>, + offsets: &[TextSize], +) -> Vec> { + let module = parsed_module(db, file.python_file(db)).load(db); + let model = SemanticModel::new(db, file); + let finders: Vec<_> = offsets + .iter() + .map(|&offset| { + let goto_target = find_goto_target(&model, &module, offset)?; + prepare_implementations_finder_for_goto_target(&model, &goto_target) + }) + .collect(); + if finders.iter().all(Option::is_none) { + return finders.iter().map(|_| None).collect(); + } + let source_file = file.file(db); + let program = file.program(db); + + let mut candidate_files: Vec = db + .project() + .files(db) + .iter() + .filter(|candidate| *candidate != source_file) + .collect(); + candidate_files.push(source_file); + + // per file, each finder's implementations in it, by index into `finders` + let batches = candidate_files + .into_par_iter() + .map_with_db(db, |db, file| { + let file = ProgramFile::new(db, file, program); + finders + .iter() + .enumerate() + .filter_map(|(index, finder)| { + let definitions = finder.as_ref()?.implementations_for_file(db, file); + let targets = definitions_to_implementation_targets(db, definitions); + (!targets.is_empty()).then_some((index, targets)) + }) + .collect::>() + }) + .collect::>(); + + let mut implementation_targets: Vec>> = finders + .into_iter() + .map(|finder| { + finder.map(|finder| { + definitions_to_implementation_targets(db, finder.into_initial_definitions()) + }) + }) + .collect(); + for (index, targets) in batches.into_iter().flatten() { + if let Some(found) = &mut implementation_targets[index] { + found.extend(targets); + } + } + implementation_targets + .into_iter() + .map(|targets| { + targets + .filter(|targets| !targets.is_empty()) + .map(|targets| targets.into_iter().collect()) + }) + .collect() +} + /// Select and prepare the appropriate `ImplementationsFinder` for `goto_target`. fn prepare_implementations_finder_for_goto_target<'db>( model: &SemanticModel<'db>, diff --git a/crates/ty_ide/src/references.rs b/crates/ty_ide/src/references.rs index adb401d344..cb65e1d5e8 100644 --- a/crates/ty_ide/src/references.rs +++ b/crates/ty_ide/src/references.rs @@ -14,7 +14,7 @@ use crate::goto::{Definitions, GotoTarget}; use crate::{Db, ReferenceKind, ReferenceTarget}; use rayon::prelude::*; use ruff_db::files::File; -use ruff_db::parsed::parsed_module; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; use ruff_python_ast::find_node::{CoveringNode, covering_node}; use ruff_python_ast::token::Tokens; use ruff_python_ast::{ @@ -97,43 +97,177 @@ pub(crate) fn references( mode: ReferencesMode, ) -> Option> { let model = SemanticModel::new(db, file); - let target_text = goto_target.to_string()?.into_owned(); - - let target_definitions = goto_target - .definitions(&model, mode.to_import_alias_resolution())? - .goto_declaration(&model, goto_target)?; - let import_alias_resolution = mode.to_import_alias_resolution(); - // An identifier can have both ordinary Python references and pytest fixture references. - // Keep its Python definitions alongside any fixture roots used for the same search. - let fixture_target = matches!( - mode, - ReferencesMode::References | ReferencesMode::ReferencesSkipDeclaration - ) - .then(|| FixtureReferenceTarget::from_goto_target(&model, goto_target, &target_text)) - .flatten(); - let fixture_resolution = fixture_target.map(|target| target.resolution); - - let is_externally_visible_symbol = - has_any_external_visible_definitions(db, &target_definitions); - - let is_parameter = parameter_owner_is_externally_visible(db, &target_definitions); - - let search = LocalReferenceSearch { - target_text, - target_definitions, - import_alias_resolution, - fixture_resolution, - }; + let prepared = PreparedSearch::new(db, &model, goto_target, mode)?; references_for_search( db, file, - &search, + &prepared.search, mode, - is_externally_visible_symbol, - is_parameter, + prepared.is_externally_visible_symbol, + prepared.is_parameter, ) } +/// What [`references`] answers for each of `goto_targets`, all of them in `file`, with the +/// searches across the project done in one pass over its files rather than one pass per target. +/// +/// A file is read, and searched for each target whose name it contains, once; [`references`] +/// reads every file once per target. What each target is found to be referenced by is the same: +/// the same search is made for it in the same files. A target that is a pytest fixture is +/// searched for on its own, since its search reaches files outside the project. +pub(crate) fn references_to_each( + db: &dyn Db, + file: ProgramFile<'_>, + goto_targets: &[Option], + mode: ReferencesMode, +) -> Vec>> { + let model = SemanticModel::new(db, file); + let prepared: Vec<_> = goto_targets + .iter() + .map(|goto_target| { + goto_target + .as_ref() + .and_then(|goto_target| PreparedSearch::new(db, &model, goto_target, mode)) + }) + .collect(); + let search_across_files = matches!( + mode, + ReferencesMode::References + | ReferencesMode::ReferencesSkipDeclaration + | ReferencesMode::RenameMultiFile + ); + + let mut found: Vec>> = vec![None; prepared.len()]; + // the targets searched for across the project together, by index into `prepared` + let mut together = Vec::new(); + for (index, prepared) in prepared.iter().enumerate() { + let Some(prepared) = prepared else { + continue; + }; + if search_across_files + && prepared.is_externally_visible_symbol + && prepared.search.fixture_resolution.is_none() + { + together.push(index); + } else { + found[index] = references_for_search( + db, + file, + &prepared.search, + mode, + prepared.is_externally_visible_symbol, + prepared.is_parameter, + ); + } + } + if together.is_empty() { + return found; + } + let searches: Vec<&LocalReferenceSearch> = together + .iter() + .filter_map(|&index| prepared[index].as_ref()) + .map(|prepared| &prepared.search) + .collect(); + + // the targets' own file first, as `references_for_search` has it + let mut references: Vec> = searches + .iter() + .map(|search| references_for_file(db, file, search, mode)) + .collect(); + + let source_file = file.file(db); + let program = file.program(db); + let files: Vec<_> = db + .project() + .files(db) + .iter() + .filter(|other| *other != source_file) + .collect(); + let minimum_job_len = minimum_parallel_job_len(files.len(), MAX_MIN_FILES_PER_PARALLEL_JOB); + let in_other_files = files + .into_par_iter() + .with_min_len(minimum_job_len) + .map_with_db(db, |db, other_file| { + let source = ruff_db::source::source_text(db, other_file); + let named: Vec<_> = (0..searches.len()) + .filter(|&search| contains_identifier(&source, &searches[search].target_text)) + .collect(); + if named.is_empty() { + return Vec::new(); + } + let other_file = ProgramFile::new(db, other_file, program); + let parsed = parsed_module(db, other_file.python_file(db)); + let module = parsed.load(db); + let model = SemanticModel::new(db, other_file); + named + .into_iter() + .map(|search| { + ( + search, + references_in_module(&model, &module, searches[search], mode), + ) + }) + .collect() + }) + .collect::>(); + for (search, in_file) in in_other_files.into_iter().flatten() { + references[search].extend(in_file); + } + + for (index, references) in together.into_iter().zip(references) { + found[index] = (!references.is_empty()).then_some(references); + } + found +} + +/// The search [`references`] makes for a target, prepared from what is at the target. +struct PreparedSearch<'db> { + search: LocalReferenceSearch<'db>, + is_externally_visible_symbol: bool, + is_parameter: bool, +} + +impl<'db> PreparedSearch<'db> { + fn new( + db: &dyn Db, + model: &SemanticModel<'db>, + goto_target: &GotoTarget, + mode: ReferencesMode, + ) -> Option { + let target_text = goto_target.to_string()?.into_owned(); + + let target_definitions = goto_target + .definitions(model, mode.to_import_alias_resolution())? + .goto_declaration(model, goto_target)?; + let import_alias_resolution = mode.to_import_alias_resolution(); + // An identifier can have both ordinary Python references and pytest fixture references. + // Keep its Python definitions alongside any fixture roots used for the same search. + let fixture_target = matches!( + mode, + ReferencesMode::References | ReferencesMode::ReferencesSkipDeclaration + ) + .then(|| FixtureReferenceTarget::from_goto_target(model, goto_target, &target_text)) + .flatten(); + let fixture_resolution = fixture_target.map(|target| target.resolution); + + let is_externally_visible_symbol = + has_any_external_visible_definitions(db, &target_definitions); + + let is_parameter = parameter_owner_is_externally_visible(db, &target_definitions); + + Some(Self { + search: LocalReferenceSearch { + target_text, + target_definitions, + import_alias_resolution, + fixture_resolution, + }, + is_externally_visible_symbol, + is_parameter, + }) + } +} + fn references_for_search( db: &dyn Db, file: ProgramFile<'_>, @@ -293,10 +427,20 @@ fn references_for_file( let parsed = parsed_module(db, file.python_file(db)); let module = parsed.load(db); let model = SemanticModel::new(db, file); + references_in_module(&model, &module, search, mode) +} + +/// [`references_for_file`] with the file's module and model already at hand. +fn references_in_module( + model: &SemanticModel<'_>, + module: &ParsedModuleRef, + search: &LocalReferenceSearch<'_>, + mode: ReferencesMode, +) -> Vec { let mut references = Vec::new(); let mut finder = LocalReferencesFinder { - model: &model, + model, search, references: &mut references, mode, diff --git a/crates/ty_ide/src/stub_mapping.rs b/crates/ty_ide/src/stub_mapping.rs index 861aa2028f..daeff6e74d 100644 --- a/crates/ty_ide/src/stub_mapping.rs +++ b/crates/ty_ide/src/stub_mapping.rs @@ -1,3 +1,5 @@ +use std::cell::OnceCell; + use itertools::Either; use ruff_db::system::SystemPathBuf; use ty_python_semantic::{ResolvedDefinition, map_stub_definition}; @@ -12,15 +14,17 @@ use crate::cached_vendored_root; /// docstrings for functions that resolve to stubs. pub(crate) struct StubMapper<'db> { db: &'db dyn ty_python_semantic::Db, - cached_vendored_root: Option, + /// Worked out when a definition is first mapped, not when the mapper is made: it reads the + /// environment, under a lock every thread shares, and a search of the workspace makes a mapper + /// for every file it looks at, most of which have nothing to map. + cached_vendored_root: OnceCell>, } impl<'db> StubMapper<'db> { pub(crate) fn new(db: &'db dyn ty_python_semantic::Db) -> Self { - let cached_vendored_root = cached_vendored_root(db); Self { db, - cached_vendored_root, + cached_vendored_root: OnceCell::new(), } } @@ -42,7 +46,10 @@ impl<'db> StubMapper<'db> { &self, def: &ResolvedDefinition<'db>, ) -> Option>> { - map_stub_definition(self.db, def, self.cached_vendored_root.as_deref()) + let cached_vendored_root = self + .cached_vendored_root + .get_or_init(|| cached_vendored_root(self.db)); + map_stub_definition(self.db, def, cached_vendored_root.as_deref()) } /// Map multiple `ResolvedDefinitions`, applying stub-to-source mapping to each. diff --git a/crates/ty_ide/src/usage_counts.rs b/crates/ty_ide/src/usage_counts.rs index 567a0aa52d..dd16964bcd 100644 --- a/crates/ty_ide/src/usage_counts.rs +++ b/crates/ty_ide/src/usage_counts.rs @@ -8,6 +8,12 @@ //! subclasses and overrides. so the count over a declaration is the number of //! places its click goes to //! +//! each of those is a search of every file of the project, which asked of each +//! declaration in turn reads every file once per declaration. the counts are +//! found instead with one pass over the files for all the declarations of the +//! document: each file is read once, and searched for each declaration whose +//! name it holds, with the same search, so each count is the same +//! //! the declarations counted are the document's outline ([`crate::document_symbols()`]) //! at the depth a reader scans a module at: its classes and functions, and the //! methods and properties of those classes. a dunder is left out, being called @@ -15,20 +21,20 @@ //! names a class it does not declare. a nested function or a class inside a //! class is not counted -use rayon::prelude::*; use ruff_db::parsed::parsed_module; use ruff_python_ast::AnyNodeRef; use ruff_python_ast::find_node::covering_node; use ruff_python_ast::helpers::is_dunder; use ruff_text_size::TextRange; -use ty_project::parallel::ParallelIteratorExt; use ty_python_core::ProgramFile; use ty_python_semantic::types::ide_support::{class_member_definitions, overridden_members}; +use crate::find_references::find_references_to_each; +use crate::goto_implementation::implementations_at_each; use crate::symbols::symbols_for_file; -use crate::{Db, SymbolKind, find_references, goto_implementation}; +use crate::{Db, SymbolKind}; -/// which counts to work out: each is a search of the workspace per declaration +/// which counts to work out: each is a search of the workspace #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct UsageCountKinds { /// how many places use each declaration @@ -79,36 +85,41 @@ pub fn document_usage_counts( return declarations; } let source_file = file.file(db); - let program = file.program(db); + let mut declarations = declarations; - let mut declarations: Vec<_> = declarations - .into_par_iter() - .map_with_db(db, |db, mut declaration| { - let file = ProgramFile::new(db, source_file, program); - let offset = declaration.name_range.start(); - if kinds.references { - declaration.references = Some( - find_references(db, file, offset, false) - .map_or(0, |references| references.len()), - ); - } - if kinds.implementations && declaration.kind != DeclarationKind::Function { - declaration.implementations = - Some(goto_implementation(db, file, offset).map_or(0, |found| { - found - .value - .iter() - .filter(|target| { - // the declaration is its own first implementation - target.file() != source_file - || target.focus_range() != declaration.name_range - }) - .count() - })); - } - declaration - }) - .collect::>(); + if kinds.references { + let offsets: Vec<_> = declarations + .iter() + .map(|declaration| declaration.name_range.start()) + .collect(); + let found = find_references_to_each(db, file, &offsets, false); + for (declaration, references) in declarations.iter_mut().zip(found) { + declaration.references = Some(references.map_or(0, |references| references.len())); + } + } + if kinds.implementations { + // a function is neither subclassed nor overridden + let (offsets, counted): (Vec<_>, Vec<_>) = declarations + .iter() + .enumerate() + .filter(|(_, declaration)| declaration.kind != DeclarationKind::Function) + .map(|(index, declaration)| (declaration.name_range.start(), index)) + .unzip(); + let found = implementations_at_each(db, file, &offsets); + for (index, implementations) in counted.into_iter().zip(found) { + let declaration = &mut declarations[index]; + declaration.implementations = Some(implementations.map_or(0, |found| { + found + .iter() + .filter(|target| { + // the declaration is its own first implementation + target.file() != source_file + || target.focus_range() != declaration.name_range + }) + .count() + })); + } + } // abstract or not only changes how a method's overrides are named, so it is // only worked out for one that has some diff --git a/crates/ty_project/src/parallel.rs b/crates/ty_project/src/parallel.rs index 4617b1c035..dcd8a4ed74 100644 --- a/crates/ty_project/src/parallel.rs +++ b/crates/ty_project/src/parallel.rs @@ -1,5 +1,23 @@ //! Helpers for parallel operations that need an independent Salsa database on each Rayon job. +//! +//! # Cancelling one operation +//! +//! Salsa cancels every computation when the database is written to, and nothing else stops one: a +//! request its client has given up on runs to the end. A [`CancellationScope`] stops the +//! computations of one operation only. The operation [enters](CancellationScope::enter) it on the +//! thread it starts on and [registers](CancellationScope::register) its database there; each Rayon +//! job the helpers here hand its work to registers the database clone it is given, and enters the +//! scope too, so that a parallel iterator started inside a job belongs to it as well. +//! [`CancellationScope::cancel`] cancels each of those databases, and every one registered later, +//! and each unwinds with [`salsa::Cancelled::Local`] at its next query. +//! +//! A computation another thread is waiting on is not lost to that thread: Salsa hands it back, +//! and the waiting thread computes it itself. +use std::cell::RefCell; +use std::sync::Arc; + +use parking_lot::Mutex; use rayon::iter::ParallelIterator; use crate::{Db, ProjectDatabase}; @@ -24,6 +42,8 @@ pub trait ParallelIteratorExt: ParallelIterator + Sized { /// indexed iterator can use [`rayon::iter::IndexedParallelIterator::with_min_len`] to reduce /// task and database-cloning overhead. /// + /// The jobs belong to the [`CancellationScope`] the calling thread is in, if any. + /// /// # Warning /// /// This method uses [`salsa::attach_allow_change`] and must never be called from within a Salsa @@ -37,9 +57,15 @@ pub trait ParallelIteratorExt: ParallelIterator + Sized { where Output: Send, { - self.map_with(ParallelDb(Db::dyn_clone(db)), move |db, item| { - salsa::attach_allow_change(&*db.0, || map(&*db.0, item)) - }) + self.map_with( + ParallelDb::new(Db::dyn_clone(db), CancellationScope::current()), + move |db, item| { + let ParallelDb { db, scope } = &*db; + run_in(scope.as_ref(), &**db, || { + salsa::attach_allow_change(&**db, || map(&**db, item)) + }) + }, + ) } /// Runs an operation in parallel with a cloned [`ProjectDatabase`] for each Rayon job. @@ -48,6 +74,8 @@ pub trait ParallelIteratorExt: ParallelIterator + Sized { /// indexed iterator can use [`rayon::iter::IndexedParallelIterator::with_min_len`] to reduce /// task and database-cloning overhead. /// + /// The jobs belong to the [`CancellationScope`] the calling thread is in, if any. + /// /// # Warning /// /// This method uses [`salsa::attach_allow_change`] and must never be called from within a Salsa @@ -58,19 +86,230 @@ pub trait ParallelIteratorExt: ParallelIterator + Sized { db: &ProjectDatabase, op: impl Fn(&ProjectDatabase, Self::Item) + Send + Sync, ) { - self.for_each_with(db.clone(), move |db, item| { - salsa::attach_allow_change(db, || op(db, item)); - }); + self.for_each_with( + ParallelProjectDb::new(db.clone(), CancellationScope::current()), + move |db, item| { + let ParallelProjectDb { db, scope } = &*db; + run_in(scope.as_ref(), db, || { + salsa::attach_allow_change(db, || op(db, item)); + }); + }, + ); } } impl ParallelIteratorExt for Iter where Iter: ParallelIterator {} -/// An owned Salsa snapshot that Rayon can clone whenever it splits a parallel iterator. -struct ParallelDb(Box); +/// An owned Salsa snapshot that Rayon can clone whenever it splits a parallel iterator, each +/// clone registered with the scope the iterator was started in. +struct ParallelDb { + db: Box, + scope: Option, +} + +impl ParallelDb { + fn new(db: Box, scope: Option) -> Self { + if let Some(scope) = &scope { + scope.register(&*db); + } + Self { db, scope } + } +} impl Clone for ParallelDb { fn clone(&self) -> Self { - Self(Db::dyn_clone(&*self.0)) + Self::new(Db::dyn_clone(&*self.db), self.scope.clone()) + } +} + +/// [`ParallelDb`] for a [`ProjectDatabase`]. +struct ParallelProjectDb { + db: ProjectDatabase, + scope: Option, +} + +impl ParallelProjectDb { + fn new(db: ProjectDatabase, scope: Option) -> Self { + if let Some(scope) = &scope { + scope.register(&db); + } + Self { db, scope } + } +} + +impl Clone for ParallelProjectDb { + fn clone(&self) -> Self { + Self::new(self.db.clone(), self.scope.clone()) + } +} + +/// Runs `op`, one item of a Rayon job on `db`, inside `scope`. +/// +/// Salsa clears a database's cancellation when the database is detached from the thread, which +/// each item does as it ends, so a cancellation that lands between two items is applied again +/// before the next one starts. +fn run_in( + scope: Option<&CancellationScope>, + db: &dyn salsa::Database, + op: impl FnOnce() -> R, +) -> R { + match scope { + Some(scope) => scope.enter(|| { + if scope.is_cancelled() { + db.cancellation_token().cancel(); + } + op() + }), + None => op(), + } +} + +/// The Salsa computations of one operation, wherever they run, to be cancelled together. See the +/// [module documentation](self). +#[derive(Clone, Default)] +pub struct CancellationScope(Arc>); + +#[derive(Default)] +struct ScopeState { + cancelled: bool, + /// The cancellation of each database handle the operation computes on. Cancelling one that + /// has since been dropped does nothing. + databases: Vec, +} + +thread_local! { + static CURRENT_SCOPE: RefCell> = const { RefCell::new(None) }; +} + +impl CancellationScope { + #[cfg(test)] + fn new() -> Self { + Self::default() + } + + /// Cancels every database registered, and each one registered from now on. + pub fn cancel(&self) { + let mut state = self.0.lock(); + state.cancelled = true; + for database in state.databases.drain(..) { + database.cancel(); + } + } + + pub fn is_cancelled(&self) -> bool { + self.0.lock().cancelled + } + + /// Makes `db`'s computations part of this scope. + pub fn register(&self, db: &dyn salsa::Database) { + let token = db.cancellation_token(); + let mut state = self.0.lock(); + if state.cancelled { + token.cancel(); + } else { + state.databases.push(token); + } + } + + /// Runs `op` with this as the thread's scope, so that the parallel iterators `op` starts + /// through [`ParallelIteratorExt`] belong to it. + pub fn enter(&self, op: impl FnOnce() -> R) -> R { + struct Restore(Option); + + impl Drop for Restore { + fn drop(&mut self) { + let previous = self.0.take(); + CURRENT_SCOPE.with(|current| *current.borrow_mut() = previous); + } + } + + let previous = CURRENT_SCOPE.with(|current| current.replace(Some(self.clone()))); + let _restore = Restore(previous); + op() + } + + /// The scope the calling thread is in. + fn current() -> Option { + CURRENT_SCOPE.with(|current| current.borrow().clone()) + } +} + +// a panic cannot leave the state half-changed: each lock sets a flag, or pushes or drains the +// list, and none of them panics while holding it +impl std::panic::UnwindSafe for CancellationScope {} +impl std::panic::RefUnwindSafe for CancellationScope {} + +impl std::fmt::Debug for CancellationScope { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CancellationScope") + .field("cancelled", &self.is_cancelled()) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use std::panic::AssertUnwindSafe; + + use rayon::prelude::*; + use ruff_db::files::{File, system_path_to_file}; + use ruff_db::source::source_text; + use ruff_db::system::{DbWithWritableSystem as _, SystemPathBuf}; + + use super::{CancellationScope, ParallelIteratorExt}; + use crate::ProjectMetadata; + use crate::db::testing::TestDb; + + fn project(files: usize) -> (TestDb, Vec) { + let root = SystemPathBuf::from("/project"); + let mut db = TestDb::new(ProjectMetadata::new("test", root.clone())); + let paths: Vec<_> = (0..files) + .map(|index| root.join(format!("module{index}.py"))) + .collect(); + db.write_files(paths.iter().map(|path| (path.clone(), "x: int = 1\n"))) + .unwrap(); + let files = paths + .iter() + .map(|path| system_path_to_file(&db, path).unwrap()) + .collect(); + (db, files) + } + + fn total_length(db: &TestDb, files: &[File]) -> usize { + files + .to_vec() + .into_par_iter() + .map_with_db(db, |db, file| source_text(db, file).len()) + .sum() + } + + /// The jobs a parallel iterator started inside a cancelled scope unwind with a local + /// cancellation instead of computing, and the same work outside it is done. + #[test] + fn a_cancelled_scope_stops_its_parallel_jobs() { + let (db, files) = project(16); + let scope = CancellationScope::new(); + scope.cancel(); + + let cancelled = salsa::Cancelled::catch(AssertUnwindSafe(|| { + scope.enter(|| total_length(&db, &files)) + })); + assert!( + matches!(cancelled, Err(salsa::Cancelled::Local)), + "{cancelled:?}" + ); + + assert_eq!(total_length(&db, &files), 16 * "x: int = 1\n".len()); + } + + /// A scope that is never cancelled changes nothing. + #[test] + fn a_scope_left_alone_computes() { + let (db, files) = project(16); + let scope = CancellationScope::new(); + assert_eq!( + scope.enter(|| total_length(&db, &files)), + 16 * "x: int = 1\n".len() + ); } } diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index a3e3b1e637..46dedd98c8 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -255,7 +255,7 @@ pub(super) fn request(req: server::Request) -> Task { requests::DocumentUsageCountsRequestHandler::METHOD => background_document_request_task::< requests::DocumentUsageCountsRequestHandler, >( - req, BackgroundSchedule::Worker + req, BackgroundSchedule::Search ), // both are asked by someone waiting on a shortcut they pressed requests::OverridableMembersRequestHandler::METHOD => { @@ -456,9 +456,12 @@ where return; } + // the parallel work the request starts stops when the client cancels it if let Err(error) = ruff_db::panic::catch_unwind(|| { let snapshot = snapshot; - R::handle_request(&id, snapshot.0, client, params); + cancellation_token + .scope() + .enter(|| R::handle_request(&id, snapshot.0, client, params)); }) { panic_response::(&id, client, &error, retry, log_guidance); } @@ -543,8 +546,12 @@ where return; } + // a cancellation from the client stops the request where it is, whether on this + // thread or in the parallel work it starts + let scope = cancellation_token.scope(); + scope.register(&db); if let Err(error) = ruff_db::panic::catch_unwind(|| { - salsa::attach(&db, || { + scope.enter(|| salsa::attach(&db, || { if let Some(asked) = asked && !holds_text(&db, &document, asked) { @@ -556,7 +563,7 @@ where return; } R::handle_request(&id, &db, document, client, params); - }); + })); }) { panic_response::(&id, client, &error, retry, log_guidance); } diff --git a/crates/ty_server/src/server/api/requests/document_usage_counts.rs b/crates/ty_server/src/server/api/requests/document_usage_counts.rs index 94cc6ec606..30feeaa6b6 100644 --- a/crates/ty_server/src/server/api/requests/document_usage_counts.rs +++ b/crates/ty_server/src/server/api/requests/document_usage_counts.rs @@ -13,9 +13,14 @@ //! not a `textDocument/codeLens`, because what a client wants of these is not //! what it wants of a lens: it draws them as its own kind of hint, under a //! setting of its own, and asks again whenever anything in the workspace changes, -//! where a lens is asked for when its document changes. the counts are a search -//! of the workspace for each declaration, so a client asks only for the ones it -//! draws +//! where a lens is asked for when its document changes. each count is a search +//! of the workspace, so a client asks only for the ones it draws, and for all of +//! them in one request +//! +//! the searches run on the search lane (`BackgroundSchedule::Search`), one at a +//! time on a thread of their own, so that a client asking on every pass never +//! fills the workers the requests someone waits on run on, and a count the +//! client cancels stops where it is //! //! one document at a time, whole. `textHash` says which revision of this //! document the answer is about, but the counts also read every other file of the diff --git a/crates/ty_server/src/server/schedule.rs b/crates/ty_server/src/server/schedule.rs index ddfc751604..50a9fb866e 100644 --- a/crates/ty_server/src/server/schedule.rs +++ b/crates/ty_server/src/server/schedule.rs @@ -33,14 +33,20 @@ pub(crate) fn spawn_main_loop( pub(crate) struct Scheduler { fmt_pool: thread::Pool, background_pool: thread::Pool, + search_lane: thread::Pool, } impl Scheduler { pub(super) fn new(worker_threads: NonZeroUsize) -> Self { const FMT_THREADS: usize = 1; + const SEARCH_THREADS: usize = 1; Self { fmt_pool: thread::Pool::new(NonZeroUsize::try_from(FMT_THREADS).unwrap()), background_pool: thread::Pool::new(worker_threads), + search_lane: thread::Pool::unbounded( + NonZeroUsize::try_from(SEARCH_THREADS).unwrap(), + "ty:search", + ), } } @@ -67,6 +73,9 @@ impl Scheduler { BackgroundSchedule::Fmt => { self.fmt_pool.spawn(ThreadPriority::LatencySensitive, task); } + BackgroundSchedule::Search => { + self.search_lane.spawn(ThreadPriority::Worker, task); + } } } } @@ -124,4 +133,34 @@ mod tests { } dispatching.join().expect("the dispatching thread to end"); } + + /// A job on the search lane runs for as long as its search of the workspace takes. Handing + /// the lane more work meanwhile must not wait for it, or the main loop, which hands the work + /// out, would stop too: answering nothing else, and unable to take the cancellation that + /// would end the long job. + #[test] + fn handing_a_busy_search_lane_more_work_never_waits() { + const JOBS: usize = 64; + const PATIENCE: Duration = Duration::from_secs(30); + + let pool = Pool::unbounded(NonZeroUsize::new(1).unwrap(), "test:search"); + let (release, released) = crossbeam::channel::bounded::<()>(0); + pool.spawn(ThreadPriority::Worker, move || { + released.recv().ok(); + }); + + let (dispatched, all_dispatched) = crossbeam::channel::bounded(1); + let dispatching = std::thread::spawn(move || { + for _ in 0..JOBS { + pool.spawn(ThreadPriority::Worker, || {}); + } + dispatched.send(()).ok(); + pool + }); + + let outcome = all_dispatched.recv_timeout(PATIENCE); + release.send(()).ok(); + drop(dispatching.join()); + outcome.expect("every job to have been handed to the lane while its one thread was busy"); + } } diff --git a/crates/ty_server/src/server/schedule/task.rs b/crates/ty_server/src/server/schedule/task.rs index 220e1f222a..708ab6e77e 100644 --- a/crates/ty_server/src/server/schedule/task.rs +++ b/crates/ty_server/src/server/schedule/task.rs @@ -24,6 +24,12 @@ pub(in crate::server) enum BackgroundSchedule { /// The default for any request that isn't in the critical path of the user typing. #[default] Worker, + /// The task searches the whole workspace for something an editor shows without being asked, + /// such as the usage counts drawn above declarations, and may take as long as the workspace is + /// large. Such tasks run one at a time on a thread of their own, queued for as long as it + /// takes, so that however many an editor asks for, every worker stays free for the requests + /// someone is waiting on. A search runs its own work in parallel. + Search, } /// A [`Task`] is a future that has not yet started, and it is the job of diff --git a/crates/ty_server/src/server/schedule/thread/pool.rs b/crates/ty_server/src/server/schedule/thread/pool.rs index 0d9cf303d0..6f6c97f838 100644 --- a/crates/ty_server/src/server/schedule/thread/pool.rs +++ b/crates/ty_server/src/server/schedule/thread/pool.rs @@ -44,19 +44,36 @@ struct Job { } impl Pool { + /// A pool whose queue holds a few jobs, so that handing it more waits for a thread to take + /// one. pub(crate) fn new(threads: NonZeroUsize) -> Pool { + let capacity = std::cmp::min(usize::from(threads) * 2, 4); + Self::with_queue(threads, "ty:worker", crossbeam::channel::bounded(capacity)) + } + + /// A pool whose queue holds every job it is handed, so that handing it one never waits, + /// however long the job before it runs. For work that may run for long, and that the main + /// loop must be able to go on handing out, and cancelling, meanwhile. + pub(crate) fn unbounded(threads: NonZeroUsize, name: &str) -> Pool { + Self::with_queue(threads, name, crossbeam::channel::unbounded()) + } + + fn with_queue( + threads: NonZeroUsize, + name: &str, + (job_sender, job_receiver): (Sender, Receiver), + ) -> Pool { const INITIAL_PRIORITY: ThreadPriority = ThreadPriority::Worker; let threads = usize::from(threads); - let (job_sender, job_receiver) = crossbeam::channel::bounded(std::cmp::min(threads * 2, 4)); let extant_tasks = Arc::new(AtomicUsize::new(0)); let mut handles = Vec::with_capacity(threads); for i in 0..threads { let handle = Builder::new(INITIAL_PRIORITY) .stack_size(STACK_SIZE) - .name(format!("ty:worker:{i}")) + .name(format!("{name}:{i}")) .spawn({ let extant_tasks = Arc::clone(&extant_tasks); let job_receiver: Receiver = job_receiver.clone(); diff --git a/crates/ty_server/src/session/request_queue.rs b/crates/ty_server/src/session/request_queue.rs index 1e1fb0c14d..b9e8b89058 100644 --- a/crates/ty_server/src/session/request_queue.rs +++ b/crates/ty_server/src/session/request_queue.rs @@ -3,9 +3,8 @@ use lsp_server::RequestId; use rustc_hash::FxHashMap; use std::cell::{Cell, OnceCell, RefCell}; use std::fmt::Formatter; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; use std::time::Instant; +use ty_project::parallel::CancellationScope; /// Tracks the pending requests between client and server. pub(crate) struct RequestQueue { @@ -142,19 +141,28 @@ impl PendingRequest { /// Token to cancel a specific request. /// -/// Can be shared between threads to check for cancellation *after* a request has been scheduled. +/// Can be shared between threads to check for cancellation *after* a request has been scheduled, +/// and stops a request that is already running: the computations it [scopes](Self::scope) are +/// cancelled with it. #[derive(Debug, Default)] -pub(crate) struct RequestCancellationToken(Arc); +pub(crate) struct RequestCancellationToken(CancellationScope); impl RequestCancellationToken { /// Returns true if the request was cancelled. pub(crate) fn is_cancelled(&self) -> bool { - self.0.load(std::sync::atomic::Ordering::Relaxed) + self.0.is_cancelled() } - /// Signals that the request should not be processed because it was cancelled. + /// The request's computations: those on a database registered with it, and on the database + /// clones of the parallel work started inside it. + pub(crate) fn scope(&self) -> &CancellationScope { + &self.0 + } + + /// Signals that the request should not be processed because it was cancelled, and stops the + /// computations of one that is running. fn cancel(&self) { - self.0.store(true, std::sync::atomic::Ordering::Relaxed); + self.0.cancel(); } fn clone(this: &Self) -> Self { diff --git a/crates/ty_server/tests/e2e/usage_counts.rs b/crates/ty_server/tests/e2e/usage_counts.rs index 67aaa04e30..ffeefc5178 100644 --- a/crates/ty_server/tests/e2e/usage_counts.rs +++ b/crates/ty_server/tests/e2e/usage_counts.rs @@ -6,7 +6,10 @@ //! `textDocument/implementation` goes to from the declaration's name, which is what a client asks //! when the count is clicked +use std::time::Duration; + use anyhow::Result; +use lsp_server::ErrorCode; use lsp_types::{ Definition, ImplementationParams, ImplementationRequest, ImplementationResponse, LspRequestMethod, MessageDirection, PartialResultParams, Position, ReferenceContext, @@ -16,7 +19,7 @@ use lsp_types::{ use ruff_db::system::SystemPath; use ty_server::ClientOptions; -use crate::{TestServer, TestServerBuilder}; +use crate::{AwaitResponseError, TestServer, TestServerBuilder}; /// the request as a client sends it, in json throughout enum DocumentUsageCounts {} @@ -230,3 +233,61 @@ fn a_count_is_the_number_of_places_the_request_it_stands_for_goes_to() -> Result } Ok(()) } + +/// a search of the workspace the client gives up on, most often because the pass that asked was +/// cancelled by a keystroke, is stopped where it is: it is answered once, as cancelled, and the +/// count asked next, which waits for the search lane, is answered +#[test] +fn a_count_the_client_cancels_while_it_runs_is_answered_once_and_the_next_is_answered() -> Result<()> +{ + const MODULES: usize = 200; + + let mut builder = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes.by"), SHAPES)? + .with_file(SystemPath::new("src/main.by"), MAIN)?; + for module in 0..MODULES { + builder = builder.with_file( + SystemPath::new(&format!("src/user{module}.by")), + format!( + "from shapes import Shape, total\n\nclass Shape{module}(Shape):\n def area(self) -> float:\n return {module}.0\n\nprint(total([Shape{module}()]), Shape{module}().describe())\n" + ), + )?; + } + let mut server = builder.build().wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/shapes.by"), SHAPES, 1); + let uri = server.file_uri(SystemPath::new("src/shapes.by")); + let params = serde_json::json!({ + "textDocument": { "uri": uri }, + "counts": ["references", "implementations"], + }); + + let cancelled = server.send_request::(params.clone()); + server.cancel(&cancelled); + match server.try_await_response::(&cancelled, None) { + Err(AwaitResponseError::RequestFailed(error)) => { + assert_eq!(error.code, ErrorCode::RequestCanceled as i32); + } + other => panic!("expected the cancellation, got {other:?}"), + } + + let declarations = server + .send_request_await::(params) + .expect("an answer for a document"); + let shape = &declarations.as_array().expect("a list of declarations")[0]; + assert_eq!(shape["name"], "Shape"); + // `total` names it, and `main.by` and every module subclass it (an import is not a usage) + assert_eq!(shape["references"].as_u64(), Some(MODULES as u64 + 2)); + assert_eq!(shape["implementations"].as_u64(), Some(MODULES as u64 + 1)); + + // and the cancelled count does not bring a second answer + assert!(matches!( + server.try_await_response::( + &cancelled, + Some(Duration::from_millis(500)) + ), + Err(AwaitResponseError::Timeout) + )); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 49f70d2847..5158a1b7a8 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -568,14 +568,23 @@ its name, and the counts named in the request's `counts`: so the count is the number of places the click on it goes to, when a client sends that request from the name. a method with implementations is marked `abstract` when it is an `@abstractmethod` or a protocol member with no body, -so that what overrides it implements it. each count is a search of the -workspace per declaration, and a count not named is not worked out +so that what overrides it implements it. a count not named is not worked out + +each count is a search of the workspace, made for all the declarations of the +document in one pass: each file is read once, and searched for each declaration +whose name it holds. the searches run one at a time, on a thread of their own, +so however many a client has asked for, the requests someone is waiting on, +such as a hover or a completion, never queue behind them. a count the client +cancels stops where it is, its work in parallel included, so a client that asks +again on every edit and cancels the ask the edit made stale leaves no searches +running behind it a count depends on every file of the workspace, and the server announces no -change to it: a client asks again after any document of the workspace changes. -it is not a `textDocument/codeLens`, since a client that draws lenses asks for -them again only when their own document changes. like the other document -requests it takes `textHash` +change to it: a client asks again after any document of the workspace changes, +and asks for every count it draws in one request. it is not a +`textDocument/codeLens`, since a client that draws lenses asks for them again +only when their own document changes. like the other document requests it takes +`textHash` ## tests and what they test