From b6e8e8106835c0eeca92be2d67f4a145acb13a21 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 15:10:43 -0300 Subject: [PATCH 01/39] feat: implement dataset column rebuilding for transformed datasets --- DashAI/back/converters/dataset_columns.py | 108 +++++++++++++++++ DashAI/back/job/converter_job.py | 109 +----------------- tests/back/converters/test_dataset_columns.py | 53 +++++++++ 3 files changed, 166 insertions(+), 104 deletions(-) create mode 100644 DashAI/back/converters/dataset_columns.py create mode 100644 tests/back/converters/test_dataset_columns.py diff --git a/DashAI/back/converters/dataset_columns.py b/DashAI/back/converters/dataset_columns.py new file mode 100644 index 000000000..cad8406c5 --- /dev/null +++ b/DashAI/back/converters/dataset_columns.py @@ -0,0 +1,108 @@ +"""Shared helper for splicing a converter's transformed columns back into a +dataset, used both by the Notebooks converter job and by session-level +preprocessing (SessionPreprocessor). +""" + +from typing import TYPE_CHECKING, List + +if TYPE_CHECKING: + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + +def rebuild_dataset_with_transformed_columns( + base: "DashAIDataset", + transformed: "DashAIDataset", + scope_column_names: List[str], + scope_column_indexes: List[int], +) -> "DashAIDataset": + """ + Replaces specific columns in the base dataset with columns from the transformed + dataset, preserving their original positions. Also appends any additional columns + that were generated by the transformer at the end. Keeps the features and metadata + consistent. + + Parameters + ---------- + base : DashAIDataset + The original dataset before transformation. + + transformed : DashAIDataset + The dataset resulting from applying a transformer, containing updated and/or + new columns. + + scope_column_names : List[str] + Names of the columns that were originally selected for transformation. + + scope_column_indexes : List[int] + The indices of the columns in the base dataset that were replaced. + Must match the order of scope_column_names. + + Returns + ------- + DashAIDataset + A new dataset with the specified columns replaced in place, new columns + appended, and original metadata and split information preserved. + """ + from DashAI.back.dataloaders.classes.dashai_dataset import modify_table + + original_columns = base.column_names + transformed_cols = transformed.column_names + + transformed_cols_set = set(transformed_cols) + scope_column_names_set = set(scope_column_names) + + removed_cols = [ + col for col in scope_column_names if col not in transformed_cols_set + ] + replacement_cols = [ + col for col in scope_column_names if col in transformed_cols_set + ] + new_cols = [col for col in transformed_cols if col not in scope_column_names_set] + + removed_cols_set = set(removed_cols) + + new_columns_order = [] + seen_cols = set() + for col in original_columns: + if col in removed_cols_set: + continue + if col not in seen_cols: + new_columns_order.append(col) + seen_cols.add(col) + + col_name_mapping = {} + for col in new_cols: + unique_col = col + counter = 1 + while unique_col in seen_cols: + unique_col = f"{col}_{counter}" + counter += 1 + new_columns_order.append(unique_col) + seen_cols.add(unique_col) + col_name_mapping[col] = unique_col + + updated_arrays = {} + for col in replacement_cols: + if col in transformed_cols_set: + updated_arrays[col] = transformed.arrow_table[col] + for col, unique_col in col_name_mapping.items(): + if col in transformed_cols_set: + updated_arrays[unique_col] = transformed.arrow_table[col] + + updated_types = base.types.copy() + + for col in removed_cols: + if col in updated_types: + del updated_types[col] + + for col in replacement_cols: + if col in transformed.types: + updated_types[col] = transformed.types[col] + for col, unique_col in col_name_mapping.items(): + if col in transformed.types: + updated_types[unique_col] = transformed.types[col] + + modified_dataset = modify_table(base, updated_arrays, types=updated_types) + modified_dataset = modified_dataset.select_columns(new_columns_order) + + return modified_dataset diff --git a/DashAI/back/job/converter_job.py b/DashAI/back/job/converter_job.py index fcbfc4d68..3c6f964f0 100644 --- a/DashAI/back/job/converter_job.py +++ b/DashAI/back/job/converter_job.py @@ -1,10 +1,13 @@ import logging -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from kink import inject from sqlalchemy import exc from DashAI.back.api.api_v1.schemas.converter_params import ConverterParams +from DashAI.back.converters.dataset_columns import ( + rebuild_dataset_with_transformed_columns, +) from DashAI.back.dependencies.database.models import Converter from DashAI.back.dependencies.database.models import Dataset as DatasetModel from DashAI.back.job.base_job import BaseJob, JobError @@ -12,112 +15,10 @@ if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker - from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset - logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) -def _rebuild_dataset_with_transformed_columns( - base: "DashAIDataset", - transformed: "DashAIDataset", - scope_column_names: List[str], - scope_column_indexes: List[int], -) -> "DashAIDataset": - """ - Replaces specific columns in the base dataset with columns from the transformed - dataset, preserving their original positions. Also appends any additional columns - that were generated by the transformer at the end. Keeps the features and metadata - consistent. - - Parameters - ---------- - base : DashAIDataset - The original dataset before transformation. - - transformed : DashAIDataset - The dataset resulting from applying a transformer, containing updated and/or - new columns. - - scope_column_names : List[str] - Names of the columns that were originally selected for transformation. - - scope_column_indexes : List[int] - The indices of the columns in the base dataset that were replaced. - Must match the order of scope_column_names. - - Returns - ------- - DashAIDataset - A new dataset with the specified columns replaced in place, new columns - appended, and original metadata and split information preserved. - """ - from DashAI.back.dataloaders.classes.dashai_dataset import modify_table - - original_columns = base.column_names - transformed_cols = transformed.column_names - - transformed_cols_set = set(transformed_cols) - scope_column_names_set = set(scope_column_names) - - removed_cols = [ - col for col in scope_column_names if col not in transformed_cols_set - ] - replacement_cols = [ - col for col in scope_column_names if col in transformed_cols_set - ] - new_cols = [col for col in transformed_cols if col not in scope_column_names_set] - - removed_cols_set = set(removed_cols) - - new_columns_order = [] - seen_cols = set() - for col in original_columns: - if col in removed_cols_set: - continue - if col not in seen_cols: - new_columns_order.append(col) - seen_cols.add(col) - - col_name_mapping = {} - for col in new_cols: - unique_col = col - counter = 1 - while unique_col in seen_cols: - unique_col = f"{col}_{counter}" - counter += 1 - new_columns_order.append(unique_col) - seen_cols.add(unique_col) - col_name_mapping[col] = unique_col - - updated_arrays = {} - for col in replacement_cols: - if col in transformed_cols_set: - updated_arrays[col] = transformed.arrow_table[col] - for col, unique_col in col_name_mapping.items(): - if col in transformed_cols_set: - updated_arrays[unique_col] = transformed.arrow_table[col] - - updated_types = base.types.copy() - - for col in removed_cols: - if col in updated_types: - del updated_types[col] - - for col in replacement_cols: - if col in transformed.types: - updated_types[col] = transformed.types[col] - for col, unique_col in col_name_mapping.items(): - if col in transformed.types: - updated_types[unique_col] = transformed.types[col] - - # Use existing modify_table (imported at module level) - modified_dataset = modify_table(base, updated_arrays, types=updated_types) - modified_dataset = modified_dataset.select_columns(new_columns_order) - - return modified_dataset - - class ConverterJob(BaseJob): """ConverterJob class to modify a dataset by applying a sequence of converters.""" @@ -410,7 +311,7 @@ def instantiate_converters( if type(converter_instance).CHANGES_ROW_COUNT: loaded_dataset = transformed_dataset else: - loaded_dataset = _rebuild_dataset_with_transformed_columns( + loaded_dataset = rebuild_dataset_with_transformed_columns( loaded_dataset, transformed_dataset, scope_column_names, diff --git a/tests/back/converters/test_dataset_columns.py b/tests/back/converters/test_dataset_columns.py new file mode 100644 index 000000000..754758d47 --- /dev/null +++ b/tests/back/converters/test_dataset_columns.py @@ -0,0 +1,53 @@ +import pandas as pd + +from DashAI.back.converters.dataset_columns import ( + rebuild_dataset_with_transformed_columns, +) +from DashAI.back.dataloaders.classes.dashai_dataset import ( + to_dashai_dataset, + transform_dataset_with_schema, +) + + +def _dataset(columns, schema): + return transform_dataset_with_schema( + to_dashai_dataset(pd.DataFrame(columns)), schema + ) + + +def test_replaces_scoped_column_in_place_and_keeps_others(): + base = _dataset( + {"a": [1, 2, 3], "b": [10, 20, 30]}, + { + "a": {"type": "Integer", "dtype": "int64"}, + "b": {"type": "Integer", "dtype": "int64"}, + }, + ) + transformed = _dataset( + {"a": [0, 1, 0]}, + {"a": {"type": "Integer", "dtype": "int64"}}, + ) + + result = rebuild_dataset_with_transformed_columns(base, transformed, ["a"], [0]) + + assert result.column_names == ["a", "b"] + assert result.to_pandas()["a"].tolist() == [0, 1, 0] + assert result.to_pandas()["b"].tolist() == [10, 20, 30] + + +def test_appends_new_columns_not_in_the_scope(): + base = _dataset( + {"a": [1, 2, 3]}, + {"a": {"type": "Integer", "dtype": "int64"}}, + ) + transformed = _dataset( + {"a_bin_0": [1, 0, 1], "a_bin_1": [0, 1, 0]}, + { + "a_bin_0": {"type": "Integer", "dtype": "int64"}, + "a_bin_1": {"type": "Integer", "dtype": "int64"}, + }, + ) + + result = rebuild_dataset_with_transformed_columns(base, transformed, ["a"], [0]) + + assert result.column_names == ["a_bin_0", "a_bin_1"] From 26cd0b3f63c19a538b41e92aec737824d99f8c3c Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 15:19:45 -0300 Subject: [PATCH 02/39] feat: add column reference models and tests for session-level preprocessing --- DashAI/back/preprocessing/__init__.py | 0 DashAI/back/preprocessing/column_ref.py | 94 +++++++++++++++++++++ tests/back/preprocessing/__init__.py | 0 tests/back/preprocessing/test_column_ref.py | 78 +++++++++++++++++ 4 files changed, 172 insertions(+) create mode 100644 DashAI/back/preprocessing/__init__.py create mode 100644 DashAI/back/preprocessing/column_ref.py create mode 100644 tests/back/preprocessing/__init__.py create mode 100644 tests/back/preprocessing/test_column_ref.py diff --git a/DashAI/back/preprocessing/__init__.py b/DashAI/back/preprocessing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/DashAI/back/preprocessing/column_ref.py b/DashAI/back/preprocessing/column_ref.py new file mode 100644 index 000000000..ad20a2a4f --- /dev/null +++ b/DashAI/back/preprocessing/column_ref.py @@ -0,0 +1,94 @@ +"""Column references used by session-level preprocessing. + +A ColumnRef identifies either a real column already present in a dataset +(RawColumnRef) or the not-yet-materialized output of a converter step in a +ConverterSequence (GroupColumnRef). Group references let the Models-module +wizard offer "whatever this converter produces" as an input column before +any fit has happened — the concrete names only exist once the +PreprocessingJob has fit the sequence (see session_preprocessor.py). +""" + +from typing import Any, Dict, List, Literal, Union + +from pydantic import BaseModel, Field, TypeAdapter +from typing_extensions import Annotated + + +class RawColumnRef(BaseModel): + kind: Literal["raw"] = "raw" + name: str + + +class GroupColumnRef(BaseModel): + kind: Literal["group"] = "group" + step: int + + +ColumnRef = Annotated[Union[RawColumnRef, GroupColumnRef], Field(discriminator="kind")] + +_ColumnRefListAdapter = TypeAdapter(List[ColumnRef]) + + +class ConverterStep(BaseModel): + converter: str + params: Dict[str, Any] = Field(default_factory=dict) + scope: List[ColumnRef] = Field(default_factory=list) + + +class ConverterSequence(BaseModel): + steps: List[ConverterStep] = Field(default_factory=list) + + def validate_scopes(self) -> None: + """Raise ValueError if any step's scope references itself or a later step. + + A step may only reference the output group of a step strictly before + it — this is what makes chaining acyclic without a separate graph + structure. + """ + for index, step in enumerate(self.steps): + for ref in step.scope: + if isinstance(ref, GroupColumnRef) and ref.step >= index: + raise ValueError( + f"Step {index} ('{step.converter}') scope references " + f"step {ref.step}, which is not strictly before it." + ) + + +def resolve_refs( + refs: List[Union[RawColumnRef, GroupColumnRef]], + resolved_columns: Dict[int, List[str]], +) -> List[str]: + """Flatten a list of ColumnRef into concrete column names. + + Parameters + ---------- + refs : list of RawColumnRef | GroupColumnRef + References to resolve, in the order they should appear in the result. + resolved_columns : dict + Maps a ConverterSequence step index to the concrete column names that + step produced in one specific fit (see SessionPreprocessor). + + Returns + ------- + list of str + Concrete column names, in order. A GroupColumnRef expands to every + column its step produced. + + Raises + ------ + KeyError + If a GroupColumnRef names a step with no entry in resolved_columns + (the step has not been fit yet). + """ + names: List[str] = [] + for ref in refs: + if ref.kind == "raw": + names.append(ref.name) + else: + names.extend(resolved_columns[ref.step]) + return names + + +def parse_column_refs(raw: List[dict]) -> List[Union[RawColumnRef, GroupColumnRef]]: + """Parse a list of plain dicts (as stored in JSON columns) into ColumnRef.""" + return _ColumnRefListAdapter.validate_python(raw) diff --git a/tests/back/preprocessing/__init__.py b/tests/back/preprocessing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/back/preprocessing/test_column_ref.py b/tests/back/preprocessing/test_column_ref.py new file mode 100644 index 000000000..84b32538a --- /dev/null +++ b/tests/back/preprocessing/test_column_ref.py @@ -0,0 +1,78 @@ +import pytest + +from DashAI.back.preprocessing.column_ref import ( + ConverterSequence, + ConverterStep, + GroupColumnRef, + RawColumnRef, + parse_column_refs, + resolve_refs, +) + + +def test_raw_ref_resolves_to_its_own_name(): + assert resolve_refs([RawColumnRef(name="age")], {}) == ["age"] + + +def test_group_ref_resolves_to_that_steps_produced_columns(): + refs = [RawColumnRef(name="age"), GroupColumnRef(step=0)] + resolved = resolve_refs(refs, {0: ["bow_apple", "bow_banana"]}) + assert resolved == ["age", "bow_apple", "bow_banana"] + + +def test_group_ref_to_an_unfit_step_raises_key_error(): + with pytest.raises(KeyError): + resolve_refs([GroupColumnRef(step=0)], {}) + + +def test_sequence_rejects_a_step_referencing_itself_or_later(): + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Binarizer", + params={}, + scope=[GroupColumnRef(step=0)], + ) + ] + ) + with pytest.raises(ValueError, match="not strictly before"): + sequence.validate_scopes() + + +def test_sequence_accepts_a_step_referencing_an_earlier_one(): + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="BagOfWordsConverter", + params={}, + scope=[RawColumnRef(name="text")], + ), + ConverterStep( + converter="StandardScaler", + params={}, + scope=[GroupColumnRef(step=0)], + ), + ] + ) + sequence.validate_scopes() # does not raise + + +def test_parse_column_refs_round_trips_json_dicts(): + raw = [{"kind": "raw", "name": "age"}, {"kind": "group", "step": 2}] + parsed = parse_column_refs(raw) + assert parsed == [RawColumnRef(name="age"), GroupColumnRef(step=2)] + + +def test_converter_sequence_model_dump_round_trips(): + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Binarizer", + params={"threshold": 0.5}, + scope=[RawColumnRef(name="age")], + ) + ] + ) + dumped = sequence.model_dump(mode="json") + restored = ConverterSequence.model_validate(dumped) + assert restored == sequence From dfdb0541c090aa6faa7d4e6172b23a55cdf136a5 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 15:27:36 -0300 Subject: [PATCH 03/39] feat: implement SessionPreprocessor for dataset transformation and fitting --- .../preprocessing/session_preprocessor.py | 143 ++++++++++++++++ .../test_session_preprocessor.py | 161 ++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 DashAI/back/preprocessing/session_preprocessor.py create mode 100644 tests/back/preprocessing/test_session_preprocessor.py diff --git a/DashAI/back/preprocessing/session_preprocessor.py b/DashAI/back/preprocessing/session_preprocessor.py new file mode 100644 index 000000000..c6e93cbd8 --- /dev/null +++ b/DashAI/back/preprocessing/session_preprocessor.py @@ -0,0 +1,143 @@ +"""Fits and applies a ConverterSequence against dataset partitions. + +SessionPreprocessor is fit once per session (per fold for Cross-Validation, +once for Holdout) by PreprocessingJob, then persisted so training, +prediction and explanation can reuse the exact fit without ever re-fitting +on new data (see load_final_preprocessor at the bottom of this module). +""" + +import os +import pickle +from typing import TYPE_CHECKING, Any, Dict, List, Tuple + +from DashAI.back.converters.dataset_columns import ( + rebuild_dataset_with_transformed_columns, +) +from DashAI.back.preprocessing.column_ref import ConverterSequence, resolve_refs + +if TYPE_CHECKING: + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + +class SessionPreprocessor: + """Fits and applies converters from a ConverterSequence. + + Each step's scope may reference raw dataset columns or the output group + of an earlier step. Fitting always uses only the "train" entry of + whatever split dict is passed in, so no step ever sees validation or + test rows during fit. + """ + + def __init__(self, sequence: ConverterSequence, component_registry: Any): + self.sequence = sequence + self.component_registry = component_registry + self.fitted_converters: List[Any] = [] + self.resolved_columns: Dict[int, List[str]] = {} + + def _instantiate(self, step) -> Any: + converter_class = self.component_registry[step.converter]["class"] + return converter_class(**step.params) + + def fit_transform( + self, split: Dict[str, "DashAIDataset"] + ) -> Tuple[Dict[str, "DashAIDataset"], Dict[int, List[str]]]: + """Fit every step on split["train"] and transform every split present. + + Parameters + ---------- + split : dict + Whatever partitions are present (e.g. {"train", "validation"}, + {"train", "validation", "test"} or {"train", "test"}) are + transformed, but only "train" is ever used to fit. + + Returns + ------- + tuple + (transformed_split, resolved_columns) where transformed_split + has the same keys as split and resolved_columns maps each + step's index to the concrete column names it produced. + """ + current: Dict[str, "DashAIDataset"] = dict(split) + self.fitted_converters = [] + self.resolved_columns = {} + + for index, step in enumerate(self.sequence.steps): + scope_names = resolve_refs(step.scope, self.resolved_columns) + converter = self._instantiate(step) + + train_scope = current["train"].select_columns(scope_names) + converter = converter.fit(train_scope) + + transformed_by_split = {} + for split_name, dataset in current.items(): + scoped = dataset.select_columns(scope_names) + transformed_by_split[split_name] = converter.transform(scoped) + + self.resolved_columns[index] = list( + transformed_by_split["train"].column_names + ) + + new_current = {} + for split_name, dataset in current.items(): + if type(converter).CHANGES_ROW_COUNT: + new_current[split_name] = transformed_by_split[split_name] + else: + scope_indexes = [ + dataset.column_names.index(name) for name in scope_names + ] + new_current[split_name] = rebuild_dataset_with_transformed_columns( + dataset, + transformed_by_split[split_name], + scope_names, + scope_indexes, + ) + current = new_current + self.fitted_converters.append(converter) + + return current, self.resolved_columns + + def transform_only( + self, split: Dict[str, "DashAIDataset"] + ) -> Dict[str, "DashAIDataset"]: + """Apply already-fitted converters to new data, without fitting. + + Used after unpickling a SessionPreprocessor that was fit earlier (by + PreprocessingJob), to transform fold data at training time, or a + prediction/explanation input. + """ + current: Dict[str, "DashAIDataset"] = dict(split) + for index, converter in enumerate(self.fitted_converters): + scope_names = resolve_refs( + self.sequence.steps[index].scope, self.resolved_columns + ) + new_current = {} + for split_name, dataset in current.items(): + scoped = dataset.select_columns(scope_names) + transformed = converter.transform(scoped) + if type(converter).CHANGES_ROW_COUNT: + new_current[split_name] = transformed + else: + scope_indexes = [ + dataset.column_names.index(name) for name in scope_names + ] + new_current[split_name] = rebuild_dataset_with_transformed_columns( + dataset, transformed, scope_names, scope_indexes + ) + current = new_current + return current + + def transform_dataset(self, dataset: "DashAIDataset") -> "DashAIDataset": + """Convenience wrapper for a single dataset (predict/explain use).""" + return self.transform_only({"train": dataset})["train"] + + +def load_final_preprocessor(model_session: Any) -> "SessionPreprocessor": + """Load the SessionPreprocessor fitted on the session's full training pool. + + Used by prediction and explanation, which must transform new raw data + the exact same way the model's training data was transformed, without + ever re-fitting on that new data. + """ + path = os.path.join(model_session.preprocessing_artifacts_path, "final.pkl") + with open(path, "rb") as f: + return pickle.load(f) diff --git a/tests/back/preprocessing/test_session_preprocessor.py b/tests/back/preprocessing/test_session_preprocessor.py new file mode 100644 index 000000000..775432311 --- /dev/null +++ b/tests/back/preprocessing/test_session_preprocessor.py @@ -0,0 +1,161 @@ +import pandas as pd + +from DashAI.back.converters.base_converter import BaseConverter +from DashAI.back.dataloaders.classes.dashai_dataset import ( + to_dashai_dataset, + transform_dataset_with_schema, +) +from DashAI.back.preprocessing.column_ref import ( + ConverterSequence, + ConverterStep, + GroupColumnRef, + RawColumnRef, +) +from DashAI.back.preprocessing.session_preprocessor import SessionPreprocessor +from DashAI.back.types.value_types import Integer + + +def _dataset(columns, schema): + return transform_dataset_with_schema( + to_dashai_dataset(pd.DataFrame(columns)), schema + ) + + +class _DoublingConverter(BaseConverter): + """Deterministic: doubles a single numeric column in place.""" + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + self._column = x.column_names[0] + return self + + def transform(self, x, y=None): + frame = x.to_pandas() + frame[self._column] = frame[self._column] * 2 + return to_dashai_dataset(frame) + + +class _FakeVocabConverter(BaseConverter): + """Variable output: emits one column per 'word' seen in fit, mirroring + Bag-of-Words' vocabulary-at-fit-time behavior without needing sklearn.""" + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + column = x.column_names[0] + self._vocab = sorted(set(x.to_pandas()[column].tolist())) + return self + + def transform(self, x, y=None): + import pyarrow as pa + + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + column = x.column_names[0] + values = x.to_pandas()[column].tolist() + output_type = self.get_output_type() + data = { + f"vocab_{word}": [1 if v == word else 0 for v in values] + for word in self._vocab + } + table = pa.table(data) + types = dict.fromkeys(data, output_type) + return DashAIDataset(table, types=types) + + +class _FakeRegistry: + def __init__(self, classes): + self._classes = classes + + def __getitem__(self, name): + return {"class": self._classes[name]} + + +def _split(train_ages, val_ages=None): + train = _dataset( + {"age": train_ages}, {"age": {"type": "Integer", "dtype": "int64"}} + ) + split = {"train": train} + if val_ages is not None: + split["validation"] = _dataset( + {"age": val_ages}, {"age": {"type": "Integer", "dtype": "int64"}} + ) + return split + + +def test_fits_only_on_train_and_transforms_every_split_present(): + registry = _FakeRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + split = _split(train_ages=[1, 2, 3], val_ages=[10, 20]) + transformed, resolved = preprocessor.fit_transform(split) + + assert transformed["train"].to_pandas()["age"].tolist() == [2, 4, 6] + assert transformed["validation"].to_pandas()["age"].tolist() == [20, 40] + assert resolved == {0: ["age"]} + + +def test_variable_output_step_records_whatever_columns_it_produced(): + registry = _FakeRegistry({"FakeVocab": _FakeVocabConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="FakeVocab", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + split = _split(train_ages=[1, 2, 1]) + transformed, resolved = preprocessor.fit_transform(split) + + assert sorted(resolved[0]) == ["vocab_1", "vocab_2"] + assert set(transformed["train"].column_names) == {"vocab_1", "vocab_2"} + + +def test_chained_step_can_reference_the_previous_steps_group(): + registry = _FakeRegistry( + {"FakeVocab": _FakeVocabConverter, "Doubler": _DoublingConverter} + ) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="FakeVocab", params={}, scope=[RawColumnRef(name="age")] + ), + ConverterStep( + converter="Doubler", params={}, scope=[GroupColumnRef(step=0)] + ), + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + split = _split(train_ages=[1, 2]) + transformed, resolved = preprocessor.fit_transform(split) + + # Doubler's scope is step 0's whole output group (both vocab columns); + # it doubles column_names[0] but its transform returns every column in + # its scope, so step 1 produces the same column set step 0 did. + assert set(resolved[1]) == set(resolved[0]) + assert len(resolved[1]) == 2 From 1e6f63b8e699e0b54cd5d482d6b27c1cf1b3eb87 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 22:15:15 -0300 Subject: [PATCH 04/39] feat: dispatch a PreprocessingJob when a session has converters Fits a session's converter sequence once at creation time (per fold for Cross-Validation, once for Holdout) instead of applying converters on the full dataset before any split exists, which leaked validation/test data into the fit. Resolves the session's input columns from the fitted sequence and persists preprocessing status/errors on the session. --- ...d8e7_add_preprocessing_to_model_session.py | 60 ++++++ .../api/api_v1/endpoints/model_sessions.py | 98 ++++++++- .../api_v1/schemas/model_sessions_params.py | 10 +- DashAI/back/config.py | 1 + DashAI/back/dependencies/config_builder.py | 1 + DashAI/back/dependencies/database/models.py | 14 ++ DashAI/back/job/preprocessing_job.py | 187 ++++++++++++++++++ tests/back/api/test_model_session_api.py | 116 +++++++++++ tests/back/migrations/__init__.py | 0 ...test_add_preprocessing_to_model_session.py | 30 +++ tests/back/test_config_builder.py | 6 + 11 files changed, 518 insertions(+), 5 deletions(-) create mode 100644 DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py create mode 100644 DashAI/back/job/preprocessing_job.py create mode 100644 tests/back/migrations/__init__.py create mode 100644 tests/back/migrations/test_add_preprocessing_to_model_session.py create mode 100644 tests/back/test_config_builder.py diff --git a/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py b/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py new file mode 100644 index 000000000..c875af368 --- /dev/null +++ b/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py @@ -0,0 +1,60 @@ +"""add preprocessing columns to model_session + +Revision ID: f4a91c62d8e7 +Revises: a5f2c71e9d40 +Create Date: 2026-09-08 10:00:00.000000 + +Sessions created before this column existed have no preprocessing, so an +empty/None value means exactly that — no backfill is needed. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "f4a91c62d8e7" +down_revision: Union[str, None] = "a5f2c71e9d40" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_NEW_COLUMNS = [ + ("preprocessing", sa.JSON(), True, None), + ("input_column_refs", sa.JSON(), True, None), + ("preprocessing_status", sa.String(), False, "ready"), + ("preprocessing_error", sa.String(), True, None), + ("preprocessing_artifacts_path", sa.String(), True, None), + ("preprocessing_job_id", sa.String(), True, None), +] + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "model_session" not in inspector.get_table_names(): + return + existing = {column["name"] for column in inspector.get_columns("model_session")} + for name, col_type, nullable, server_default in _NEW_COLUMNS: + if name in existing: + continue + op.add_column( + "model_session", + sa.Column( + name, + col_type, + nullable=nullable, + server_default=server_default, + ), + ) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "model_session" not in inspector.get_table_names(): + return + existing = {column["name"] for column in inspector.get_columns("model_session")} + for name, _, _, _ in _NEW_COLUMNS: + if name not in existing: + continue + op.drop_column("model_session", name) diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index c68153df9..72ff17c0d 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -14,6 +14,8 @@ ) from DashAI.back.api.utils import remove_path from DashAI.back.dependencies.database.models import Dataset, ModelSession, Run +from DashAI.back.job.preprocessing_job import PreprocessingJob +from DashAI.back.preprocessing.column_ref import ConverterSequence, RawColumnRef from DashAI.back.splitters.splits_payload import ( META_KEYS, normalize_splits_payload, @@ -23,6 +25,7 @@ if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker + from DashAI.back.dependencies.job_queues import BaseJobQueue from DashAI.back.dependencies.registry import ComponentRegistry from DashAI.back.tasks.base_task import BaseTask @@ -32,6 +35,22 @@ router = APIRouter() +# Every concrete value type name that DashAIValue's wildcard covers (i.e. every +# DashAIValue subclass in DashAI/back/types/value_types.py), used so a task +# declaring the wildcard (e.g. ClassificationTask) still accepts a converter +# group whose declared type is one of these, not just "DashAIValue" itself. +_DASHAI_VALUE_TYPE_NAMES = { + "Integer", + "Float", + "Text", + "Time", + "Timestamp", + "Duration", + "Decimal", + "Date", + "Binary", +} + @router.get("/") @inject @@ -139,13 +158,23 @@ async def validate_columns( column_names = minimal_dataset.column_names - if len(params.inputs_columns + params.outputs_columns) > len(column_names): + group_refs = [ + ref for ref in (params.input_refs or []) if ref.kind == "group" + ] + + if not group_refs and len( + params.inputs_columns + params.outputs_columns + ) > len(column_names): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Column index out of range", ) - inputs_names = params.inputs_columns + inputs_names = ( + [r.name for r in params.input_refs if r.kind == "raw"] + if params.input_refs + else params.inputs_columns + ) outputs_names = params.outputs_columns except exc.SQLAlchemyError as e: @@ -162,6 +191,27 @@ async def validate_columns( ) task: "BaseTask" = component_registry[params.task_name]["class"]() + + if group_refs: + declared_types = params.converter_output_types or {} + task_metadata = task.get_metadata() + allowed_input_types = set(task_metadata.get("inputs_types", [])) + for ref in group_refs: + declared_type = declared_types.get(str(ref.step)) + type_ok = declared_type in allowed_input_types or ( + "DashAIValue" in allowed_input_types + and declared_type in _DASHAI_VALUE_TYPE_NAMES + ) + if allowed_input_types and not type_ok: + return { + "dataset_status": "invalid", + "error": ( + f"Converter step {ref.step} declares output type " + f"'{declared_type}', which is not one of the task's " + f"allowed input types {sorted(allowed_input_types)}." + ), + } + validation_response = {} try: @@ -236,6 +286,7 @@ async def create_model_session( params: ModelSessionParams, session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), + job_queue: "BaseJobQueue" = Depends(lambda: di["job_queue"]), ): """Create a new model session. @@ -268,6 +319,28 @@ async def create_model_session( _validate_splits(params.splits, component_registry) + sequence = ConverterSequence(steps=params.preprocessing) + try: + sequence.validate_scopes() + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(e), + ) from e + has_preprocessing = len(sequence.steps) > 0 + + if has_preprocessing and not params.input_column_refs: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=( + "input_column_refs is required when preprocessing steps are provided" + ), + ) + + input_column_refs = params.input_column_refs or [ + RawColumnRef(name=name) for name in params.input_columns + ] + with session_factory() as db: try: dataset = db.get(Dataset, params.dataset_id) @@ -283,7 +356,12 @@ async def create_model_session( schema = reader.schema column_names = schema.names - if len(params.input_columns + params.output_columns) > len(column_names): + # When preprocessing is configured, input_columns is only a + # placeholder until PreprocessingJob resolves the real ones, so + # it cannot be checked against the raw dataset's column count. + if not has_preprocessing and len( + params.input_columns + params.output_columns + ) > len(column_names): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Column index out of range", @@ -300,10 +378,24 @@ async def create_model_session( test_metrics=params.test_metrics, evaluation_strategy=params.evaluation_strategy, splits=params.splits, + preprocessing=sequence.model_dump(mode="json"), + input_column_refs=[ + ref.model_dump(mode="json") for ref in input_column_refs + ], + preprocessing_status="pending" if has_preprocessing else "ready", ) db.add(model_session) db.commit() db.refresh(model_session) + + if has_preprocessing: + job_id = job_queue.put( + PreprocessingJob(model_session_id=model_session.id) + ).id + model_session.preprocessing_job_id = str(job_id) + db.commit() + db.refresh(model_session) + return model_session except exc.IntegrityError as e: db.rollback() diff --git a/DashAI/back/api/api_v1/schemas/model_sessions_params.py b/DashAI/back/api/api_v1/schemas/model_sessions_params.py index 5de8f632f..f5e35e415 100644 --- a/DashAI/back/api/api_v1/schemas/model_sessions_params.py +++ b/DashAI/back/api/api_v1/schemas/model_sessions_params.py @@ -1,6 +1,8 @@ -from typing import List +from typing import Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field + +from DashAI.back.preprocessing.column_ref import ColumnRef, ConverterStep class ModelSessionParams(BaseModel): @@ -14,6 +16,8 @@ class ModelSessionParams(BaseModel): test_metrics: List[str] evaluation_strategy: str splits: str + preprocessing: List[ConverterStep] = Field(default_factory=list) + input_column_refs: Optional[List[ColumnRef]] = None class ColumnsValidationParams(BaseModel): @@ -21,6 +25,8 @@ class ColumnsValidationParams(BaseModel): dataset_id: int inputs_columns: List[str] outputs_columns: List[str] + input_refs: Optional[List[ColumnRef]] = None + converter_output_types: Optional[Dict[str, str]] = None class ModelSessionBulkDeleteParams(BaseModel): diff --git a/DashAI/back/config.py b/DashAI/back/config.py index ef88d6031..98c1dc989 100644 --- a/DashAI/back/config.py +++ b/DashAI/back/config.py @@ -19,6 +19,7 @@ class DefaultSettings(BaseSettings): DATASETS_PATH: str = "datasets" IMAGES_PATH: str = "images" RUNS_PATH: str = "runs" + PREPROCESSING_PATH: str = "preprocessing" EXPLANATIONS_PATH: str = "explanations" EXPLORATIONS_PATH: str = "explorations" DOCUMENTS_PATH: str = "documents" diff --git a/DashAI/back/dependencies/config_builder.py b/DashAI/back/dependencies/config_builder.py index ef6c1ba2a..6a57c52cd 100644 --- a/DashAI/back/dependencies/config_builder.py +++ b/DashAI/back/dependencies/config_builder.py @@ -57,6 +57,7 @@ def build_config_dict( config["EXPLANATIONS_PATH"] = local_path / config["EXPLANATIONS_PATH"] config["NOTEBOOK_PATH"] = local_path / config["NOTEBOOK_PATH"] config["RUNS_PATH"] = local_path / config["RUNS_PATH"] + config["PREPROCESSING_PATH"] = local_path / config["PREPROCESSING_PATH"] config["IMAGES_PATH"] = local_path / config["IMAGES_PATH"] config["DATAFILE_PATH"] = local_path / config["DATAFILE_PATH"] config["CREDENTIALS_KEY_PATH"] = local_path / config["CREDENTIALS_KEY_PATH"] diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index fa7b2fe6e..dc6df0d61 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -145,6 +145,20 @@ class ModelSession(Base): evaluation_strategy: Mapped[str] = mapped_column(String, nullable=False) splits: Mapped[str] = mapped_column(JSON, nullable=False) + preprocessing: Mapped[dict] = mapped_column(JSON, nullable=True) + input_column_refs: Mapped[list] = mapped_column(JSON, nullable=True) + preprocessing_status: Mapped[str] = mapped_column( + String, nullable=False, default="ready", server_default="ready" + ) + preprocessing_error: Mapped[str] = mapped_column(String, nullable=True) + preprocessing_artifacts_path: Mapped[str] = mapped_column(String, nullable=True) + # The Huey task id PreprocessingJob was enqueued with — lets the frontend + # track this specific job via the same shared job-polling mechanism the + # Job Queue widget uses (useJobTracker/startJobPolling), instead of + # polling preprocessing_status on its own separate timer. Keeping both + # "is it done" signals on the same underlying poll loop is what keeps + # them from ever drifting out of sync with each other. + preprocessing_job_id: Mapped[str] = mapped_column(String, nullable=True) created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) last_modified: Mapped[DateTime] = mapped_column( DateTime, diff --git a/DashAI/back/job/preprocessing_job.py b/DashAI/back/job/preprocessing_job.py new file mode 100644 index 000000000..be34995db --- /dev/null +++ b/DashAI/back/job/preprocessing_job.py @@ -0,0 +1,187 @@ +import json +import logging +import os +import pickle +from typing import TYPE_CHECKING + +from kink import inject +from sqlalchemy import exc + +from DashAI.back.core.atomic import atomic_directory +from DashAI.back.dependencies.database.models import Dataset, ModelSession +from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.preprocessing.column_ref import ( + ConverterSequence, + parse_column_refs, + resolve_refs, +) +from DashAI.back.preprocessing.session_preprocessor import SessionPreprocessor +from DashAI.back.splitters.splits_payload import normalize_splits_payload + +if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + + +class PreprocessingJob(BaseJob): + """Fits a session's ConverterSequence once, ahead of any model training. + + Fits per fold for Cross-Validation (plus a final fit on the full + training pool) or once for Holdout, and persists each fitted + SessionPreprocessor so ModelJob, predict_job and explainer_job can + reuse it without ever re-fitting on new data. + """ + + @inject + def set_status_as_delivered( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + model_session_id = self.kwargs["model_session_id"] + with session_factory() as db: + model_session = db.get(ModelSession, model_session_id) + if model_session is None: + raise JobError( + f"Model session {model_session_id} does not exist in DB." + ) + try: + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Error setting preprocessing status as delivered") from e + + @inject + def set_status_as_error( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + model_session_id = self.kwargs.get("model_session_id") + if model_session_id is None: + return + with session_factory() as db: + model_session = db.get(ModelSession, model_session_id) + if model_session is None: + return + model_session.preprocessing_status = "failed" + try: + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + + @inject + def get_job_name(self) -> str: + model_session_id = self.kwargs.get("model_session_id") + return f"Preprocessing: session {model_session_id}" + + @inject + def run(self) -> None: + from kink import di + + from DashAI.back.dataloaders.classes.dashai_dataset import load_dataset + + session_factory = di["session_factory"] + component_registry = di["component_registry"] + config = di["config"] + + model_session_id = self.kwargs["model_session_id"] + + with session_factory() as db: + model_session: ModelSession = db.get(ModelSession, model_session_id) + if not model_session: + raise JobError( + f"Model session {model_session_id} does not exist in DB." + ) + + try: + model_session.preprocessing_status = "pending" + db.commit() + + dataset = db.get(Dataset, model_session.dataset_id) + if not dataset: + raise JobError( + f"Dataset {model_session.dataset_id} does not exist in DB." + ) + loaded_dataset = load_dataset(f"{dataset.file_path}/dataset") + + sequence = ConverterSequence.model_validate( + model_session.preprocessing or {"steps": []} + ) + sequence.validate_scopes() + input_refs = parse_column_refs(model_session.input_column_refs or []) + + self.report_progress(0.05, "Splitting dataset") + splits_data = normalize_splits_payload(json.loads(model_session.splits)) + splitter_name = splits_data.get("splitter_name") + splitter = component_registry[splitter_name]["class"]( + splits_data=splits_data + ) + + y_for_split = loaded_dataset.select_columns( + model_session.output_columns + ) + x, _, _ = splitter.split(loaded_dataset, y_for_split) + + is_cv = isinstance(x, list) + x_folds = x if is_cv else [x] + total_folds = len(x_folds) - 1 if is_cv else 0 + + artifacts_dir = os.path.join( + str(config["PREPROCESSING_PATH"]), str(model_session.id) + ) + + final_transformed = None + final_resolved = None + with atomic_directory(artifacts_dir) as tmp_dir: + for i in range(total_folds): + self.report_progress( + 0.1 + 0.7 * (i / max(total_folds, 1)), + f"Fitting preprocessing for fold {i + 1}/{total_folds}", + ) + fold_preprocessor = SessionPreprocessor( + sequence, component_registry + ) + fold_preprocessor.fit_transform(x_folds[i]) + with open(os.path.join(tmp_dir, f"fold_{i}.pkl"), "wb") as f: + pickle.dump(fold_preprocessor, f) + + self.report_progress(0.85, "Fitting final preprocessing") + final_preprocessor = SessionPreprocessor( + sequence, component_registry + ) + final_transformed, final_resolved = ( + final_preprocessor.fit_transform(x_folds[-1]) + ) + with open(os.path.join(tmp_dir, "final.pkl"), "wb") as f: + pickle.dump(final_preprocessor, f) + + resolved_input_columns = resolve_refs(input_refs, final_resolved) + + self.report_progress(0.95, "Validating against the task") + task = component_registry[model_session.task_name]["class"]() + task.prepare_for_task( + dataset=final_transformed["train"], + input_columns=resolved_input_columns, + output_columns=model_session.output_columns, + ) + + model_session.input_columns = resolved_input_columns + model_session.preprocessing_artifacts_path = artifacts_dir + model_session.preprocessing_status = "ready" + model_session.preprocessing_error = None + db.commit() + except (TypeError, ValueError) as e: + log.exception(e) + model_session.preprocessing_status = "failed" + model_session.preprocessing_error = str(e) + db.commit() + raise JobError( + f"Preprocessing produced columns invalid for the task: {e}" + ) from e + except Exception as e: + log.exception(e) + model_session.preprocessing_status = "failed" + model_session.preprocessing_error = str(e) + db.commit() + raise JobError( + f"Error running preprocessing for session {model_session_id}: {e}" + ) from e diff --git a/tests/back/api/test_model_session_api.py b/tests/back/api/test_model_session_api.py index 23624d537..c6167a959 100644 --- a/tests/back/api/test_model_session_api.py +++ b/tests/back/api/test_model_session_api.py @@ -50,6 +50,122 @@ def test_delete_model_session(client: TestClient, dataset_1: Dataset) -> None: assert response.status_code == 404, response.text +def test_create_model_session_without_preprocessing_is_ready_immediately( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "no-preprocessing-session", + }, + ) + assert response.status_code == 201, response.text + assert response.json()["preprocessing_status"] == "ready" + + session_id = response.json()["id"] + client.delete(f"/api/v1/model-session/{session_id}") + + +def test_create_model_session_with_preprocessing_fits_and_resolves_columns( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "with-preprocessing-session", + "input_columns": [], + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert response.status_code == 201, response.text + body = response.json() + # The test client runs the job queue in immediate mode, so by the time + # the request returns, PreprocessingJob already fit and resolved this. + # Binarizer is an EncodingConverter: it appends "bin_" and keeps the + # original column unchanged (a passthrough, like BagOfWords keeping its + # source text column) — the group is only the genuinely new column it + # produced, not the untouched original alongside it. + assert body["preprocessing_status"] == "ready" + assert body["input_columns"] == ["bin_SepalLengthCm"] + # The frontend tracks this exact job via the shared job-polling + # mechanism (useJobTracker), so it must be a real, non-empty id. + assert body["preprocessing_job_id"] + + session_id = body["id"] + client.delete(f"/api/v1/model-session/{session_id}") + + +def test_create_model_session_with_preprocessing_requires_input_column_refs( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "missing-refs-session", + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + }, + ) + assert response.status_code == 422, response.text + + +def test_validate_columns_accepts_a_group_ref_matching_the_task_type( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": ["SepalWidthCm"], + "outputs_columns": ["Species"], + "input_refs": [ + {"kind": "raw", "name": "SepalWidthCm"}, + {"kind": "group", "step": 0}, + ], + "converter_output_types": {"0": "Integer"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "valid" + + +def test_validate_columns_rejects_a_group_ref_with_an_incompatible_type( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": [], + "outputs_columns": ["Species"], + "input_refs": [{"kind": "group", "step": 0}], + "converter_output_types": {"0": "Text"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "invalid" + + def test_bulk_delete_model_sessions(client: TestClient, dataset_1: Dataset) -> None: created_ids = [] for name in ["bulk_delete_session_1", "bulk_delete_session_2"]: diff --git a/tests/back/migrations/__init__.py b/tests/back/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/back/migrations/test_add_preprocessing_to_model_session.py b/tests/back/migrations/test_add_preprocessing_to_model_session.py new file mode 100644 index 000000000..6dd1ddf2b --- /dev/null +++ b/tests/back/migrations/test_add_preprocessing_to_model_session.py @@ -0,0 +1,30 @@ +import sqlalchemy as sa +from alembic import command +from alembic.config import Config + + +def test_migration_adds_preprocessing_columns(tmp_path): + db_path = tmp_path / "test.db" + alembic_cfg = Config("alembic.ini") + alembic_cfg.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}") + + command.upgrade(alembic_cfg, "head") + + engine = sa.create_engine(f"sqlite:///{db_path}") + inspector = sa.inspect(engine) + columns = {c["name"] for c in inspector.get_columns("model_session")} + + assert { + "preprocessing", + "input_column_refs", + "preprocessing_status", + "preprocessing_error", + "preprocessing_artifacts_path", + "preprocessing_job_id", + } <= columns + + command.downgrade(alembic_cfg, "-1") + columns_after_downgrade = { + c["name"] for c in sa.inspect(engine).get_columns("model_session") + } + assert "preprocessing" not in columns_after_downgrade diff --git a/tests/back/test_config_builder.py b/tests/back/test_config_builder.py new file mode 100644 index 000000000..c67411003 --- /dev/null +++ b/tests/back/test_config_builder.py @@ -0,0 +1,6 @@ +from DashAI.back.dependencies.config_builder import build_config_dict + + +def test_preprocessing_path_is_under_local_path(tmp_path): + config = build_config_dict(local_path=tmp_path, logging_level="ERROR") + assert config["PREPROCESSING_PATH"] == tmp_path / "preprocessing" From a3b5e5e1fad843155d0693e1e64f9def094352d0 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 22:15:27 -0300 Subject: [PATCH 05/39] feat: apply persisted preprocessing to training, prediction and explanation Training reuses the SessionPreprocessor already fitted by PreprocessingJob instead of ever re-fitting on new data. Prediction and explanation apply the final fitted preprocessor to raw/manual input before running. Run creation is blocked with a clear error while a session's preprocessing is still pending or failed. --- DashAI/back/api/api_v1/endpoints/runs.py | 13 ++ DashAI/back/job/explainer_job.py | 23 +++ DashAI/back/job/model_job.py | 140 ++++++++++--- DashAI/back/job/predict_job.py | 8 + .../back/api/test_explainer_preprocessing.py | 186 ++++++++++++++++++ .../back/api/test_model_job_preprocessing.py | 178 +++++++++++++++++ tests/back/api/test_predict_preprocessing.py | 107 ++++++++++ .../back/api/test_run_preprocessing_guard.py | 145 ++++++++++++++ 8 files changed, 772 insertions(+), 28 deletions(-) create mode 100644 tests/back/api/test_explainer_preprocessing.py create mode 100644 tests/back/api/test_model_job_preprocessing.py create mode 100644 tests/back/api/test_predict_preprocessing.py create mode 100644 tests/back/api/test_run_preprocessing_guard.py diff --git a/DashAI/back/api/api_v1/endpoints/runs.py b/DashAI/back/api/api_v1/endpoints/runs.py index 18c54ae04..ddb6b46d1 100644 --- a/DashAI/back/api/api_v1/endpoints/runs.py +++ b/DashAI/back/api/api_v1/endpoints/runs.py @@ -294,6 +294,19 @@ async def upload_run( status_code=status.HTTP_404_NOT_FOUND, detail="Model session not found", ) + if model_session.preprocessing_status == "pending": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="This session's preprocessing has not finished yet.", + ) + if model_session.preprocessing_status == "failed": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This session's preprocessing failed: " + f"{model_session.preprocessing_error}" + ), + ) # REQUIRES_DOWNLOAD is the static contract; the download state is # reconciled against the filesystem so a model downloaded after # startup (in the worker process) is recognised without a restart. diff --git a/DashAI/back/job/explainer_job.py b/DashAI/back/job/explainer_job.py index 2033353e1..8a9fb64dd 100644 --- a/DashAI/back/job/explainer_job.py +++ b/DashAI/back/job/explainer_job.py @@ -186,6 +186,7 @@ def _generate_local_explanation( splits: Dict[str, Any], task: BaseTask, same_dataset: bool, + preprocessor: Any = None, ) -> None: import json import os @@ -242,12 +243,20 @@ def _generate_local_explanation( manual_input_data, f"{instance.file_path}/dataset", ) + if preprocessor is not None: + prepared_instance = preprocessor.transform_dataset( + prepared_instance + ) # Manual input carries only the input columns (no target), so # keep just those instead of the standard input/output split. # select_columns returns a DashAIDataset (same shape the # split path produces), which is what the explainers expect. X = prepared_instance.select_columns(self.input_columns) else: + if preprocessor is not None: + loaded_instance = preprocessor.transform_dataset( + loaded_instance + ) prepared_instance = task.prepare_for_task( loaded_instance, input_columns=self.input_columns, @@ -445,6 +454,16 @@ def run( self.input_columns = model_session.input_columns self.output_columns = model_session.output_columns + preprocessor = None + if model_session.preprocessing and model_session.preprocessing.get( + "steps" + ): + from DashAI.back.preprocessing.session_preprocessor import ( + load_final_preprocessor, + ) + + preprocessor = load_final_preprocessor(model_session) + try: run_model_class = component_registry[run.model_name]["class"] except Exception as e: @@ -527,6 +546,9 @@ def run( log.exception(e) raise JobError(str(e)) from e try: + if preprocessor is not None: + loaded_dataset = preprocessor.transform_dataset(loaded_dataset) + loaded_dataset = split_dataset( loaded_dataset, train_indexes=train_idx, @@ -596,6 +618,7 @@ def run( splits=splits, task=task, same_dataset=same_dataset, + preprocessor=preprocessor, ) else: raise JobError(f"{explainer_scope} is an invalid explainer type") diff --git a/DashAI/back/job/model_job.py b/DashAI/back/job/model_job.py index 7ae8528c7..5db3fe021 100644 --- a/DashAI/back/job/model_job.py +++ b/DashAI/back/job/model_job.py @@ -25,6 +25,62 @@ log = logging.getLogger(__name__) +def apply_persisted_preprocessing(model_session: ModelSession, x, y): + """Transform already-split fold data using the SessionPreprocessor that + PreprocessingJob fit on this session's training data, then narrow to the + resolved concrete input columns. y is returned unchanged: output columns + are always raw in v1, so the splitter already narrowed y correctly. + + Parameters + ---------- + model_session : ModelSession + The session whose preprocessing_artifacts_path holds one fitted + SessionPreprocessor per fold (plus a "final" one), persisted by + PreprocessingJob. + x : list of DatasetDict | DatasetDict + The splitter's output for the input side: a list of per-fold dicts + for Cross-Validation, or a single dict for Holdout. + y : list of DatasetDict | DatasetDict + The splitter's output for the output side, already narrowed to + model_session.output_columns. Returned unchanged. + + Returns + ------- + tuple + (x, y) with x's datasets transformed and narrowed to the resolved + input columns for each fold/holdout split. + """ + import os + import pickle + + from DashAI.back.preprocessing.column_ref import parse_column_refs, resolve_refs + + input_refs = parse_column_refs(model_session.input_column_refs or []) + is_cv = isinstance(x, list) + x_folds = x if is_cv else [x] + total_folds = len(x_folds) - 1 if is_cv else 0 + fold_names = [f"fold_{i}" for i in range(total_folds)] + ["final"] + + new_x = [] + for split_dict, fold_name in zip(x_folds, fold_names, strict=True): + artifact_path = os.path.join( + model_session.preprocessing_artifacts_path, f"{fold_name}.pkl" + ) + with open(artifact_path, "rb") as f: + preprocessor = pickle.load(f) + + transformed = preprocessor.transform_only(split_dict) + resolved_input_columns = resolve_refs(input_refs, preprocessor.resolved_columns) + + fold_x = { + split_name: dataset.select_columns(resolved_input_columns) + for split_name, dataset in transformed.items() + } + new_x.append(fold_x) + + return (new_x, y) if is_cv else (new_x[0], y) + + class ModelJob(BaseJob): """ModelJob class to run the model training.""" @@ -134,6 +190,12 @@ def run( # save the obtained splits into the database run.split_indexes = json.dumps(splits) + + model_session = preparation_results["model_session"] + if model_session.preprocessing and model_session.preprocessing.get( + "steps" + ): + x, y = apply_persisted_preprocessing(model_session, x, y) except Exception as e: log.exception(e) raise JobError( @@ -301,16 +363,56 @@ def _prepare_dataset_and_components( ), ) from e + has_preprocessing = bool( + model_session.preprocessing and model_session.preprocessing.get("steps") + ) + try: - # Prepare dataset for the task and get number of labels of the task - prepared_dataset = task.prepare_for_task( - dataset=loaded_dataset, - input_columns=model_session.input_columns, - output_columns=model_session.output_columns, - ) - n_labels = task.num_labels( - prepared_dataset, model_session.output_columns[0] - ) + if has_preprocessing: + from DashAI.back.preprocessing.column_ref import parse_column_refs + + input_refs = parse_column_refs(model_session.input_column_refs or []) + raw_input_names = [r.name for r in input_refs if r.kind == "raw"] + # Group-produced columns do not exist in loaded_dataset yet: + # only the raw subset can go through prepare_for_task before + # the converters run per fold (see apply_persisted_preprocessing, + # called from run() after splitting). + prepared_dataset = task.prepare_for_task( + dataset=loaded_dataset, + input_columns=raw_input_names, + output_columns=model_session.output_columns, + ) + n_labels = task.num_labels( + prepared_dataset, model_session.output_columns[0] + ) + # X keeps every raw column (not just the input ones): the + # converters may need columns that are not themselves final + # inputs. Y is safe to narrow now because v1 requires every + # output ColumnRef to be raw. + X = prepared_dataset + Y = prepared_dataset.select_columns(model_session.output_columns) + else: + # Prepare dataset for the task and get number of labels of the task + prepared_dataset = task.prepare_for_task( + dataset=loaded_dataset, + input_columns=model_session.input_columns, + output_columns=model_session.output_columns, + ) + n_labels = task.num_labels( + prepared_dataset, model_session.output_columns[0] + ) + # Divide the dataset into two datasets: + # one with the input columns and another with the output column. + # This reads the prepared dataset rather than the loaded one: a + # task may reorder or otherwise adjust the rows, and forecasting + # does, sorting them by date so the temporal splitter carves real + # periods of time. Selecting from the loaded dataset would drop + # that work on the floor. + X, Y = select_columns( + prepared_dataset, + model_session.input_columns, + model_session.output_columns, + ) except Exception as e: log.exception(e) raise JobError( @@ -318,25 +420,6 @@ def _prepare_dataset_and_components( for Task {model_session.task_name}""", ) from e - try: - # Divide the dataset into two datasets: - # one with the input columns and another with the output column. - # This reads the prepared dataset rather than the loaded one: a - # task may reorder or otherwise adjust the rows, and forecasting - # does, sorting them by date so the temporal splitter carves real - # periods of time. Selecting from the loaded dataset would drop - # that work on the floor. - X, Y = select_columns( - prepared_dataset, - model_session.input_columns, - model_session.output_columns, - ) - except Exception as e: - log.exception(e) - raise JobError( - f"Error selecting input and output columns from dataset {dataset.id}", - ) from e - try: # Get splits data from model session splits_data = json.loads(model_session.splits) @@ -465,4 +548,5 @@ def _prepare_dataset_and_components( "Y": Y, "splitter": splitter, "evaluation_strategy": evaluation_strategy, + "model_session": model_session, } diff --git a/DashAI/back/job/predict_job.py b/DashAI/back/job/predict_job.py index f509c101d..085e37537 100644 --- a/DashAI/back/job/predict_job.py +++ b/DashAI/back/job/predict_job.py @@ -53,6 +53,14 @@ def _run_prediction_pipeline( """ import numpy as np + if model_session.preprocessing and model_session.preprocessing.get("steps"): + from DashAI.back.preprocessing.session_preprocessor import ( + load_final_preprocessor, + ) + + preprocessor = load_final_preprocessor(model_session) + loaded_dataset = preprocessor.transform_dataset(loaded_dataset) + prepared_dataset = loaded_dataset.select_columns(model_session.input_columns) y_pred_proba = np.array(trained_model.predict(prepared_dataset)) y_pred = task.process_predictions( diff --git a/tests/back/api/test_explainer_preprocessing.py b/tests/back/api/test_explainer_preprocessing.py new file mode 100644 index 000000000..5b36f031e --- /dev/null +++ b/tests/back/api/test_explainer_preprocessing.py @@ -0,0 +1,186 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import ( + Dataset, + GlobalExplainer, + LocalExplainer, +) + + +def _create_and_train_session_with_binarizer(client: TestClient, dataset_id: int): + session_response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_id, + "task_name": "TabularClassificationTask", + "name": "explainer-preprocessing-session", + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert session_response.status_code == 201, session_response.text + model_session_id = session_response.json()["id"] + + run_response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "ExplainerPreprocessingRun", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Training for explainer preprocessing test", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + job_response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + return model_session_id, run_id + + +def test_local_explainer_manual_input_applies_the_persisted_preprocessor( + client: TestClient, dataset_1: Dataset +): + """A manual-mode local explanation, like manual prediction, only ever + carries the raw feature the converter scoped on. The explainer's own + fit dataset (the model_session's training data) and the explained + instance must both be transformed by the persisted Binarizer before + reaching the model's feature space. + """ + model_session_id, run_id = _create_and_train_session_with_binarizer( + client, dataset_1.id + ) + + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + local_explainer = LocalExplainer( + name="preprocessing_local_explainer", + run_id=run_id, + explainer_name="KernelShap", + dataset_id=dataset_1.id, + scope={"mode": "manual"}, + parameters={}, + fit_parameters={ + "sample_background_data": False, + "background_fraction": 0.5, + "sampling_method": "shuffle", + }, + ) + db.add(local_explainer) + db.commit() + db.refresh(local_explainer) + explainer_id = local_explainer.id + + job_response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + { + "explainer_id": explainer_id, + "explainer_scope": "local", + "manual_input_data": [{"SepalLengthCm": 3.0}], + } + ), + }, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + client.delete(f"/api/v1/model-session/{model_session_id}") + + +def test_global_explainer_applies_the_persisted_preprocessor( + client: TestClient, dataset_1: Dataset +): + """The global explainer reuses the same background-data preparation as + the local one (data_x/data_y computed once in ExplainerJob.run before + branching on scope), so it inherits the same preprocessor fix without + needing its own — this test exists to confirm that, not just assume it. + """ + model_session_id, run_id = _create_and_train_session_with_binarizer( + client, dataset_1.id + ) + + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + global_explainer = GlobalExplainer( + name="preprocessing_global_explainer", + run_id=run_id, + explainer_name="PermutationFeatureImportance", + parameters={"scoring": "accuracy", "n_repeats": 5}, + ) + db.add(global_explainer) + db.commit() + db.refresh(global_explainer) + explainer_id = global_explainer.id + + job_response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + { + "explainer_id": explainer_id, + "explainer_scope": "global", + } + ), + }, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + client.delete(f"/api/v1/model-session/{model_session_id}") diff --git a/tests/back/api/test_model_job_preprocessing.py b/tests/back/api/test_model_job_preprocessing.py new file mode 100644 index 000000000..c9c22efe1 --- /dev/null +++ b/tests/back/api/test_model_job_preprocessing.py @@ -0,0 +1,178 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import Dataset + + +def _create_session_with_binarizer(client: TestClient, dataset_id: int, name: str): + return client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_id, + "task_name": "TabularClassificationTask", + "name": name, + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + + +def test_model_job_trains_using_a_converter_produced_group_column( + client: TestClient, dataset_1: Dataset +): + session_response = _create_session_with_binarizer( + client, dataset_1.id, "preprocessing-training-session" + ) + assert session_response.status_code == 201, session_response.text + session_body = session_response.json() + assert session_body["preprocessing_status"] == "ready" + model_session_id = session_body["id"] + + run_response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "PreprocessingRun", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Training with a group column from a converter", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + job_response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + run_after = client.get(f"/api/v1/run/{run_id}").json() + assert run_after["status"] == 3 # RunStatus.FINISHED + assert run_after["run_path"] is not None + + client.delete(f"/api/v1/run/{run_id}") + client.delete(f"/api/v1/model-session/{model_session_id}") + + +def test_model_job_trains_with_cross_validation_and_per_fold_preprocessing( + client: TestClient, dataset_1: Dataset +): + response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_1.id, + "task_name": "TabularClassificationTask", + "name": "preprocessing-cv-session", + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "CrossValidationEvaluationStrategy", + "splits": json.dumps( + { + "n_splits": 3, + "test_size": 0.0, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + "splitter_name": "KFoldSplitter", + } + ), + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert response.status_code == 201, response.text + session_body = response.json() + assert session_body["preprocessing_status"] == "ready" + model_session_id = session_body["id"] + + run_response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "PreprocessingCVRun", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "CV training with per-fold preprocessing", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + job_response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + run_after = client.get(f"/api/v1/run/{run_id}").json() + assert run_after["status"] == 3 # RunStatus.FINISHED + + client.delete(f"/api/v1/run/{run_id}") + client.delete(f"/api/v1/model-session/{model_session_id}") diff --git a/tests/back/api/test_predict_preprocessing.py b/tests/back/api/test_predict_preprocessing.py new file mode 100644 index 000000000..ebd9a9d04 --- /dev/null +++ b/tests/back/api/test_predict_preprocessing.py @@ -0,0 +1,107 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import Dataset + + +def _create_and_train_session_with_binarizer(client: TestClient, dataset_id: int): + session_response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_id, + "task_name": "TabularClassificationTask", + "name": "predict-preprocessing-session", + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert session_response.status_code == 201, session_response.text + model_session_id = session_response.json()["id"] + + run_response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "PredictPreprocessingRun", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Training for prediction preprocessing test", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + job_response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + return model_session_id, run_id + + +def test_manual_prediction_applies_the_persisted_preprocessor( + client: TestClient, dataset_1: Dataset +): + """A manual prediction row carries only the raw feature the converter + scoped on ("SepalLengthCm"), never the converter-produced column + ("bin_SepalLengthCm") the model was actually trained on. Predicting + must apply the persisted fitted Binarizer to the raw row before handing + it to the model, without ever re-fitting on it. + """ + model_session_id, run_id = _create_and_train_session_with_binarizer( + client, dataset_1.id + ) + + response = client.post( + "/api/v1/predict/preview", + data={ + "run_id": str(run_id), + "manual_input_data": json.dumps([{"SepalLengthCm": 3.0}]), + }, + ) + assert response.status_code == 200, response.text + body = response.json() + assert "columns" in body + assert len(body["rows"]) == 1 + + client.delete(f"/api/v1/model-session/{model_session_id}") diff --git a/tests/back/api/test_run_preprocessing_guard.py b/tests/back/api/test_run_preprocessing_guard.py new file mode 100644 index 000000000..4ea339e22 --- /dev/null +++ b/tests/back/api/test_run_preprocessing_guard.py @@ -0,0 +1,145 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import Dataset, ModelSession + + +def test_run_creation_is_rejected_while_preprocessing_is_pending( + client: TestClient, dataset_1: Dataset +): + session_response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_1.id, + "task_name": "TabularClassificationTask", + "name": "pending-guard-session", + "input_columns": ["SepalLengthCm"], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + }, + ) + assert session_response.status_code == 201, session_response.text + model_session_id = session_response.json()["id"] + + # The test client runs the job queue in immediate mode, so a session + # never actually stays "pending" long enough to observe over the API. + # Force it here to exercise the guard the way production would hit it + # while PreprocessingJob is still running asynchronously. + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + model_session = db.get(ModelSession, model_session_id) + model_session.preprocessing_status = "pending" + db.commit() + + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "ShouldBeBlocked", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Should be rejected", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 409, response.text + + client.delete(f"/api/v1/model-session/{model_session_id}") + + +def test_run_creation_surfaces_the_preprocessing_error_when_failed( + client: TestClient, dataset_1: Dataset +): + session_response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_1.id, + "task_name": "TabularClassificationTask", + "name": "failed-guard-session", + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + "preprocessing": [ + { + # Binarizer only accepts numeric columns; scoping it on the + # categorical target makes PreprocessingJob's fit fail. + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "Species"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert session_response.status_code == 201, session_response.text + body = session_response.json() + assert body["preprocessing_status"] == "failed" + model_session_id = body["id"] + + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "ShouldBeBlocked", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Should be rejected", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 409, response.text + assert "preprocessing" in response.json()["detail"].lower() + + client.delete(f"/api/v1/model-session/{model_session_id}") From 7777eb67a53e79b291d7c13f240b32894b9c4973 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 22:15:45 -0300 Subject: [PATCH 06/39] fix: exclude a converter's untouched passthrough column from its output group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BagOfWordsConverter (and any converter with the same shape, e.g. Binarizer) keeps its scope column unchanged and appends new derived columns instead of replacing it. SessionPreprocessor was recording every column the converter's transform() returned as the step's output group, so the untouched original column leaked into it too — selecting only "this converter's output" as a session's input still smuggled in the raw column, which fails task validation when it isn't an allowed input type (e.g. Text). --- .../preprocessing/session_preprocessor.py | 84 +++++++- .../test_session_preprocessor.py | 199 ++++++++++++++++++ 2 files changed, 275 insertions(+), 8 deletions(-) diff --git a/DashAI/back/preprocessing/session_preprocessor.py b/DashAI/back/preprocessing/session_preprocessor.py index c6e93cbd8..8dddf0224 100644 --- a/DashAI/back/preprocessing/session_preprocessor.py +++ b/DashAI/back/preprocessing/session_preprocessor.py @@ -38,6 +38,38 @@ def _instantiate(self, step) -> Any: converter_class = self.component_registry[step.converter]["class"] return converter_class(**step.params) + @staticmethod + def _transform_split(converter, dataset, scope_names, train_transformed): + """Transform one split's scoped columns, without crashing on 0 rows. + + A "test" (or similar) partition can legitimately have 0 rows — e.g. + a session that reserved nothing for the final refit — and several + sklearn transformers raise on an empty array. Since train_transformed + (computed first) has the same columns any non-empty split would + produce, an empty split reuses that shape with 0 rows instead of + calling the converter at all. + """ + scoped = dataset.select_columns(scope_names) + if scoped.num_rows == 0 and train_transformed is not None: + return train_transformed.select([]) + return converter.transform(scoped) + + def __getstate__(self): + """Exclude component_registry from pickling. + + The registry is only needed to instantiate converters during + fit_transform; a fitted preprocessor is persisted precisely so that + step never runs again. The registry itself is not picklable (it + holds RelationshipManager lambdas), so it must never travel with the + pickled object. + """ + state = self.__dict__.copy() + state["component_registry"] = None + return state + + def __setstate__(self, state): + self.__dict__.update(state) + def fit_transform( self, split: Dict[str, "DashAIDataset"] ) -> Tuple[Dict[str, "DashAIDataset"], Dict[int, List[str]]]: @@ -68,13 +100,33 @@ def fit_transform( train_scope = current["train"].select_columns(scope_names) converter = converter.fit(train_scope) - transformed_by_split = {} + train_transformed = converter.transform(train_scope) + transformed_by_split = {"train": train_transformed} for split_name, dataset in current.items(): - scoped = dataset.select_columns(scope_names) - transformed_by_split[split_name] = converter.transform(scoped) - - self.resolved_columns[index] = list( - transformed_by_split["train"].column_names + if split_name == "train": + continue + transformed_by_split[split_name] = self._transform_split( + converter, dataset, scope_names, train_transformed + ) + + # A converter that only rewrites its scope columns in place (e.g. a + # scaler: "age" in, scaled "age" out) has no other way to expose + # its result, so the scope names ARE the group. But a converter + # like Bag-of-Words additionally keeps its scope column verbatim + # alongside brand-new derived columns (see BagOfWordsConverter. + # transform's docstring: "the source text column is preserved + # unchanged") — for those, the untouched scope column is a + # passthrough, not this step's own output, so it must not leak + # into the group a later step or the wizard's input selection can + # reference (it would still carry the pre-conversion dtype, e.g. + # Text, which is never valid as a resolved input column). + new_columns = [ + name + for name in train_transformed.column_names + if name not in scope_names + ] + self.resolved_columns[index] = ( + new_columns if new_columns else list(train_transformed.column_names) ) new_current = {} @@ -110,10 +162,26 @@ def transform_only( scope_names = resolve_refs( self.sequence.steps[index].scope, self.resolved_columns ) + + train_transformed = None + if "train" in current: + train_transformed = converter.transform( + current["train"].select_columns(scope_names) + ) + + transformed_by_split = {} + if train_transformed is not None: + transformed_by_split["train"] = train_transformed + for split_name, dataset in current.items(): + if split_name == "train": + continue + transformed_by_split[split_name] = self._transform_split( + converter, dataset, scope_names, train_transformed + ) + new_current = {} for split_name, dataset in current.items(): - scoped = dataset.select_columns(scope_names) - transformed = converter.transform(scoped) + transformed = transformed_by_split[split_name] if type(converter).CHANGES_ROW_COUNT: new_current[split_name] = transformed else: diff --git a/tests/back/preprocessing/test_session_preprocessor.py b/tests/back/preprocessing/test_session_preprocessor.py index 775432311..6fbdec08c 100644 --- a/tests/back/preprocessing/test_session_preprocessor.py +++ b/tests/back/preprocessing/test_session_preprocessor.py @@ -78,6 +78,32 @@ def transform(self, x, y=None): return DashAIDataset(table, types=types) +class _FakeAdditiveConverter(BaseConverter): + """Mirrors BagOfWordsConverter's real behavior: keeps its scope column + verbatim and appends brand-new derived columns alongside it, instead of + replacing the scope column or dropping it.""" + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + frame = x.to_pandas() + column = x.column_names[0] + frame["derived"] = frame[column] * 10 + output_type = self.get_output_type() + types = {**x.types, "derived": output_type} + return to_dashai_dataset(frame, types=types) + + class _FakeRegistry: def __init__(self, classes): self._classes = classes @@ -135,6 +161,27 @@ def test_variable_output_step_records_whatever_columns_it_produced(): assert set(transformed["train"].column_names) == {"vocab_1", "vocab_2"} +def test_additive_step_that_keeps_its_scope_column_excludes_it_from_the_group(): + registry = _FakeRegistry({"Additive": _FakeAdditiveConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Additive", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + split = _split(train_ages=[1, 2, 3]) + transformed, resolved = preprocessor.fit_transform(split) + + # "age" is carried through unchanged (a passthrough, like BagOfWords + # keeping its source text column) — it must not be part of the step's + # own group, only the genuinely new "derived" column is. + assert resolved == {0: ["derived"]} + assert set(transformed["train"].column_names) == {"age", "derived"} + + def test_chained_step_can_reference_the_previous_steps_group(): registry = _FakeRegistry( {"FakeVocab": _FakeVocabConverter, "Doubler": _DoublingConverter} @@ -159,3 +206,155 @@ def test_chained_step_can_reference_the_previous_steps_group(): # its scope, so step 1 produces the same column set step 0 did. assert set(resolved[1]) == set(resolved[0]) assert len(resolved[1]) == 2 + + +def test_transform_only_applies_an_already_fitted_sequence_without_refitting(): + registry = _FakeRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform(_split(train_ages=[1, 2, 3])) + + new_split = _split(train_ages=[100]) + transformed = preprocessor.transform_only(new_split) + + assert transformed["train"].to_pandas()["age"].tolist() == [200] + + +def test_transform_dataset_is_a_single_dataset_convenience_wrapper(): + registry = _FakeRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform(_split(train_ages=[1, 2, 3])) + + single = _dataset({"age": [5]}, {"age": {"type": "Integer", "dtype": "int64"}}) + result = preprocessor.transform_dataset(single) + + assert result.to_pandas()["age"].tolist() == [10] + + +def test_a_pickled_and_restored_preprocessor_still_transforms_correctly(): + import pickle + + registry = _FakeRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform(_split(train_ages=[1, 2, 3])) + + restored = pickle.loads(pickle.dumps(preprocessor)) + single = _dataset({"age": [5]}, {"age": {"type": "Integer", "dtype": "int64"}}) + + assert restored.transform_dataset(single).to_pandas()["age"].tolist() == [10] + + +class _CrashesOnEmptyConverter(_DoublingConverter): + """Mirrors sklearn transformers that reject a 0-row array, e.g. Binarizer.""" + + def transform(self, x, y=None): + if x.num_rows == 0: + raise ValueError("Found array with 0 sample(s)") + return super().transform(x, y) + + +def test_fit_transform_does_not_crash_on_an_empty_split(): + registry = _FakeRegistry({"CrashesOnEmpty": _CrashesOnEmptyConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="CrashesOnEmpty", + params={}, + scope=[RawColumnRef(name="age")], + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + train = _dataset({"age": [1, 2, 3]}, {"age": {"type": "Integer", "dtype": "int64"}}) + empty_test = _dataset({"age": []}, {"age": {"type": "Integer", "dtype": "int64"}}) + split = {"train": train, "test": empty_test} + + transformed, resolved = preprocessor.fit_transform(split) + + assert transformed["train"].to_pandas()["age"].tolist() == [2, 4, 6] + assert transformed["test"].num_rows == 0 + assert transformed["test"].column_names == transformed["train"].column_names + assert resolved == {0: ["age"]} + + +def test_transform_only_does_not_crash_on_an_empty_split(): + registry = _FakeRegistry({"CrashesOnEmpty": _CrashesOnEmptyConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="CrashesOnEmpty", + params={}, + scope=[RawColumnRef(name="age")], + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform( + { + "train": _dataset( + {"age": [1, 2, 3]}, {"age": {"type": "Integer", "dtype": "int64"}} + ) + } + ) + + empty_test = _dataset({"age": []}, {"age": {"type": "Integer", "dtype": "int64"}}) + transformed = preprocessor.transform_only( + { + "train": _dataset( + {"age": [5]}, {"age": {"type": "Integer", "dtype": "int64"}} + ), + "test": empty_test, + } + ) + + assert transformed["train"].to_pandas()["age"].tolist() == [10] + assert transformed["test"].num_rows == 0 + + +def test_a_preprocessor_pickles_even_when_its_registry_cannot_be_pickled(): + import pickle + + class _UnpicklableRegistry(_FakeRegistry): + def __init__(self, classes): + super().__init__(classes) + # Mirrors ComponentRegistry, which is not picklable because it + # holds RelationshipManager lambdas. + self._unpicklable = lambda: None + + registry = _UnpicklableRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform(_split(train_ages=[1, 2, 3])) + + restored = pickle.loads(pickle.dumps(preprocessor)) + single = _dataset({"age": [5]}, {"age": {"type": "Integer", "dtype": "int64"}}) + + assert restored.transform_dataset(single).to_pandas()["age"].tolist() == [10] + assert restored.component_registry is None From 6c88e194daeb4b37a297dd740ecc688657bc1939 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 22:15:51 -0300 Subject: [PATCH 07/39] feat: expose a converter's concrete output dtype in its metadata output_type already reported the semantic type name (e.g. "Integer") from a default-constructed instance; output_dtype adds the concrete storage dtype (e.g. "int64") the same way, so a converter's not-yet-materialized output group can show a real dtype instead of falling back to unknown. --- DashAI/back/converters/base_converter.py | 24 +++++++++ .../test_base_converter_metadata.py | 49 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/DashAI/back/converters/base_converter.py b/DashAI/back/converters/base_converter.py index 8b636356b..7805d8d0b 100644 --- a/DashAI/back/converters/base_converter.py +++ b/DashAI/back/converters/base_converter.py @@ -108,6 +108,30 @@ def get_metadata(cls) -> Dict[str, Any]: # Drop restricted_dtypes (no converter uses it; it is always []) meta.pop("restricted_dtypes", None) + # A representative output type, so the Models-module wizard can show + # "this converter's group is typed X" before any real fit exists. + # Not every converter can be instantiated with no arguments (some + # require constructor params with no default), so this is + # best-effort: None means "unknown until configured". + try: + output_type = cls().get_output_type() + meta["output_type"] = ( + output_type.display_name() + if output_type is not None and hasattr(output_type, "display_name") + else None + ) + # The concrete storage dtype (e.g. "int64"), so a group column can + # show one instead of "unknown" before any real fit exists — same + # best-effort default-constructed instance as output_type above. + meta["output_dtype"] = ( + output_type.to_string().get("dtype") + if output_type is not None and hasattr(output_type, "to_string") + else None + ) + except Exception: + meta["output_type"] = None + meta["output_dtype"] = None + return meta @abstractmethod diff --git a/tests/back/converters/test_base_converter_metadata.py b/tests/back/converters/test_base_converter_metadata.py index 3ac5f1307..58ed056e7 100644 --- a/tests/back/converters/test_base_converter_metadata.py +++ b/tests/back/converters/test_base_converter_metadata.py @@ -137,3 +137,52 @@ def transform(self, x, y=None): meta = _CatTextConverter.get_metadata() assert meta["allowed_types"] == ["Categorical", "Text"] assert meta["allowed_dtypes"] == ["string"] + + +def test_get_metadata_reports_a_representative_output_type_when_declared(): + class _WithOutput(BaseConverter): + SCHEMA = None + metadata = {"allowed_types": [Integer]} + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + meta = _WithOutput.get_metadata() + assert meta["output_type"] == "Integer" + assert meta["output_dtype"] == "int64" + + +def test_get_metadata_output_type_is_none_when_get_output_type_returns_none(): + meta = _FloatIntConverter.get_metadata() # returns None today, by design + assert meta["output_type"] is None + assert meta["output_dtype"] is None + + +def test_get_metadata_output_type_is_none_when_the_converter_cannot_be_built(): + class _RequiresArgConverter(BaseConverter): + SCHEMA = None + metadata = {"allowed_types": [Integer]} + + def __init__(self, required_param): + self.required_param = required_param + + def get_output_type(self, column_name=None): + return None + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + meta = _RequiresArgConverter.get_metadata() + assert meta["output_type"] is None + assert meta["output_dtype"] is None From 74573be2e49a5f05d4103a909105a2f0d3a6419f Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 22:16:32 -0300 Subject: [PATCH 08/39] feat: build the session preprocessing wizard, mirroring Notebooks' converter picker A Models-module session can now optionally configure a sequence of converters as part of creation. The wizard reuses Notebooks' own tool picker (search, list/grid, drag-and-drop) and column selector so the scope/output-group UX matches exactly, and represents a converter's not-yet-materialized output group as a synthetic column key so it can be picked as an input before any real fit exists (and chained into a later converter's own scope). --- DashAI/front/src/api/modelSession.test.ts | 68 +++ DashAI/front/src/api/modelSession.ts | 14 +- .../components/models/CreateSessionSteps.jsx | 187 ++++++- .../modelSession/AppliedConvertersView.jsx | 473 ++++++++++++++++++ .../AppliedConvertersView.test.jsx | 174 +++++++ .../modelSession/DivideDatasetColumns.jsx | 38 +- .../DivideDatasetColumns.test.jsx | 70 +++ .../FormSessionConverterSection.jsx | 96 ++++ .../modelSession/PrepareDatasetStep.jsx | 460 +++-------------- .../models/modelSession/PreprocessingStep.jsx | 61 +++ .../modelSession/PreprocessingStep.test.jsx | 50 ++ .../ScopeStepSessionConverter.jsx | 108 ++++ .../models/modelSession/SelectColumnsStep.jsx | 343 +++++++++++++ .../SessionConvertersRightBar.jsx | 229 +++++++++ .../models/modelSession/sessionColumnRefs.js | 63 +++ .../modelSession/sessionColumnRefs.test.js | 59 +++ .../front/src/pages/models/ModelsContent.jsx | 86 ++-- DashAI/front/src/types/modelSession.ts | 25 + .../src/utils/i18n/locales/de/models.json | 1 + .../src/utils/i18n/locales/en/datasets.json | 3 + .../utils/i18n/locales/en/experiments.json | 1 + .../src/utils/i18n/locales/en/models.json | 10 + .../src/utils/i18n/locales/es/datasets.json | 3 + .../utils/i18n/locales/es/experiments.json | 1 + .../src/utils/i18n/locales/es/models.json | 10 + .../src/utils/i18n/locales/pt/models.json | 1 + .../src/utils/i18n/locales/zh/models.json | 1 + 27 files changed, 2168 insertions(+), 467 deletions(-) create mode 100644 DashAI/front/src/api/modelSession.test.ts create mode 100644 DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx create mode 100644 DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx create mode 100644 DashAI/front/src/components/models/modelSession/DivideDatasetColumns.test.jsx create mode 100644 DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx create mode 100644 DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx create mode 100644 DashAI/front/src/components/models/modelSession/PreprocessingStep.test.jsx create mode 100644 DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx create mode 100644 DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx create mode 100644 DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx create mode 100644 DashAI/front/src/components/models/modelSession/sessionColumnRefs.js create mode 100644 DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js diff --git a/DashAI/front/src/api/modelSession.test.ts b/DashAI/front/src/api/modelSession.test.ts new file mode 100644 index 000000000..6a13f142d --- /dev/null +++ b/DashAI/front/src/api/modelSession.test.ts @@ -0,0 +1,68 @@ +jest.mock("./api"); + +import api from "./api"; +import { createModelSession } from "./modelSession"; + +describe("createModelSession", () => { + beforeEach(() => { + jest.clearAllMocks(); + (api.post as jest.Mock).mockResolvedValue({ data: { id: "session-1" } }); + }); + + it("includes preprocessing and input_column_refs in the request body", async () => { + await createModelSession( + 1, + "TabularClassificationTask", + "session-1", + [], + ["label"], + [], + [], + [], + "holdout", + JSON.parse("{}"), + [ + { + converter: "Binarizer", + params: { threshold: 0.5 }, + scope: [{ kind: "raw", name: "age" }], + }, + ], + [{ kind: "group", step: 0 }], + ); + + expect(api.post).toHaveBeenCalledWith( + "/v1/model-session/", + expect.objectContaining({ + preprocessing: [ + { + converter: "Binarizer", + params: { threshold: 0.5 }, + scope: [{ kind: "raw", name: "age" }], + }, + ], + input_column_refs: [{ kind: "group", step: 0 }], + }), + ); + }); + + it("defaults preprocessing and input_column_refs to empty arrays", async () => { + await createModelSession( + 1, + "TabularClassificationTask", + "session-2", + ["age"], + ["label"], + [], + [], + [], + "holdout", + JSON.parse("{}"), + ); + + expect(api.post).toHaveBeenCalledWith( + "/v1/model-session/", + expect.objectContaining({ preprocessing: [], input_column_refs: [] }), + ); + }); +}); diff --git a/DashAI/front/src/api/modelSession.ts b/DashAI/front/src/api/modelSession.ts index bcdf0b89c..0c85e6f9e 100644 --- a/DashAI/front/src/api/modelSession.ts +++ b/DashAI/front/src/api/modelSession.ts @@ -1,5 +1,9 @@ import api from "./api"; -import type { IModelSession } from "../types/modelSession"; +import type { + IColumnRef, + IConverterStep, + IModelSession, +} from "../types/modelSession"; const endpointURL = "/v1/model-session"; @@ -26,6 +30,8 @@ export const createModelSession = async ( testMetrics: string[], evaluationStrategy: string, splitsValue: JSON, + preprocessing: IConverterStep[] = [], + inputColumnRefs: IColumnRef[] = [], ): Promise => { const data = { dataset_id: datasetId, @@ -38,6 +44,8 @@ export const createModelSession = async ( test_metrics: testMetrics, evaluation_strategy: evaluationStrategy, splits: splitsValue, + preprocessing: preprocessing, + input_column_refs: inputColumnRefs, }; const response = await api.post("/v1/model-session/", data); @@ -75,12 +83,16 @@ export const validateColumns = async ( datasetId: number, inputColumns: string[], outputColumns: string[], + inputRefs?: IColumnRef[], + converterOutputTypes?: Record, ): Promise => { const formData = { task_name: taskName, dataset_id: datasetId, inputs_columns: inputColumns, outputs_columns: outputColumns, + input_refs: inputRefs, + converter_output_types: converterOutputTypes, }; const response = await api.post( "/v1/model-session/validation", diff --git a/DashAI/front/src/components/models/CreateSessionSteps.jsx b/DashAI/front/src/components/models/CreateSessionSteps.jsx index 0d16c3008..05d5a0da9 100644 --- a/DashAI/front/src/components/models/CreateSessionSteps.jsx +++ b/DashAI/front/src/components/models/CreateSessionSteps.jsx @@ -1,14 +1,21 @@ import { useState, useMemo, useEffect, useRef } from "react"; import PropTypes from "prop-types"; -import { Box, Typography } from "@mui/material"; +import { Box, Typography, CircularProgress } from "@mui/material"; import { useSnackbar } from "notistack"; import { useFormik } from "formik"; import { useTourContext } from "../tour/TourProvider"; import SetNameAndDatasetStep from "./SetNameAndDatasetStep"; import PrepareDatasetStep from "./modelSession/PrepareDatasetStep"; +import PreprocessingStep from "./modelSession/PreprocessingStep"; +import SelectColumnsStep from "./modelSession/SelectColumnsStep"; import DatasetAutocomplete from "../notebooks/notebookCreation/DatasetAutocomplete"; import { createModelSession } from "../../api/modelSession"; +import { forceRefreshNow } from "../../utils/jobPoller"; import { getComponents } from "../../api/component"; +import { + getDatasetInfo as getDatasetInfoRequest, + getDatasetTypes as getDatasetTypesRequest, +} from "../../api/datasets"; import { generateSequentialName, getNextAvailableName, @@ -18,6 +25,10 @@ import { useModels } from "./ModelsContext"; import StepperNavigationFooter from "../shared/StepperNavigationFooter"; import { hasPartition } from "../../utils/splitsPayload"; +const STEP_PREPARE_DATASET = "prepareDataset"; +const STEP_PREPROCESSING = "preprocessing"; +const STEP_SELECT_COLUMNS = "selectColumns"; + function CreateSessionSteps({ backHome, selectedTask, @@ -54,9 +65,60 @@ function CreateSessionSteps({ evaluation_strategy: "", splits: {}, runs: [], + applyPreprocessing: false, + preprocessing: [], + input_column_refs: [], }); const [nextEnabled, setNextEnabled] = useState(false); + const [currentStep, setCurrentStep] = useState(STEP_PREPARE_DATASET); + + const [datasetInfo, setDatasetInfo] = useState({}); + const [datasetTypes, setDatasetTypes] = useState({}); + const [infoLoading, setInfoLoading] = useState(false); + + useEffect(() => { + if (!selectedDataset?.id) { + setDatasetInfo({}); + setDatasetTypes({}); + return; + } + let cancelled = false; + setInfoLoading(true); + (async () => { + try { + const [fetchedInfo, fetchedTypes] = await Promise.all([ + getDatasetInfoRequest(selectedDataset.id), + getDatasetTypesRequest(selectedDataset.id), + ]); + if (cancelled) return; + setDatasetInfo(fetchedInfo); + setDatasetTypes(fetchedTypes); + } catch (error) { + if (!cancelled) { + enqueueSnackbar(t("experiments:error.errorFetchingDatasetInfo")); + console.error("Error fetching dataset info:", error); + } + } finally { + if (!cancelled) setInfoLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [selectedDataset?.id]); + + const steps = useMemo( + () => [ + STEP_PREPARE_DATASET, + ...(newExp.applyPreprocessing ? [STEP_PREPROCESSING] : []), + STEP_SELECT_COLUMNS, + ], + [newExp.applyPreprocessing], + ); + const currentStepIndex = Math.max(0, steps.indexOf(currentStep)); + const isFirstStep = currentStepIndex === 0; + const isLastStep = currentStepIndex === steps.length - 1; const handleDatasetChange = (newDataset) => { setSelectedDataset(newDataset); @@ -66,6 +128,7 @@ function CreateSessionSteps({ dataset: newDataset, input_columns: [], output_columns: [], + input_column_refs: [], splits: {}, })); if ( @@ -144,10 +207,32 @@ function CreateSessionSteps({ return () => setSessionRightContent(null); }, [selectedDataset]); - const isNextEnabled = - formik.values.name.trim().length >= 4 && - selectedDataset !== null && - nextEnabled; + const isNextEnabled = isFirstStep + ? formik.values.name.trim().length >= 4 && + selectedDataset !== null && + nextEnabled + : nextEnabled; + + const goToStep = (step) => { + setNextEnabled(false); + setCurrentStep(step); + }; + + const handleFooterBack = () => { + if (isFirstStep) { + backHome(); + return; + } + goToStep(steps[currentStepIndex - 1]); + }; + + const handleFooterNext = () => { + if (isLastStep) { + formik.handleSubmit(); + return; + } + goToStep(steps[currentStepIndex + 1]); + }; const createSession = async (sessionName) => { try { @@ -182,6 +267,8 @@ function CreateSessionSteps({ hasTest ? allMetricNames : [], newExp.evaluation_strategy, JSON.stringify(newExp.splits), + newExp.preprocessing, + newExp.input_column_refs, ); } catch (createError) { if (createError?.response?.status === 409) { @@ -198,12 +285,21 @@ function CreateSessionSteps({ hasTest ? allMetricNames : [], newExp.evaluation_strategy, JSON.stringify(newExp.splits), + newExp.preprocessing, + newExp.input_column_refs, ); } else { throw createError; } } + // The session's PreprocessingJob (if any) is already enqueued by the + // time this response comes back — force an immediate poll so the job + // queue widget picks it up right away instead of waiting for its next + // 1s tick, which the job can easily finish before (see + // ConfigureAndUploadDatasetStep.jsx for the same pattern). + forceRefreshNow(); + enqueueSnackbar(t("models:message.sessionCreatedSuccess"), { variant: "success", }); @@ -226,6 +322,18 @@ function CreateSessionSteps({ } }; + const stepTitle = { + [STEP_PREPARE_DATASET]: t("models:label.prepareDataset"), + [STEP_PREPROCESSING]: t("models:label.preprocessingOptional"), + [STEP_SELECT_COLUMNS]: t("models:label.selectColumnsTitle"), + }[currentStep]; + + const stepSubtitle = { + [STEP_PREPARE_DATASET]: t("models:label.selectDatasetAndPrepare"), + [STEP_PREPROCESSING]: t("models:label.preprocessingOptionalDescription"), + [STEP_SELECT_COLUMNS]: t("models:label.selectColumnsDescription"), + }[currentStep]; + return ( - {t("models:label.prepareDataset")} + {stepTitle} - {t("models:label.selectDatasetAndPrepare")} + {stepSubtitle} @@ -255,30 +363,67 @@ function CreateSessionSteps({ gap: 4, }} > - - - {selectedDataset && ( - + + + {selectedDataset && ( + + )} + + )} + + {currentStep === STEP_PREPROCESSING && selectedDataset && ( + )} + + {currentStep === STEP_SELECT_COLUMNS && selectedDataset && ( + <> + {infoLoading ? ( + + + + ) : ( + + )} + + )} diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx new file mode 100644 index 000000000..280848a3a --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx @@ -0,0 +1,473 @@ +import React, { useEffect, useMemo, useState } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Paper, + Typography, + Chip, + IconButton, + Tooltip, +} from "@mui/material"; +import { + MaterialReactTable, + useMaterialReactTable, +} from "material-react-table"; +import DeleteIcon from "@mui/icons-material/Delete"; +import Transform from "@mui/icons-material/Transform"; +import { useTheme } from "@mui/material/styles"; +import { useTranslation } from "react-i18next"; +import { getColorByColumnType } from "../../../utils"; +import { getComponents } from "../../../api/component"; +import { useTableLocalization } from "../../../utils/useTableLocalization"; +import DeleteConfirmationModal from "../../threeSectionLayout/DeleteConfirmationModal"; +import ItemsToDeleteList from "../../notebooks/converter/ItemsToDeleteList"; +import { useExplorersAndConverters } from "../../notebooks/context/ExplorersAndConvertersContext"; +import { buildColumnKeysAndTypes, refToKey } from "./sessionColumnRefs"; + +function TypeChip({ type }) { + if (!type) return null; + return ( + getColorByColumnType(type, theme), + color: "#fff", + fontWeight: 600, + fontSize: "0.65rem", + height: "18px", + ml: 1, + }} + /> + ); +} + +TypeChip.propTypes = { type: PropTypes.string }; + +function RefChip({ refKey, label, columnTypes }) { + return ( + + {label} + + + } + /> + ); +} + +RefChip.propTypes = { + refKey: PropTypes.string.isRequired, + label: PropTypes.string.isRequired, + columnTypes: PropTypes.object.isRequired, +}; + +/** + * Mirrors the notebook's own ConverterParametersTable (same columns, same + * MaterialReactTable setup): a compact two-column table with one row per + * converter attribute, values rendered as wrapped chips (each with a + * colored type badge) instead of plain text, since a session converter's + * scope/output are references to columns or groups, not literal strings. + */ +function SessionConverterParametersTable({ + step, + optionLabels, + columnTypes, + outputKey, + outputLabel, + t, + localization, +}) { + const paramColumns = [ + { accessorKey: "key", header: t("common:parameter"), grow: 1 }, + { accessorKey: "value", header: t("common:value"), grow: 4 }, + ]; + + const paramRows = [ + { + // Plain comma-joined text, matching the notebook's own + // ConverterParametersTable (ConverterBox.jsx) — no chips here, since + // that's how Notebooks shows an already-applied converter's scope. + key: t("datasets:label.scopeColumns"), + value: step.scope + .map((ref) => { + const key = refToKey(ref); + return optionLabels[key] || key; + }) + .join(", "), + }, + { + key: t("datasets:label.converterOutput"), + value: ( + + ), + }, + ]; + + const table = useMaterialReactTable({ + columns: paramColumns, + data: paramRows, + muiTableBodyCellProps: { sx: { whiteSpace: "pre" } }, + localization, + initialState: { density: "compact" }, + enablePagination: false, + enableTopToolbar: false, + enableBottomToolbar: false, + enableColumnActions: false, + enableSorting: false, + enableColumnFilter: false, + muiTablePaperProps: { elevation: 0 }, + }); + + return ; +} + +SessionConverterParametersTable.propTypes = { + step: PropTypes.object.isRequired, + optionLabels: PropTypes.object.isRequired, + columnTypes: PropTypes.object.isRequired, + outputKey: PropTypes.string.isRequired, + outputLabel: PropTypes.string.isRequired, + t: PropTypes.func.isRequired, + localization: PropTypes.object.isRequired, +}; + +/** + * A single applied-converter card, styled after the notebook's own + * ConverterBox (icon + real component display name + description + + * parameters table), but built against the session's preprocessing step + * shape (`{converter, scope, outputType}`) instead of the notebook's + * (`{parameters: {scope, target}}`), and with a "Salida" row showing the + * converter's declared output type, since that group doesn't exist as a + * real column until the session is created. + */ +function SessionConverterCard({ + step, + index, + displayName, + description, + onDelete, + datasetTypes, + preprocessing, + stepDisplayNames, +}) { + const theme = useTheme(); + const { t } = useTranslation(["datasets", "models", "common"]); + const localization = useTableLocalization(); + + const { columnTypes } = buildColumnKeysAndTypes({ + datasetTypes, + preprocessing, + }); + // Every earlier step's group label uses its resolved display name (e.g. + // "Bag of Words: output"), matching this card's own output label below — + // not the raw registry name buildColumnKeysAndTypes falls back to when it + // has no display-name lookup of its own. + const optionLabels = Object.fromEntries( + stepDisplayNames.map((name, i) => [`__group__${i}`, `${name}: output`]), + ); + const outputKey = `__group__${index}`; + const outputLabel = `${displayName}: output`; + + return ( + + + + + {displayName} + + + onDelete(index)} + aria-label={t("common:remove")} + > + + + + + + {description && ( + + {description} + + )} + + + + ); +} + +SessionConverterCard.propTypes = { + step: PropTypes.object.isRequired, + index: PropTypes.number.isRequired, + displayName: PropTypes.string.isRequired, + description: PropTypes.string, + onDelete: PropTypes.func.isRequired, + datasetTypes: PropTypes.object, + preprocessing: PropTypes.array, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, +}; + +/** + * Cards for every converter already added to the session's preprocessing + * sequence, mirroring the notebook module's applied-tool cards. + * + * Deleting a converter cascades: any later converter whose scope + * references its output group would be left pointing at a step that no + * longer exists, so every converter configured after it is removed too. + * Confirmed first via the same `DeleteConfirmationModal` + + * `ItemsToDeleteList` the notebook's own converter deletion uses, so the + * user sees exactly what else is about to go before confirming. + */ +export default function AppliedConvertersView({ + newExp, + setNewExp, + datasetTypes, +}) { + const { t } = useTranslation(["experiments", "models", "datasets", "common"]); + const theme = useTheme(); + const { setPendingDropTool } = useExplorersAndConverters(); + const [convertersMeta, setConvertersMeta] = useState({}); + const [deleteIndex, setDeleteIndex] = useState(null); + const [isDragOver, setIsDragOver] = useState(false); + const [isDragging, setIsDragging] = useState(false); + const steps = newExp.preprocessing || []; + + // A converter is draggable from SessionConvertersRightBar's ToolList/ + // ToolGrid for free (drag-start lives in the shared ToolListItem/ + // ToolGridItem, not in those wrappers), so this view only needs to be a + // drop target — same "application/x-dashai-tool" payload and the same + // setPendingDropTool hand-off Notebooks' own NotebookView.jsx uses, which + // ToolList/ToolGrid already resolve through their normal click-to-add path. + useEffect(() => { + const onStart = (e) => { + if (e.dataTransfer.types.includes("application/x-dashai-tool")) { + setIsDragging(true); + } + }; + const onEnd = () => { + setIsDragging(false); + setIsDragOver(false); + }; + window.addEventListener("dragstart", onStart); + window.addEventListener("dragend", onEnd); + return () => { + window.removeEventListener("dragstart", onStart); + window.removeEventListener("dragend", onEnd); + }; + }, []); + + const handleDragOver = (e) => { + if (!e.dataTransfer.types.includes("application/x-dashai-tool")) return; + e.preventDefault(); + e.dataTransfer.dropEffect = "copy"; + }; + + const handleDragEnter = (e) => { + if (!e.dataTransfer.types.includes("application/x-dashai-tool")) return; + e.preventDefault(); + setIsDragOver(true); + }; + + const handleDragLeave = (e) => { + const related = e.relatedTarget; + if (!related || !e.currentTarget.contains(related)) { + setIsDragOver(false); + } + }; + + const handleDrop = (e) => { + e.preventDefault(); + setIsDragOver(false); + try { + const tool = JSON.parse( + e.dataTransfer.getData("application/x-dashai-tool"), + ); + if (tool?.name) setPendingDropTool(tool); + } catch { + // ignore invalid drops + } + }; + + useEffect(() => { + let cancelled = false; + getComponents({ selectTypes: ["Converter"] }) + .then((data) => { + if (cancelled) return; + const byName = Object.fromEntries( + (data || []).map((component) => [component.name, component]), + ); + setConvertersMeta(byName); + }) + .catch((error) => + console.error("Failed to fetch converter metadata:", error), + ); + return () => { + cancelled = true; + }; + }, []); + + const stepDisplayNames = steps.map( + (step) => convertersMeta[step.converter]?.display_name || step.converter, + ); + + const itemsToDelete = useMemo(() => { + if (deleteIndex === null) return []; + return steps.slice(deleteIndex).map((step, i) => ({ + id: deleteIndex + i, + type: "converter", + converter: stepDisplayNames[deleteIndex + i], + })); + }, [steps, deleteIndex, stepDisplayNames]); + + const handleConfirmDelete = () => { + setNewExp({ + ...newExp, + preprocessing: steps.slice(0, deleteIndex), + }); + setDeleteIndex(null); + }; + + return ( + + {isDragging && ( + + + {t("datasets:label.dropToolHere")} + + + )} + + {steps.length === 0 ? ( + + {t("experiments:label.noConverterAdded")} + + ) : ( + steps.map((step, index) => { + const meta = convertersMeta[step.converter]; + return ( + setDeleteIndex(i)} + datasetTypes={datasetTypes} + preprocessing={steps} + stepDisplayNames={stepDisplayNames} + /> + ); + }) + )} + + setDeleteIndex(null)} + onConfirm={handleConfirmDelete} + content={ + + + {t("datasets:label.deleteConverterConfirmation", { + converter: stepDisplayNames[deleteIndex], + })} + + + + } + /> + + ); +} + +AppliedConvertersView.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + datasetTypes: PropTypes.object, +}; diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx new file mode 100644 index 000000000..5723802fd --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx @@ -0,0 +1,174 @@ +import React from "react"; +import { screen, fireEvent } from "@testing-library/react"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import { getComponents } from "../../../api/component"; +import { useExplorersAndConverters } from "../../notebooks/context/ExplorersAndConvertersContext"; + +jest.mock("../../../api/component", () => ({ + getComponents: jest.fn(), +})); + +jest.mock("../../notebooks/context/ExplorersAndConvertersContext", () => ({ + useExplorersAndConverters: jest.fn(), +})); + +import AppliedConvertersView from "./AppliedConvertersView"; + +const renderView = (props) => + renderWithProviders(); + +const makeDataTransfer = (payload) => ({ + types: payload ? ["application/x-dashai-tool"] : [], + getData: () => (payload ? JSON.stringify(payload) : ""), + dropEffect: "none", +}); + +const AVAILABLE_CONVERTERS = [ + { + name: "Binarizer", + display_name: "Binarizador", + description: "Binariza datos.", + metadata: { output_type: "Integer" }, + }, + { + name: "BagOfWordsConverter", + display_name: "Bag of Words", + description: "Bolsa de palabras.", + metadata: { output_type: "Integer" }, + }, +]; + +const datasetTypes = { age: { type: "Integer" }, text: { type: "Text" } }; + +describe("AppliedConvertersView", () => { + let setPendingDropTool; + + beforeEach(() => { + getComponents.mockResolvedValue(AVAILABLE_CONVERTERS); + setPendingDropTool = jest.fn(); + useExplorersAndConverters.mockReturnValue({ setPendingDropTool }); + }); + + it("shows a message when there are no converters", () => { + renderView({ + newExp: { preprocessing: [] }, + setNewExp: () => {}, + datasetTypes, + }); + + expect(screen.getByText("No converter was added.")).toBeInTheDocument(); + }); + + it("shows a converter's real display name, scope and output type chip", async () => { + const newExp = { + preprocessing: [ + { + converter: "Binarizer", + params: { threshold: 0.5 }, + scope: [{ kind: "raw", name: "age" }], + outputType: "Integer", + }, + ], + }; + + renderView({ newExp, setNewExp: () => {}, datasetTypes }); + + expect(await screen.findByText("Binarizador")).toBeInTheDocument(); + expect(screen.getByText("age")).toBeInTheDocument(); + expect(screen.getByText("Binarizador: output")).toBeInTheDocument(); + expect(screen.getAllByText("Integer").length).toBeGreaterThan(0); + }); + + it("shows a chained converter's scope as the earlier converter's output group", async () => { + const newExp = { + preprocessing: [ + { + converter: "BagOfWordsConverter", + params: {}, + scope: [{ kind: "raw", name: "text" }], + outputType: "Integer", + }, + { + converter: "Binarizer", + params: {}, + scope: [{ kind: "group", step: 0 }], + outputType: "Integer", + }, + ], + }; + + renderView({ newExp, setNewExp: () => {}, datasetTypes }); + + await screen.findByText("Bag of Words"); + // The second card's scope shows the first converter's output label. + expect(screen.getAllByText("Bag of Words: output").length).toBe(2); + }); + + it("cascades deletion to every converter configured after the deleted one", async () => { + const setNewExp = jest.fn(); + const newExp = { + preprocessing: [ + { + converter: "BagOfWordsConverter", + params: {}, + scope: [{ kind: "raw", name: "text" }], + outputType: "Integer", + }, + { + converter: "Binarizer", + params: {}, + scope: [{ kind: "group", step: 0 }], + outputType: "Integer", + }, + ], + }; + + renderView({ newExp, setNewExp, datasetTypes }); + + await screen.findByText("Bag of Words"); + const deleteButtons = screen.getAllByRole("button", { name: "Remove" }); + fireEvent.click(deleteButtons[0]); + + // Deleting the first converter cascades to the second (its scope + // references the first one's output group), so the confirmation modal + // must warn about both before anything is actually removed. + expect( + await screen.findByText("The following items will be deleted:"), + ).toBeInTheDocument(); + expect(setNewExp).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByRole("button", { name: "Delete" })); + + expect(setNewExp).toHaveBeenCalledWith({ + ...newExp, + preprocessing: [], + }); + }); + + it("hands a dropped tool off to setPendingDropTool, just like Notebooks' drop target", () => { + renderView({ + newExp: { preprocessing: [] }, + setNewExp: () => {}, + datasetTypes, + }); + + const dropZone = screen.getByText("No converter was added.").parentElement; + const tool = { name: "Binarizer", display_name: "Binarizador" }; + fireEvent.drop(dropZone, { dataTransfer: makeDataTransfer(tool) }); + + expect(setPendingDropTool).toHaveBeenCalledWith(tool); + }); + + it("ignores a drop with no recognizable tool payload", () => { + renderView({ + newExp: { preprocessing: [] }, + setNewExp: () => {}, + datasetTypes, + }); + + const dropZone = screen.getByText("No converter was added.").parentElement; + fireEvent.drop(dropZone, { dataTransfer: makeDataTransfer(null) }); + + expect(setPendingDropTool).not.toHaveBeenCalled(); + }); +}); diff --git a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx index 20351a932..0b8c35d11 100644 --- a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx +++ b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx @@ -15,6 +15,8 @@ import { useTranslation } from "react-i18next"; function DivideDatasetColumns({ allColumnNames, columnTypes = {}, + inputOptionNames, + optionLabels = {}, selectedInputColumnNames, onInputColumnNamesChange, selectedOutputColumnNames, @@ -28,6 +30,8 @@ function DivideDatasetColumns({ const { t } = useTranslation(["experiments", "common"]); const theme = useTheme(); + const inputOptions = inputOptionNames || allColumnNames; + const handleInputAutocompleteChange = (event, newValue) => { onInputColumnNamesChange(newValue); }; @@ -36,13 +40,12 @@ function DivideDatasetColumns({ onOutputColumnNamesChange(newValue); }; - const getColumnLabel = (columnName) => { - const columnType = columnTypes[columnName]; - if (columnType && columnType.type) { - return `${columnName} (${columnType.type})`; - } - return columnName; - }; + // Each option's identity (used for selection/value tracking) doesn't have + // to be its display text — `optionLabels` lets a caller show a readable + // label (e.g. a converter's output group label) for an option whose real + // identity is an internal synthetic key. Defaults to the identity itself, + // so a plain raw column name (with no entry in optionLabels) is unaffected. + const getOptionLabel = (option) => optionLabels[option] || option; const renderColumnOption = (props, option) => { const { key, ...otherProps } = props; @@ -58,7 +61,7 @@ function DivideDatasetColumns({ {...otherProps} sx={{ display: "flex", alignItems: "center", gap: 2 }} > - {option} + {getOptionLabel(option)} {columnType && columnType.type && ( - {option} + {getOptionLabel(option)} ) : ( - option + getOptionLabel(option) ); return ; @@ -129,10 +132,10 @@ function DivideDatasetColumns({ data-tour="dataset-input-columns-autocomplete" multiple id="dataset-input-columns-autocomplete" - options={allColumnNames} + options={inputOptions} value={selectedInputColumnNames} onChange={handleInputAutocompleteChange} - getOptionLabel={(option) => option} + getOptionLabel={getOptionLabel} renderOption={renderColumnOption} renderTags={renderTags} filterSelectedOptions @@ -146,14 +149,14 @@ function DivideDatasetColumns({ error={inputError} helperText={inputHelperText} placeholder={ - allColumnNames.length > 0 + inputOptions.length > 0 ? t("common:selectColumns") : t("common:loadingColumns") } /> )} sx={{ mb: 8 }} - disabled={disabled || allColumnNames.length === 0} + disabled={disabled || inputOptions.length === 0} /> option} + getOptionLabel={getOptionLabel} renderOption={renderColumnOption} renderTags={renderTags} filterSelectedOptions @@ -192,6 +195,11 @@ function DivideDatasetColumns({ DivideDatasetColumns.propTypes = { allColumnNames: PropTypes.arrayOf(PropTypes.string).isRequired, columnTypes: PropTypes.object, + // Input options can include synthetic keys (a converter's output group) + // beyond the raw dataset columns; output stays raw-only (see spec: every + // output ColumnRef must be raw). Defaults to allColumnNames. + inputOptionNames: PropTypes.arrayOf(PropTypes.string), + optionLabels: PropTypes.object, selectedInputColumnNames: PropTypes.arrayOf(PropTypes.string).isRequired, onInputColumnNamesChange: PropTypes.func.isRequired, selectedOutputColumnNames: PropTypes.arrayOf(PropTypes.string).isRequired, diff --git a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.test.jsx b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.test.jsx new file mode 100644 index 000000000..53a5d657e --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.test.jsx @@ -0,0 +1,70 @@ +import React from "react"; +import { screen, fireEvent } from "@testing-library/react"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import DivideDatasetColumns from "./DivideDatasetColumns"; + +describe("DivideDatasetColumns", () => { + it("offers a converter's output group as an input column option, with its label and type chip", async () => { + const onInputColumnNamesChange = jest.fn(); + renderWithProviders( + {}} + />, + ); + + const input = screen.getByLabelText(/Input Columns/i); + fireEvent.mouseDown(input); + fireEvent.click(await screen.findByText("Binarizador: output")); + + expect(onInputColumnNamesChange).toHaveBeenCalledWith(["__group__0"]); + }); + + it("does not offer converter groups as output column options", () => { + renderWithProviders( + {}} + selectedOutputColumnNames={[]} + onOutputColumnNamesChange={() => {}} + />, + ); + + const output = screen.getByLabelText(/Output Columns/i); + fireEvent.mouseDown(output); + expect(screen.queryByText("Binarizador: output")).not.toBeInTheDocument(); + }); + + it("falls back to allColumnNames for input options when inputOptionNames is not passed", () => { + renderWithProviders( + {}} + selectedOutputColumnNames={[]} + onOutputColumnNamesChange={() => {}} + />, + ); + + const input = screen.getByLabelText(/Input Columns/i); + fireEvent.mouseDown(input); + expect(screen.getAllByText("age").length).toBeGreaterThan(0); + expect(screen.getAllByText("name").length).toBeGreaterThan(0); + }); +}); diff --git a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx new file mode 100644 index 000000000..5c5a9178a --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx @@ -0,0 +1,96 @@ +import React, { useState } from "react"; +import PropTypes from "prop-types"; +import { Box } from "@mui/material"; +import ScopeStepSessionConverter from "./ScopeStepSessionConverter"; +import ParameterStepConverter from "../../notebooks/converterCreation/ParameterStepConverter"; + +/** + * "Add a converter" form for the session wizard's preprocessing step — + * mirrors the notebook's FormConverterSection (same two-step scope/params + * flow, same ParameterStepConverter for the second step), but a session + * converter is never fit immediately: saving here only appends + * `{converter, params, scope}` to the session's local `preprocessing` list. + * The actual fit happens once, later, when the session is created (see the + * backend's PreprocessingJob) — never from this form. + */ +export default function FormSessionConverterSection({ + step, + setStep, + handleClose, + tool, + newExp, + setNewExp, + datasetTypes, + filePath, + hideButtons = false, +}) { + const [scope, setScope] = useState([]); + + const handleSaveConverter = async (params) => { + const newStep = { + converter: tool.name, + params: params || {}, + scope, + outputType: tool?.metadata?.output_type || null, + outputDtype: tool?.metadata?.output_dtype || null, + }; + setNewExp({ + ...newExp, + preprocessing: [...(newExp.preprocessing || []), newStep], + }); + handleClose(); + }; + + return ( + + {step === 0 && ( + 0 + ? () => setStep((s) => s + 1) + : () => handleSaveConverter({}) + } + /> + )} + + {step === 1 && ( + + )} + + ); +} + +FormSessionConverterSection.propTypes = { + step: PropTypes.number.isRequired, + setStep: PropTypes.func.isRequired, + handleClose: PropTypes.func.isRequired, + tool: PropTypes.object.isRequired, + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + datasetTypes: PropTypes.object.isRequired, + filePath: PropTypes.string, + hideButtons: PropTypes.bool, +}; diff --git a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx index 85859df9b..889e51d9d 100644 --- a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx +++ b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx @@ -1,26 +1,17 @@ -import React, { useEffect, useLayoutEffect, useState } from "react"; +import React, { useEffect, useState } from "react"; import PropTypes from "prop-types"; import { Grid, - CircularProgress, Box, Alert, AlertTitle, - Chip, + FormControlLabel, + Switch, + Typography, } from "@mui/material"; -import DivideDatasetColumns from "./DivideDatasetColumns"; import SplitDatasetRows from "./SplitDatasetRows"; -import { - getDatasetInfo as getDatasetInfoRequest, - getDatasetTypes as getDatasetTypesRequest, -} from "../../../api/datasets"; -import { getComponents as getComponentsRequest } from "../../../api/component"; -import { validateColumns as validateColumnsRequest } from "../../../api/modelSession"; -import { useSnackbar } from "notistack"; -import { getColorByColumnType } from "../../../utils"; import { useTranslation } from "react-i18next"; -import { Trans } from "react-i18next"; import { useModels } from "../ModelsContext"; import { buildSplitsPayload, @@ -29,13 +20,17 @@ import { SPLIT_TYPES, } from "../../../utils/splitsPayload"; /** - * Step of the experiment modal: Set the input and output columns to use for clasification - * and the splits for training, validation and testing. - * @param {object} newExp object that contains the Experiment Modal state - * @param {function} setNewExp updates the Eperimento Modal state (newExp) - * @param {function} setNextEnabled function to enable or disable the "Next" button in the modal - * @param {string} evaluationStrategy the evaluation strategy selected for the experiment, either holdout or cross-validation - * @param {function} setEvaluationStrategy function to update the evaluation strategy in the parent component (CreateSessionSteps) + * Step of the session wizard: configure the evaluation strategy, partitions, + * and whether preprocessing converters should be applied before column + * selection. Column selection itself lives in SelectColumnsStep. + * @param {object} newExp object that contains the Session wizard state + * @param {function} setNewExp updates the session wizard state (newExp) + * @param {function} setNextEnabled function to enable or disable the "Next" button + * @param {string} evaluationStrategy the evaluation strategy selected for the session + * @param {function} setEvaluationStrategy function to update the evaluation strategy + * @param {object} dataset the selected dataset + * @param {object} datasetInfo dataset metadata fetched by the parent step + * @param {boolean} infoLoading whether datasetInfo is still being fetched */ function PrepareDatasetStep({ newExp, @@ -44,35 +39,15 @@ function PrepareDatasetStep({ dataset, evaluationStrategy, setEvaluationStrategy, + datasetInfo, + infoLoading, }) { const { setSessionRightContent } = useModels(); - const [datasetInfo, setDatasetInfo] = useState({}); - const [datasetTypes, setDatasetTypes] = useState({}); - const { enqueueSnackbar } = useSnackbar(); - const [infoLoading, setInfoLoading] = useState(true); - const { t } = useTranslation(["experiments", "common"]); - - // null means "not fetched yet" — distinct from the empty-but-loaded shape - // getTaskRequirements falls back to when the task genuinely isn't found. - // The banner below only renders once this is non-null, otherwise it briefly - // interpolates its message with blank task name/types/cardinality. - const [taskRequirements, setTaskRequirements] = useState(null); + const { t } = useTranslation(["experiments", "models", "common"]); - const [inputColumnNames, setInputColumnNames] = useState( - newExp.input_columns, + const [applyPreprocessing, setApplyPreprocessing] = useState( + Boolean(newExp.applyPreprocessing), ); - const [outputColumnNames, setOutputColumnNames] = useState( - newExp.output_columns, - ); - - const columnsReady = - inputColumnNames.length >= 1 && outputColumnNames.length >= 1; - const [columnsAreValid, setColumnsAreValid] = useState(false); - // True until the current column selection has actually been checked against - // the backend at least once — distinct from columnsAreValid=false, so the - // banner doesn't flash red while columns are still being auto-selected or a - // check is in flight, only once a real valid/invalid result is known. - const [validationPending, setValidationPending] = useState(true); // Values submitted by the schema generated splitter form, and whether that // form currently reports a validation error. @@ -104,158 +79,26 @@ function PrepareDatasetStep({ const [splitsReady, setSplitsReady] = useState(false); - const getDatasetInfo = async () => { - if (!dataset?.id) return; - setInfoLoading(true); - setInputColumnNames([]); - setOutputColumnNames([]); - try { - const [fetchedDatasetInfo, fetchedDatasetTypes] = await Promise.all([ - getDatasetInfoRequest(dataset.id), - getDatasetTypesRequest(dataset.id), - ]); - setDatasetInfo(fetchedDatasetInfo); - setDatasetTypes(fetchedDatasetTypes); - - if (fetchedDatasetInfo) { - setDatasetPartitionsIndex({ - train: fetchedDatasetInfo.train_indices || [], - validation: fetchedDatasetInfo.val_indices || [], - test: fetchedDatasetInfo.test_indices || [], - }); - } - - if ( - fetchedDatasetInfo && - fetchedDatasetInfo.column_names && - fetchedDatasetInfo.column_names.length > 0 - ) { - const allNames = fetchedDatasetInfo.column_names; - if ( - inputColumnNames.length === 0 && - (!newExp.input_columns || newExp.input_columns.length === 0) - ) { - if (allNames.length > 1) { - setInputColumnNames(allNames.slice(0, -1)); - } else if (allNames.length === 1) { - setInputColumnNames([allNames[0]]); - } - } - - if ( - outputColumnNames.length === 0 && - (!newExp.output_columns || newExp.output_columns.length === 0) - ) { - if (allNames.length > 0) { - setOutputColumnNames([allNames[allNames.length - 1]]); - } - } - } - } catch (error) { - enqueueSnackbar(t("experiments:error.errorFetchingDatasetInfo")); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - } finally { - setInfoLoading(false); - } - }; - - const getTaskRequirements = async () => { - try { - const taskComponents = await getComponentsRequest({ - selectTypes: ["Task"], - }); - - const currentTask = taskComponents.find( - (task) => task.name === newExp.task_name, - ); - if (currentTask) { - setTaskRequirements(currentTask); - } else { - enqueueSnackbar( - t("experiments:error.taskRequirementsNotFound", { - taskName: newExp.task_name, - }), - ); - setTaskRequirements({ - name: newExp.task_name, - metadata: { - inputs_types: [], - inputs_cardinality: "", - outputs_types: [], - outputs_cardinality: "", - }, - }); - } - } catch (error) { - enqueueSnackbar(t("experiments:error.errorFetchingTaskRequirements")); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - } - }; - - const validateColumns = async () => { - try { - if ( - !datasetInfo || - !datasetInfo.column_names || - datasetInfo.column_names.length === 0 - ) { - setColumnsAreValid(false); - return; - } - - if (inputColumnNames.length === 0 || outputColumnNames.length === 0) { - setColumnsAreValid(false); - return; - } - - const validation = await validateColumnsRequest( - newExp.task_name, - dataset.id, - inputColumnNames, - outputColumnNames, - ); - setColumnsAreValid(validation.dataset_status === "valid"); - } catch (error) { - enqueueSnackbar(t("experiments:error.errorFetchingColumnsValidation")); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - setColumnsAreValid(false); - } finally { - setValidationPending(false); - } - }; - - const updateExperiment = () => { + useEffect(() => { if ( - !datasetInfo || - !datasetInfo.column_names || - datasetInfo.column_names.length === 0 + datasetInfo && + (datasetInfo.train_indices || + datasetInfo.val_indices || + datasetInfo.test_indices) ) { - return; + setDatasetPartitionsIndex({ + train: datasetInfo.train_indices || [], + validation: datasetInfo.val_indices || [], + test: datasetInfo.test_indices || [], + }); } + }, [datasetInfo]); + const updateExperiment = () => { const updatedExpData = { ...newExp, - input_columns: inputColumnNames, - output_columns: outputColumnNames, evaluation_strategy: evaluationStrategy, + applyPreprocessing: applyPreprocessing, }; const splitterName = resolveSplitterName(strategyKind, cvType, holdoutType); @@ -282,87 +125,31 @@ function PrepareDatasetStep({ setNewExp(updatedExpData); }; - // Column validity depends on the columns, the dataset and the task, never on - // the split configuration. Gating it on the splits being ready made every - // split change re-check the columns over HTTP and blank the requirements - // banner in the meantime. - // - // When columnsReady is false we already know the selection is invalid - // (input or output is empty) without asking the backend, so - // validationPending goes straight to false — otherwise the requirements - // banner below stayed hidden forever after the user cleared a column, - // instead of showing why the selection is invalid. - // - // This runs as a layout effect (not a regular effect) so that when - // columns go from empty back to a ready selection — e.g. the dataset - // finishes loading and auto-selects defaults — validationPending flips - // back to true synchronously before the browser paints. A regular effect - // runs one commit too late: React would paint a frame with the stale - // "already validated" state (an incorrect red banner) against the new, - // not-yet-checked columns before the effect corrected it. - useLayoutEffect(() => { - if (!columnsReady) { - setColumnsAreValid(false); - setValidationPending(false); - return; - } - if ( - datasetInfo && - datasetInfo.column_names && - datasetInfo.column_names.length > 0 - ) { - setValidationPending(true); - validateColumns(); - } - }, [columnsReady, inputColumnNames, outputColumnNames, datasetInfo]); - useEffect(() => { - if (columnsAreValid && splitsReady && columnsReady) { + if (splitsReady) { updateExperiment(); setNextEnabled(true); } else { setNextEnabled(false); } }, [ - columnsReady, splitsReady, - columnsAreValid, splitType, splitterParams, - inputColumnNames, - outputColumnNames, cvType, holdoutType, strategyKind, groupColumn, evaluationStrategy, + applyPreprocessing, rowsPartitionsIndex, datasetPartitionsIndex, ]); - useEffect(() => { - getDatasetInfo(); - }, [dataset?.id]); - - useEffect(() => { - getTaskRequirements(); - }, []); - // Push SplitDatasetRows (or loading spinner) into the right bar useEffect(() => { if (infoLoading) { - setSessionRightContent( - - - , - ); + setSessionRightContent(null); return () => setSessionRightContent(null); } setSessionRightContent( @@ -388,7 +175,7 @@ function PrepareDatasetStep({ setStrategyKind={setStrategyKind} groupColumn={groupColumn} setGroupColumn={setGroupColumn} - inputColumnNames={inputColumnNames} + inputColumnNames={datasetInfo.column_names || []} taskName={newExp.task_name} />, ); @@ -405,71 +192,8 @@ function PrepareDatasetStep({ holdoutType, strategyKind, groupColumn, - inputColumnNames, ]); - const columnGroupsOf = (side) => { - const metadata = taskRequirements?.metadata ?? {}; - if (Array.isArray(metadata[side]) && metadata[side].length > 0) { - return metadata[side]; - } - const cardinality = metadata[`${side}_cardinality`]; - return [ - { - types: metadata[`${side}_types`] ?? [], - min: cardinality === "n" ? 0 : cardinality, - max: cardinality, - }, - ]; - }; - - const describeCardinality = ({ min, max }) => { - if (max === "n") { - return min ? t("experiments:label.cardinalityAtLeast", { min }) : "n"; - } - if (min === max) { - return String(max); - } - return t("experiments:label.cardinalityBetween", { min, max }); - }; - - const renderTypesAsChips = (typesList) => { - if (!typesList || typesList.length === 0) { - return {t("common:any")}; - } - - return ( - - {typesList.map((type, index) => ( - - - {index < typesList.length - 1 && ( - {t("common:or")} - )} - - ))} - - ); - }; - return ( {!infoLoading && datasetInfo.nan ? ( @@ -499,90 +223,29 @@ function PrepareDatasetStep({ ) : null ) : null} - {taskRequirements && !infoLoading && !validationPending && ( - - `${theme.palette[columnsAreValid ? "success" : "error"].main}40`, - border: (theme) => - `1px solid ${ - theme.palette[columnsAreValid ? "success" : "error"].main - }`, - }} - data-tour="models-validation-alert" - > - - {t( - columnsAreValid - ? "experiments:label.columnsValidRequirements" - : "experiments:label.columnsInvalidRequirements", - { taskName: taskRequirements.display_name }, - )} - - - {["inputs", "outputs"].map((side) => - columnGroupsOf(side).map((group, index) => ( - - - - The columns must be of the types - {renderTypesAsChips(group.types)} - , and they should have a cardinality of - - {{ cardinality: describeCardinality(group) }}. - - - - - )), - )} - - - )} - {!infoLoading ? ( - - - - ) : ( - - - - )} + + setApplyPreprocessing(event.target.checked)} + /> + } + label={t("models:label.applyPreprocessing")} + /> + + {t("models:label.applyPreprocessingDescription")} + + ); } @@ -600,9 +263,14 @@ PrepareDatasetStep.propTypes = { created: PropTypes.instanceOf(Date), last_modified: PropTypes.instanceOf(Date), runs: PropTypes.array, + applyPreprocessing: PropTypes.bool, }), setNewExp: PropTypes.func.isRequired, setNextEnabled: PropTypes.func.isRequired, dataset: PropTypes.object.isRequired, + evaluationStrategy: PropTypes.string, + setEvaluationStrategy: PropTypes.func, + datasetInfo: PropTypes.object, + infoLoading: PropTypes.bool, }; export default PrepareDatasetStep; diff --git a/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx b/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx new file mode 100644 index 000000000..41a6d2051 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx @@ -0,0 +1,61 @@ +import React, { useEffect } from "react"; +import PropTypes from "prop-types"; +import { useModels } from "../ModelsContext"; +import SessionConvertersRightBar from "./SessionConvertersRightBar"; +import AppliedConvertersView from "./AppliedConvertersView"; + +/** + * Optional preprocessing step of the session wizard: pick converters to fit + * on training data only (see the backend's PreprocessingJob), and configure + * each one's scope. A converter's scope may include the not-yet-materialized + * output group of an earlier converter in the same list, which is how + * chaining works before any real fit exists. + * + * Styled like the notebook module's own converter picker: the catalog + * (search + category list/grid) lives in the right bar + * (SessionConvertersRightBar), the already-added converters are shown as + * cards in the main content area (AppliedConvertersView). + */ +function PreprocessingStep({ + newExp, + setNewExp, + setNextEnabled, + dataset, + datasetTypes, +}) { + const { setSessionRightContent } = useModels(); + + useEffect(() => { + setNextEnabled(true); + }, [setNextEnabled]); + + useEffect(() => { + setSessionRightContent( + , + ); + return () => setSessionRightContent(null); + }, [newExp, setNewExp, dataset, datasetTypes]); + + return ( + + ); +} + +PreprocessingStep.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + setNextEnabled: PropTypes.func.isRequired, + dataset: PropTypes.object, + datasetTypes: PropTypes.object, +}; + +export default PreprocessingStep; diff --git a/DashAI/front/src/components/models/modelSession/PreprocessingStep.test.jsx b/DashAI/front/src/components/models/modelSession/PreprocessingStep.test.jsx new file mode 100644 index 000000000..fc979eb08 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/PreprocessingStep.test.jsx @@ -0,0 +1,50 @@ +import React from "react"; +import { screen } from "@testing-library/react"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import { getComponents } from "../../../api/component"; + +// react-markdown ships ESM only and Create React App's jest does not transform +// node_modules, so importing PreprocessingStep (which transitively imports the +// converter parameter form, all the way down to FormTooltip) pulls in a parse +// error before a single assertion runs. Same stub used elsewhere in this repo +// (e.g. FormSchemaRenderFields.test.jsx) for the same reason. +jest.mock("react-markdown", () => ({ + __esModule: true, + default: ({ children }) => {children}, +})); + +jest.mock("../../../api/component", () => ({ + getComponents: jest.fn(), +})); + +jest.mock("../ModelsContext", () => ({ + useModels: () => ({ setSessionRightContent: () => {} }), +})); + +jest.mock("../../notebooks/context/ExplorersAndConvertersContext", () => ({ + useExplorersAndConverters: () => ({ setPendingDropTool: () => {} }), +})); + +import PreprocessingStep from "./PreprocessingStep"; + +describe("PreprocessingStep", () => { + beforeEach(() => { + getComponents.mockResolvedValue([]); + }); + + it("shows a message when no converter has been added yet", async () => { + renderWithProviders( + {}} + setNextEnabled={() => {}} + datasetInfo={{ column_names: [] }} + datasetTypes={{}} + />, + ); + + expect( + await screen.findByText("No converter was added."), + ).toBeInTheDocument(); + }); +}); diff --git a/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx new file mode 100644 index 000000000..7d1976e0a --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx @@ -0,0 +1,108 @@ +import { useMemo, useState } from "react"; +import PropTypes from "prop-types"; +import { Box, Typography } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; +import { useTranslation } from "react-i18next"; +import ColumnSelector from "../../notebooks/ColumnSelector"; +import FormSchemaButtonGroup from "../../shared/FormSchemaButtonGroup"; +import { buildColumnKeysAndTypes, keyToRef } from "./sessionColumnRefs"; + +/** + * Scope step for a converter being added to a session's preprocessing + * sequence: column selection only (there's no row-level scope — the + * train/test split doesn't exist yet at config time, and a session + * converter never sees rows outside its own fold's training partition at + * fit time regardless of what's picked here). + * + * The scope offered isn't just the dataset's raw columns: it's every raw + * column plus one synthetic key per output group of every converter + * already configured *before* this one in the sequence (chaining). The + * shared `ColumnSelector` only knows a plain `{name: {type, dtype}}` map, + * so groups are represented as synthetic keys built by `sessionColumnRefs` + * and translated back into real ColumnRef scope entries once the user + * picks a selection. + */ +export default function ScopeStepSessionConverter({ + tool, + datasetTypes, + preprocessing, + filePath, + scope, + setScope, + nextStep, +}) { + const theme = useTheme(); + const { t } = useTranslation(["common", "datasets", "models"]); + const [isColumnSelectionValid, setIsColumnSelectionValid] = useState(false); + + const allowedTypes = tool?.metadata?.allowed_types || []; + const allowedDtypes = tool?.metadata?.allowed_dtypes || []; + const nonAllowedDtypes = tool?.metadata?.non_allowed_dtypes || []; + const inputCardinality = tool?.metadata?.input_cardinality || {}; + + const { columnTypes: columnTypesForSelector } = useMemo( + () => buildColumnKeysAndTypes({ datasetTypes, preprocessing }), + [datasetTypes, preprocessing], + ); + + const handleSelectionChange = (selected) => { + setScope(selected.map((col) => keyToRef(col.columnName))); + }; + + const hasParams = Object.values(tool.schema.properties).length > 0; + + return ( + + + + {t("datasets:label.selectScopeDescriptionColumns")} + + + + + + + + + + ); +} + +ScopeStepSessionConverter.propTypes = { + tool: PropTypes.object.isRequired, + datasetTypes: PropTypes.object.isRequired, + preprocessing: PropTypes.array, + filePath: PropTypes.string, + scope: PropTypes.array.isRequired, + setScope: PropTypes.func.isRequired, + nextStep: PropTypes.func.isRequired, +}; diff --git a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx new file mode 100644 index 000000000..ab825916c --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx @@ -0,0 +1,343 @@ +import React, { useEffect, useLayoutEffect, useState } from "react"; +import PropTypes from "prop-types"; + +import { + Grid, + CircularProgress, + Box, + Alert, + AlertTitle, + Chip, +} from "@mui/material"; +import DivideDatasetColumns from "./DivideDatasetColumns"; +import { getComponents as getComponentsRequest } from "../../../api/component"; +import { validateColumns as validateColumnsRequest } from "../../../api/modelSession"; +import { useSnackbar } from "notistack"; +import { getColorByColumnType } from "../../../utils"; +import { useTranslation, Trans } from "react-i18next"; +import { + isGroupKey, + refToKey, + keyToRef, + buildColumnKeysAndTypes, +} from "./sessionColumnRefs"; + +/** + * Step of the session wizard: pick input/output columns. A column can be a + * raw dataset column, or (when preprocessing is configured) the whole output + * group of one of the session's converter steps, represented here as a + * synthetic key (e.g. "__group__0") so the underlying Autocomplete only + * ever deals with plain strings — the same shape a raw column name has. + */ +function SelectColumnsStep({ + newExp, + setNewExp, + setNextEnabled, + dataset, + datasetInfo, + datasetTypes, +}) { + const { enqueueSnackbar } = useSnackbar(); + const { t } = useTranslation(["experiments", "models", "common"]); + + const [taskRequirements, setTaskRequirements] = useState(null); + + const rawColumnNames = datasetInfo.column_names || []; + + const { + allKeys: inputOptionNames, + columnTypes: columnTypesForSelector, + optionLabels, + } = buildColumnKeysAndTypes({ + datasetTypes, + preprocessing: newExp.preprocessing, + }); + + const [inputSelection, setInputSelection] = useState(() => + newExp.input_column_refs && newExp.input_column_refs.length > 0 + ? newExp.input_column_refs.map(refToKey) + : newExp.input_columns || [], + ); + const [outputColumnNames, setOutputColumnNames] = useState( + newExp.output_columns || [], + ); + + const inputColumnRefs = inputSelection.map(keyToRef); + const rawInputNames = inputSelection.filter((key) => !isGroupKey(key)); + + const columnsReady = + inputSelection.length >= 1 && outputColumnNames.length >= 1; + const [columnsAreValid, setColumnsAreValid] = useState(false); + const [validationPending, setValidationPending] = useState(true); + + useEffect(() => { + // Auto-select sensible defaults the first time the dataset's columns load. + if (rawColumnNames.length === 0) return; + if ( + inputSelection.length === 0 && + (!newExp.input_columns || newExp.input_columns.length === 0) + ) { + setInputSelection( + rawColumnNames.length > 1 + ? rawColumnNames.slice(0, -1) + : [rawColumnNames[0]], + ); + } + if ( + outputColumnNames.length === 0 && + (!newExp.output_columns || newExp.output_columns.length === 0) + ) { + setOutputColumnNames([rawColumnNames[rawColumnNames.length - 1]]); + } + }, [rawColumnNames.join(",")]); + + const getTaskRequirements = async () => { + try { + const taskComponents = await getComponentsRequest({ + selectTypes: ["Task"], + }); + const currentTask = taskComponents.find( + (task) => task.name === newExp.task_name, + ); + if (currentTask) { + setTaskRequirements(currentTask); + } else { + setTaskRequirements({ + name: newExp.task_name, + metadata: { + inputs_types: [], + inputs_cardinality: "", + outputs_types: [], + outputs_cardinality: "", + }, + }); + } + } catch (error) { + enqueueSnackbar(t("experiments:error.errorFetchingTaskRequirements")); + console.error("Error fetching task requirements:", error); + } + }; + + const validateColumns = async () => { + try { + if ( + rawColumnNames.length === 0 || + inputSelection.length === 0 || + outputColumnNames.length === 0 + ) { + setColumnsAreValid(false); + return; + } + const hasGroupRef = inputSelection.some(isGroupKey); + const converterOutputTypes = Object.fromEntries( + (newExp.preprocessing || []).map((step, index) => [ + String(index), + step.outputType || null, + ]), + ); + const validation = await validateColumnsRequest( + newExp.task_name, + dataset.id, + hasGroupRef ? rawInputNames : inputSelection, + outputColumnNames, + hasGroupRef ? inputColumnRefs : undefined, + hasGroupRef ? converterOutputTypes : undefined, + ); + setColumnsAreValid(validation.dataset_status === "valid"); + } catch (error) { + enqueueSnackbar(t("experiments:error.errorFetchingColumnsValidation")); + console.error("Error validating columns:", error); + setColumnsAreValid(false); + } finally { + setValidationPending(false); + } + }; + + useLayoutEffect(() => { + if (!columnsReady) { + setColumnsAreValid(false); + setValidationPending(false); + return; + } + if (rawColumnNames.length > 0) { + setValidationPending(true); + validateColumns(); + } + }, [columnsReady, inputSelection.join(","), outputColumnNames.join(",")]); + + useEffect(() => { + if (columnsAreValid && columnsReady) { + setNewExp({ + ...newExp, + input_columns: rawInputNames, + output_columns: outputColumnNames, + input_column_refs: inputColumnRefs, + }); + setNextEnabled(true); + } else { + setNextEnabled(false); + } + }, [ + columnsAreValid, + columnsReady, + inputSelection.join(","), + outputColumnNames.join(","), + ]); + + useEffect(() => { + getTaskRequirements(); + }, []); + + const columnGroupsOf = (side) => { + const metadata = taskRequirements?.metadata ?? {}; + if (Array.isArray(metadata[side]) && metadata[side].length > 0) { + return metadata[side]; + } + const cardinality = metadata[`${side}_cardinality`]; + return [ + { + types: metadata[`${side}_types`] ?? [], + min: cardinality === "n" ? 0 : cardinality, + max: cardinality, + }, + ]; + }; + + const describeCardinality = ({ min, max }) => { + if (max === "n") { + return min ? t("experiments:label.cardinalityAtLeast", { min }) : "n"; + } + if (min === max) { + return String(max); + } + return t("experiments:label.cardinalityBetween", { min, max }); + }; + + const renderTypesAsChips = (typesList) => { + if (!typesList || typesList.length === 0) { + return {t("common:any")}; + } + return ( + + {typesList.map((type, index) => ( + + + {index < typesList.length - 1 && ( + {t("common:or")} + )} + + ))} + + ); + }; + + return ( + + {taskRequirements && !validationPending && ( + + `${theme.palette[columnsAreValid ? "success" : "error"].main}40`, + border: (theme) => + `1px solid ${theme.palette[columnsAreValid ? "success" : "error"].main}`, + }} + data-tour="models-validation-alert" + > + + {t( + columnsAreValid + ? "experiments:label.columnsValidRequirements" + : "experiments:label.columnsInvalidRequirements", + { taskName: taskRequirements.display_name }, + )} + + + {["inputs", "outputs"].map((side) => + columnGroupsOf(side).map((group, index) => ( + + + + The columns must be of the types + {renderTypesAsChips(group.types)} + , and they should have a cardinality of + + {{ cardinality: describeCardinality(group) }}. + + + + + )), + )} + + + )} + + + + + + ); +} + +SelectColumnsStep.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + setNextEnabled: PropTypes.func.isRequired, + dataset: PropTypes.object.isRequired, + datasetInfo: PropTypes.object, + datasetTypes: PropTypes.object, +}; + +export default SelectColumnsStep; diff --git a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx new file mode 100644 index 000000000..88c9db8cf --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx @@ -0,0 +1,229 @@ +import { useEffect, useMemo, useState } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Typography, + ToggleButtonGroup, + ToggleButton, +} from "@mui/material"; +import { useTheme } from "@mui/material/styles"; +import { ViewList, ViewModule } from "@mui/icons-material"; +import { useSnackbar } from "notistack"; +import { useTranslation } from "react-i18next"; +import SearchBar from "../../threeSectionLayout/SearchBar"; +import ToolList from "../../notebooks/tool/ToolList"; +import ToolGrid from "../../notebooks/tool/ToolGrid"; +import { getComponents } from "../../../api/component"; +import { evaluateColumnEligibility } from "../../../utils/columnEligibility"; +import FormSessionConverterSection from "./FormSessionConverterSection"; + +/** + * Converters-only sidebar for the session wizard's preprocessing step, + * styled like the notebook module's own converters picker (same SearchBar, + * ToolList/ToolGrid, category accordion, list/grid toggle). Unlike the + * notebook flow, picking a converter here never fits anything or POSTs + * anywhere — see FormSessionConverterSection, which just appends the + * configured step to newExp.preprocessing. + */ +export default function SessionConvertersRightBar({ + newExp, + setNewExp, + dataset, + datasetTypes, +}) { + const theme = useTheme(); + const { t } = useTranslation(["models", "datasets", "common"]); + const { enqueueSnackbar } = useSnackbar(); + const [converters, setConverters] = useState([]); + const [searchQuery, setSearchQuery] = useState(""); + const [viewMode, setViewMode] = useState("list"); + + useEffect(() => { + let cancelled = false; + getComponents({ selectTypes: ["Converter"] }) + .then((data) => { + if (!cancelled) setConverters(Array.isArray(data) ? data : []); + }) + .catch((error) => { + console.error("Failed to fetch converters:", error); + enqueueSnackbar(t("datasets:error.fetchingConverters"), { + variant: "error", + }); + }); + return () => { + cancelled = true; + }; + }, [t]); + + // Only the raw dataset columns are relevant to "is this converter usable + // at all" gating — a converter that can only ever be scoped on a group + // atom (never offered as scope-eligible on its own) is not a case this + // gate needs to handle, since every converter's very first use is always + // against raw columns. + const datasetColumns = useMemo( + () => + Object.entries(datasetTypes || {}).map(([columnName, typeInfo], idx) => ({ + id: idx, + columnName, + valueType: typeInfo.type || t("common:unknown"), + dataType: typeInfo.dtype || t("common:unknown"), + order: idx, + })), + [datasetTypes, t], + ); + + const validateConverter = (converter) => { + if (!datasetColumns.length) return { disabled: false, tooltip: "" }; + + const { validColumns, shortfall, restrictions, restricted } = + evaluateColumnEligibility(converter?.metadata, datasetColumns, { + unknownLabel: t("common:unknown"), + }); + + let disabled = shortfall !== null; + let tooltip = + converter.description || converter.metadata?.short_description || ""; + + if (shortfall !== null) { + const key = + shortfall.kind === "exact" + ? "datasets:error.requiresExactColumns" + : "datasets:error.requiresMinColumns"; + tooltip += `\n\n${t(key, { + required: shortfall.required, + available: shortfall.available, + count: shortfall.required, + })}`; + } + + if (validColumns.length === 0 && restricted && restrictions.length > 0) { + disabled = true; + tooltip += `\n\n${t("datasets:error.noValidColumnsWithDtypesMentioned", { + dtypes: restrictions.join(", "), + })}`; + } + + return { disabled, tooltip, validColumns }; + }; + + const validatedConverters = useMemo( + () => + converters.map((converter) => { + const validation = validateConverter(converter); + return { ...converter, ...validation }; + }), + [converters, datasetColumns, t], + ); + + const filteredConverters = useMemo(() => { + const query = searchQuery.trim().toLowerCase(); + if (!query) return validatedConverters; + const tokens = query.split(/\s+/).filter(Boolean); + return validatedConverters.filter((item) => { + const displayName = (item.display_name || item.name || "").toLowerCase(); + return tokens.every((token) => displayName.includes(token)); + }); + }, [searchQuery, validatedConverters]); + + // Stable identity across re-renders (e.g. every keystroke in the search + // box) so ConfigureToolModal never remounts FormSessionConverterSection + // — and loses in-progress scope/parameter input — while it's open. + const SessionFormSection = useMemo(() => { + function Wrapped(sectionProps) { + return ( + + ); + } + return Wrapped; + }, [newExp, setNewExp, datasetTypes, dataset]); + + return ( + + + setSearchQuery(e.target.value)} + onClear={() => setSearchQuery("")} + placeholder={t("datasets:label.searchConverters")} + /> + + + + {t("datasets:label.viewMode")} + + newMode && setViewMode(newMode)} + size="small" + > + + + + + + + + + {(() => { + const ListComponent = viewMode === "list" ? ToolList : ToolGrid; + const containerSx = + viewMode === "list" + ? { + flex: 1, + overflowY: "auto", + overflowX: "hidden", + p: 4, + minWidth: 0, + } + : { flex: 1, overflow: "auto", p: 4 }; + return ( + + + + ); + })()} + + ); +} + +SessionConvertersRightBar.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + dataset: PropTypes.object, + datasetTypes: PropTypes.object, +}; diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js new file mode 100644 index 000000000..38eb9ce26 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js @@ -0,0 +1,63 @@ +/** + * Synthetic-key representation of a session's ColumnRef (see the backend's + * DashAI.back.preprocessing.column_ref module). A raw dataset column is + * just its own name; a converter's not-yet-materialized output group is + * represented as a synthetic string key, so every UI piece that already + * knows how to work with a flat `{name: string}` column list (Autocomplete + * options, MaterialReactTable rows, columnTypes maps) can represent a group + * without knowing groups exist at all. + */ + +const GROUP_KEY_PREFIX = "__group__"; + +export const groupKey = (step) => `${GROUP_KEY_PREFIX}${step}`; + +export const isGroupKey = (key) => + typeof key === "string" && key.startsWith(GROUP_KEY_PREFIX); + +export const stepFromGroupKey = (key) => + Number(key.slice(GROUP_KEY_PREFIX.length)); + +/** ColumnRef -> synthetic key */ +export const refToKey = (ref) => + ref.kind === "raw" ? ref.name : groupKey(ref.step); + +/** synthetic key -> ColumnRef */ +export const keyToRef = (key) => + isGroupKey(key) + ? { kind: "group", step: stepFromGroupKey(key) } + : { kind: "raw", name: key }; + +/** + * Every column key a session's preprocessing sequence can be scoped over, + * up to (and not including) `uptoStep`: every raw dataset column, plus one + * group key per converter step before it. Passing no `uptoStep` includes + * every configured step (used once a sequence is final and being displayed, + * e.g. in SelectColumnsStep, where every step is already "before" the + * column-selection step that comes after all of them). + */ +export function buildColumnKeysAndTypes({ + datasetTypes, + preprocessing, + uptoStep, +}) { + const steps = preprocessing || []; + const limit = uptoStep === undefined ? steps.length : uptoStep; + + const columnTypes = { ...datasetTypes }; + const optionLabels = {}; + const allKeys = Object.keys(datasetTypes || {}); + + for (let index = 0; index < limit; index += 1) { + const step = steps[index]; + const key = groupKey(index); + columnTypes[key] = { + type: step.outputType || null, + dtype: step.outputDtype || null, + }; + optionLabels[key] = `${step.converter}: output`; + allKeys.push(key); + } + + return { allKeys, columnTypes, optionLabels }; +} diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js new file mode 100644 index 000000000..ff43497e1 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -0,0 +1,59 @@ +import { + groupKey, + isGroupKey, + stepFromGroupKey, + refToKey, + keyToRef, + buildColumnKeysAndTypes, +} from "./sessionColumnRefs"; + +describe("sessionColumnRefs", () => { + it("round-trips a raw ColumnRef through a key", () => { + const ref = { kind: "raw", name: "age" }; + expect(keyToRef(refToKey(ref))).toEqual(ref); + }); + + it("round-trips a group ColumnRef through a key", () => { + const ref = { kind: "group", step: 2 }; + expect(keyToRef(refToKey(ref))).toEqual(ref); + }); + + it("recognizes group keys and extracts their step", () => { + expect(isGroupKey(groupKey(3))).toBe(true); + expect(isGroupKey("age")).toBe(false); + expect(stepFromGroupKey(groupKey(3))).toBe(3); + }); + + it("builds keys/types/labels for raw columns plus every prior step", () => { + const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ + datasetTypes: { age: { type: "Integer" }, text: { type: "Text" } }, + preprocessing: [ + { + converter: "BagOfWordsConverter", + outputType: "Integer", + outputDtype: "int64", + }, + { converter: "Binarizer", outputType: "Integer" }, + ], + uptoStep: 1, + }); + + expect(allKeys).toEqual(["age", "text", "__group__0"]); + expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); + expect(optionLabels.__group__0).toBe("BagOfWordsConverter: output"); + // step 1 (Binarizer) is excluded: uptoStep=1 only includes steps before it + expect(allKeys).not.toContain(groupKey(1)); + }); + + it("includes every step when uptoStep is omitted", () => { + const { allKeys } = buildColumnKeysAndTypes({ + datasetTypes: { age: { type: "Integer" } }, + preprocessing: [ + { converter: "BagOfWordsConverter", outputType: "Integer" }, + { converter: "Binarizer", outputType: "Integer" }, + ], + }); + + expect(allKeys).toEqual(["age", "__group__0", "__group__1"]); + }); +}); diff --git a/DashAI/front/src/pages/models/ModelsContent.jsx b/DashAI/front/src/pages/models/ModelsContent.jsx index b51d1d993..80fda32a9 100644 --- a/DashAI/front/src/pages/models/ModelsContent.jsx +++ b/DashAI/front/src/pages/models/ModelsContent.jsx @@ -15,6 +15,7 @@ import { useThreePanelLayout } from "../../hooks/useThreePanelsLayout"; import { ThreePanelLayoutContext } from "../../components/threeSectionLayout/panels/ThreePanelLayoutContext"; import { useModels } from "../../components/models/ModelsContext"; import { getDatasetInfo } from "../../api/datasets"; +import { ExplorersAndConvertersProvider } from "../../components/notebooks/context/ExplorersAndConvertersContext"; export default function ModelsContent() { const location = useLocation(); @@ -134,40 +135,57 @@ export default function ModelsContent() { }, [selectedSessionId, sessions]); return ( - - - - {/* Left Panel */} - - - - {selectedSessionId ? ( - - - - - - - - - ) : ( - <> - - - + // ExplorersAndConvertersProvider is shared with the notebooks module: + // the session wizard's preprocessing step reuses the notebook's + // converter sidebar (ToolList/ToolGrid), which reads this context — + // wrapping the whole page here, so the center panel and the right bar + // share one instance of it, the same as the notebooks module does. + + + + + {/* Left Panel */} + + + + {selectedSessionId ? ( + + + + + + + + + ) : ( + <> + + + - {/* Right Panel */} - - - - - )} - - - + {/* Right Panel */} + + + + + )} + + + + ); } diff --git a/DashAI/front/src/types/modelSession.ts b/DashAI/front/src/types/modelSession.ts index d674dcaa8..0048d86b2 100644 --- a/DashAI/front/src/types/modelSession.ts +++ b/DashAI/front/src/types/modelSession.ts @@ -1,6 +1,26 @@ import type { IDataset } from "./dataset"; import type { IRun } from "./run"; +export interface IRawColumnRef { + kind: "raw"; + name: string; +} + +export interface IGroupColumnRef { + kind: "group"; + step: number; +} + +export type IColumnRef = IRawColumnRef | IGroupColumnRef; + +export interface IConverterStep { + converter: string; + params: Record; + scope: IColumnRef[]; +} + +export type IPreprocessingStatus = "ready" | "pending" | "failed"; + export interface IModelSession { id: string; dataset: IDataset; @@ -12,4 +32,9 @@ export interface IModelSession { created: Date; last_modified: Date; runs: IRun[]; + preprocessing?: IConverterStep[]; + input_column_refs?: IColumnRef[]; + preprocessing_status?: IPreprocessingStatus; + preprocessing_error?: string | null; + preprocessing_job_id?: string | null; } diff --git a/DashAI/front/src/utils/i18n/locales/de/models.json b/DashAI/front/src/utils/i18n/locales/de/models.json index 4538ee7f6..e8a76d581 100644 --- a/DashAI/front/src/utils/i18n/locales/de/models.json +++ b/DashAI/front/src/utils/i18n/locales/de/models.json @@ -29,6 +29,7 @@ "error": { "completeRequiredFields": "Bitte füllen Sie alle Pflichtfelder aus", "createRun": "Fehler beim Erstellen eines neuen Durchlaufs: {{name}}", + "createRunReason": "Fehler beim Erstellen eines neuen Durchlaufs \"{{name}}\": {{reason}}", "createSession": "Fehler beim Erstellen einer neuen Sitzung", "datasetRequired": "Datensatz ist erforderlich", "enterModelName": "Bitte geben Sie einen Namen für das Modell ein", diff --git a/DashAI/front/src/utils/i18n/locales/en/datasets.json b/DashAI/front/src/utils/i18n/locales/en/datasets.json index 6de5912f5..4b5d5b748 100644 --- a/DashAI/front/src/utils/i18n/locales/en/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/en/datasets.json @@ -52,6 +52,7 @@ "uploadFile": "Upload a file" }, "error": { + "fetchingConverters": "Error fetching the available converters.", "converterFailed": "An error occurred during processing the converter.", "converterFailedWithInfo": "Error processing converter: {{error}}", "createConverterError": "Failed to create converter", @@ -174,6 +175,7 @@ "constantColumns": "Constant Columns", "convert": "Convert", "converter": "Converter", + "converterOutput": "Output", "correlation": "Correlation", "correlationAnalysis": "Correlation Analysis", "correlations": "Correlations", @@ -340,6 +342,7 @@ "scopeRows": "Scope - Rows", "searchDatasetsNotebooks": "Search Datasets and Notebooks", "searchExplorersConverters": "Search explorers/converters", + "searchConverters": "Search converters", "selectColorscale": "Select Colorscale", "selectColumnsForExplorerScope": "Select the columns to be used by the explorer.", "selectDataloader": "Select Dataloader", diff --git a/DashAI/front/src/utils/i18n/locales/en/experiments.json b/DashAI/front/src/utils/i18n/locales/en/experiments.json index d166e2a37..f02175efb 100644 --- a/DashAI/front/src/utils/i18n/locales/en/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/en/experiments.json @@ -56,6 +56,7 @@ "modelName": "Model Name", "modelsInExperiment": "Current Models in the Experiment", "noDatasetsAvailable": "There are no Datasets Available", + "noConverterAdded": "No converter was added.", "noDatasetsAvailableGoToDataTab": "Go to <1>data tab to upload one first.", "noModelsAvailable": "No Models Available", "noOptimizersNoMetric": "No hyperparameter optimization", diff --git a/DashAI/front/src/utils/i18n/locales/en/models.json b/DashAI/front/src/utils/i18n/locales/en/models.json index db12745b8..07c86abba 100644 --- a/DashAI/front/src/utils/i18n/locales/en/models.json +++ b/DashAI/front/src/utils/i18n/locales/en/models.json @@ -1,5 +1,6 @@ { "button": { + "addConverter": "Add Converter", "createExplainer": "Create Explainer", "createGlobalExplainer": "New Global Explainer", "createLocalExplainer": "New Local Explainer", @@ -29,6 +30,7 @@ "error": { "completeRequiredFields": "Please complete all required fields", "createRun": "Error while trying to create a new run: {{name}}", + "createRunReason": "Error while trying to create a new run \"{{name}}\": {{reason}}", "createSession": "Error while trying to create a new session", "datasetRequired": "Dataset is required", "enterModelName": "Please enter a name for the model", @@ -230,6 +232,14 @@ "predictionsCount_other": "• <1>{{count}} predictions", "divideColumnsAndSplits": "Divide columns and configure dataset splits", "prepareDataset": "Prepare Dataset", + "applyPreprocessing": "Apply preprocessing", + "applyPreprocessingDescription": "Selected converters are fit only on training data and applied to the rest, to avoid data leakage. If not applied, the model trains on the data as-is.", + "preprocessingOptional": "Preprocessing (optional)", + "preprocessingOptionalDescription": "Selected converters are fit only on training data and applied to the rest, to avoid data leakage.", + "preprocessingInProgress": "Preprocessing in progress...", + "preprocessingFailed": "Preprocessing failed", + "selectColumnsTitle": "Select Columns", + "selectColumnsDescription": "Choose which columns are inputs and which one is the output for this session.", "retrainConfirmDetails": "Are you sure you want to re-train run \"<1>{{runName}}\"?", "retrainModel": "Re-train Model?", "retrainWillDeleteOperations": "This run has existing operations that will be deleted", diff --git a/DashAI/front/src/utils/i18n/locales/es/datasets.json b/DashAI/front/src/utils/i18n/locales/es/datasets.json index 1c636ae09..9e2493d3c 100644 --- a/DashAI/front/src/utils/i18n/locales/es/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/es/datasets.json @@ -54,6 +54,7 @@ "uploadFile": "Subir un archivo" }, "error": { + "fetchingConverters": "Error al obtener los converters disponibles.", "converterFailed": "Ocurrió un error durante el procesamiento del convertidor.", "converterFailedWithInfo": "Error al procesar convertidor: {{error}}", "createConverterError": "Fallo al crear convertidor", @@ -180,6 +181,7 @@ "constantColumns": "Columnas Constantes", "convert": "Convertir", "converter": "Convertidor", + "converterOutput": "Salida", "correlation": "Correlación", "correlationAnalysis": "Análisis de Correlación", "correlations": "Correlaciones", @@ -348,6 +350,7 @@ "scopeRows": "Alcance - Filas", "searchDatasetsNotebooks": "Buscar Datasets y Cuadernos", "searchExplorersConverters": "Buscar exploradores/convertidores", + "searchConverters": "Buscar converters", "selectColorscale": "Seleccionar Escala de Colores", "selectColumnsForExplorerScope": "Selecciona las columnas a ser utilizadas por el explorador.", "selectDataloader": "Seleccionar Dataloader", diff --git a/DashAI/front/src/utils/i18n/locales/es/experiments.json b/DashAI/front/src/utils/i18n/locales/es/experiments.json index 53cdeb71b..cd6efcbd4 100644 --- a/DashAI/front/src/utils/i18n/locales/es/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/es/experiments.json @@ -56,6 +56,7 @@ "modelName": "Nombre del Modelo", "modelsInExperiment": "Modelos Actuales en el Experimento", "noDatasetsAvailable": "No hay Datasets Disponibles", + "noConverterAdded": "No se agregó ningún converter.", "noDatasetsAvailableGoToDataTab": "Vaya a <1>pestaña de datos para subir uno primero.", "noModelsAvailable": "No hay Modelos Disponibles", "noOptimizersNoMetric": "Sin optimización de hiperparámetros", diff --git a/DashAI/front/src/utils/i18n/locales/es/models.json b/DashAI/front/src/utils/i18n/locales/es/models.json index 63802b062..ef0d0f57e 100644 --- a/DashAI/front/src/utils/i18n/locales/es/models.json +++ b/DashAI/front/src/utils/i18n/locales/es/models.json @@ -1,5 +1,6 @@ { "button": { + "addConverter": "Agregar Converter", "createExplainer": "Crear Explicador", "createGlobalExplainer": "Nuevo Explicador Global", "createLocalExplainer": "Nuevo Explicador Local", @@ -29,6 +30,7 @@ "error": { "completeRequiredFields": "Por favor complete todos los campos requeridos", "createRun": "Error al intentar crear una nueva ejecución: {{name}}", + "createRunReason": "Error al intentar crear una nueva ejecución \"{{name}}\": {{reason}}", "createSession": "Error al intentar crear una nueva sesión", "datasetRequired": "Se requiere un dataset", "enterModelName": "Por favor ingrese un nombre para el modelo", @@ -236,6 +238,14 @@ "predictionsCount_other": "• <1>{{count}} predicciones", "divideColumnsAndSplits": "Divide columnas y configura las particiones del dataset", "prepareDataset": "Preparar Dataset", + "applyPreprocessing": "Aplicar preprocesamiento", + "applyPreprocessingDescription": "Los converters seleccionados se ajustan solo sobre los datos de entrenamiento y se aplican al resto, para evitar fuga de datos. Si no se aplica, el modelo se entrena con los datos tal como están.", + "preprocessingOptional": "Preprocesamiento (opcional)", + "preprocessingOptionalDescription": "Los converters seleccionados se ajustan solo sobre los datos de entrenamiento y se aplican al resto, para evitar fuga de datos.", + "preprocessingInProgress": "Procesando...", + "preprocessingFailed": "El preprocesamiento falló", + "selectColumnsTitle": "Seleccionar Columnas", + "selectColumnsDescription": "Elige qué columnas son de entrada y cuál es la de salida para esta sesión.", "retrainConfirmDetails": "¿Está seguro de que desea re-entrenar la ejecución \"<1>{{runName}}\"?", "retrainModel": "¿Re-entrenar Modelo?", "retrainWillDeleteOperations": "Esta ejecución tiene operaciones existentes que serán eliminadas", diff --git a/DashAI/front/src/utils/i18n/locales/pt/models.json b/DashAI/front/src/utils/i18n/locales/pt/models.json index 521d38d20..aa2855b7b 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/models.json +++ b/DashAI/front/src/utils/i18n/locales/pt/models.json @@ -29,6 +29,7 @@ "error": { "completeRequiredFields": "Por favor, preencha todos os campos obrigatórios", "createRun": "Erro ao tentar criar uma nova execução: {{name}}", + "createRunReason": "Erro ao tentar criar uma nova execução \"{{name}}\": {{reason}}", "createSession": "Erro ao tentar criar uma nova sessão", "datasetRequired": "É necessário um conjunto de dados", "enterModelName": "Por favor, insira um nome para o modelo", diff --git a/DashAI/front/src/utils/i18n/locales/zh/models.json b/DashAI/front/src/utils/i18n/locales/zh/models.json index e5a2aac54..47f6c9242 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/models.json +++ b/DashAI/front/src/utils/i18n/locales/zh/models.json @@ -29,6 +29,7 @@ "error": { "completeRequiredFields": "请填写所有必填字段", "createRun": "创建新运行时出错：{{name}}", + "createRunReason": "创建新运行 \"{{name}}\" 时出错：{{reason}}", "createSession": "创建新会话时出错", "datasetRequired": "数据集为必填项", "enterModelName": "请输入模型名称", From a2e21a527d05956dde60881861215f4946fd124c Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 8 Sep 2026 22:16:41 -0300 Subject: [PATCH 09/39] feat: track session preprocessing status via the shared job-polling mechanism Uses useJobTracker (the same mechanism every other job-backed indicator in the app already relies on: RunnerDialog, ComponentDownloadControl, prediction/explainer panels) instead of an independent timer polling preprocessing_status, so the session's processing/failed views never drift out of sync with what the Job Queue widget shows for the same job. --- .../src/components/models/ModelsContext.jsx | 20 +++++++ .../models/SessionVisualization.jsx | 58 ++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/DashAI/front/src/components/models/ModelsContext.jsx b/DashAI/front/src/components/models/ModelsContext.jsx index 3c0578a7d..0443e2479 100644 --- a/DashAI/front/src/components/models/ModelsContext.jsx +++ b/DashAI/front/src/components/models/ModelsContext.jsx @@ -10,6 +10,7 @@ import { useTranslation } from "react-i18next"; import { useSharedDatasets } from "../../contexts/DatasetsContext"; import { useSessions } from "../../hooks/models/useSessions"; import { useModelComponents } from "../../hooks/models/useModelComponents"; +import { useJobTracker } from "../../hooks/useJobPolling"; const ModelsContext = createContext(null); export const useModels = () => useContext(ModelsContext); @@ -149,6 +150,25 @@ export function ModelsProvider({ children }) { fetchTasks(); }, [i18n.language]); + // Track the selected session's PreprocessingJob through the same shared + // job-polling mechanism the Job Queue widget itself uses (jobPoller.js), + // instead of polling preprocessing_status on an independent timer — this + // is what every other job-backed "processing" indicator in the app does + // (RunnerDialog, ComponentDownloadControl, prediction/explainer panels, + // ...). Sharing the exact same poll loop for the exact same job id is what + // keeps this indicator and the widget from ever showing contradictory + // states. Refreshing the whole session list on success/error is enough, + // since ModelsContent re-derives `selectedSession` from it. No-op for + // sessions with no preprocessing steps (preprocessing_job_id stays null). + const hasPendingPreprocessing = + (selectedSession?.preprocessing?.steps || []).length > 0 && + selectedSession?.preprocessing_status === "pending"; + useJobTracker( + hasPendingPreprocessing ? selectedSession?.preprocessing_job_id : null, + fetchSessions, + fetchSessions, + ); + // Memoized — this context wraps the entire models page tree, so a fresh // object literal every render would force every consumer (RunCard, // ModelDetailView, ModelsRightBar, ...) to re-render whenever ANY field diff --git a/DashAI/front/src/components/models/SessionVisualization.jsx b/DashAI/front/src/components/models/SessionVisualization.jsx index 2bedbe66d..3e1f29cd8 100644 --- a/DashAI/front/src/components/models/SessionVisualization.jsx +++ b/DashAI/front/src/components/models/SessionVisualization.jsx @@ -1,7 +1,14 @@ import React, { useState, useEffect } from "react"; import { useStrategyKind } from "../../hooks/useStrategyKind"; import { STRATEGY_KINDS } from "../../utils/splitsPayload"; -import { Box, Typography, Divider, Button, ToggleButton } from "@mui/material"; +import { + Box, + Typography, + Divider, + Button, + ToggleButton, + CircularProgress, +} from "@mui/material"; import { useTheme } from "@mui/material/styles"; import { useParams, useNavigate } from "react-router-dom"; import { PlayArrow } from "@mui/icons-material"; @@ -293,6 +300,55 @@ export default function SessionVisualization() { ); } + // Sessions with preprocessing steps run a PreprocessingJob (fit/transform + // on train, persist to disk) right after creation — no Run can train, and + // nothing about the session is safe to show, until it finishes. + const hasPreprocessing = (session.preprocessing?.steps || []).length > 0; + if (hasPreprocessing && session.preprocessing_status === "failed") { + return ( + + + {t("models:label.preprocessingFailed")} + + {session.preprocessing_error && ( + + {session.preprocessing_error} + + )} + + ); + } + if (hasPreprocessing && session.preprocessing_status === "pending") { + return ( + + + + {t("models:label.preprocessingInProgress")} + + + ); + } + return ( <> Date: Tue, 8 Sep 2026 22:16:55 -0300 Subject: [PATCH 10/39] fix: surface the real backend error when creating a Run fails handleCreateRun always showed the same generic message regardless of cause, so a session blocked by the preprocessing-not-ready guard (or any other backend rejection) never told the user why. Uses the project's existing getApiErrorMessage helper (already used by RAG) to show the backend's actual detail, falling back to the generic message only when there isn't one. --- .../front/src/components/models/AddModelDialog.jsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/DashAI/front/src/components/models/AddModelDialog.jsx b/DashAI/front/src/components/models/AddModelDialog.jsx index 02e5c5e54..8812f3dcb 100644 --- a/DashAI/front/src/components/models/AddModelDialog.jsx +++ b/DashAI/front/src/components/models/AddModelDialog.jsx @@ -38,6 +38,7 @@ import { useTourContext } from "../tour/TourProvider"; import { checkIfHaveOptimazers } from "../../utils/schema"; import { getDatasetInfo } from "../../api/datasets"; import { useModels } from "./ModelsContext"; +import { getApiErrorMessage } from "../../utils/apiError"; const DEFAULT_INNER_CONFIG = { splitterType: null, // null means: derive from outer splitter on mount @@ -328,9 +329,16 @@ function AddModelDialog({ console.error("Unknown Error", error.message); } - enqueueSnackbar(t("models:error.createRun", { name }), { - variant: "error", - }); + enqueueSnackbar( + t("models:error.createRunReason", { + name, + reason: getApiErrorMessage( + error, + t("models:error.createRun", { name }), + ), + }), + { variant: "error" }, + ); } finally { setLoading(false); } From c697cdf34b9703d37cec3cdcc14cf7dbaf591b9e Mon Sep 17 00:00:00 2001 From: Creylay Date: Thu, 10 Sep 2026 13:15:43 -0300 Subject: [PATCH 11/39] feat: Enhance session preprocessing to support output slots for mixed column types - Introduced `_classify_by_type` method in `SessionPreprocessor` to categorize output columns by their types. - Updated the fitting process to resolve output slots for each step, allowing converters to handle mixed types. - Modified `AppliedConvertersView` to display output entries as chips for each declared slot. - Adjusted tests to cover new functionality, including validation of slotted group references and output slot resolution. - Enhanced `resolveDeclaredOutputSlots` to determine output types based on the converter's scope and type preservation. - Updated frontend components to accommodate changes in output structure and ensure proper rendering of converter outputs. --- .../api/api_v1/endpoints/model_sessions.py | 8 +- DashAI/back/converters/base_converter.py | 10 + .../converters/category/feature_selection.py | 1 + .../scikit_learn/variance_threshold.py | 1 + DashAI/back/job/model_job.py | 4 +- DashAI/back/job/preprocessing_job.py | 4 +- DashAI/back/preprocessing/column_ref.py | 28 ++- .../preprocessing/session_preprocessor.py | 40 +++- .../modelSession/AppliedConvertersView.jsx | 77 +++++-- .../AppliedConvertersView.test.jsx | 47 +++- .../FormSessionConverterSection.jsx | 11 +- .../models/modelSession/SelectColumnsStep.jsx | 21 +- .../SessionConvertersRightBar.jsx | 40 ++-- .../models/modelSession/sessionColumnRefs.js | 174 +++++++++++++-- .../modelSession/sessionColumnRefs.test.js | 210 +++++++++++++++++- tests/back/api/test_model_session_api.py | 36 +++ .../test_base_converter_metadata.py | 36 +++ tests/back/preprocessing/test_column_ref.py | 49 ++++ .../test_session_preprocessor.py | 142 ++++++++++++ 19 files changed, 856 insertions(+), 83 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index 72ff17c0d..aad2cc72b 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -197,7 +197,13 @@ async def validate_columns( task_metadata = task.get_metadata() allowed_input_types = set(task_metadata.get("inputs_types", [])) for ref in group_refs: - declared_type = declared_types.get(str(ref.step)) + # A step with a heterogeneous scope (see SessionPreprocessor. + # _classify_by_type) can declare more than one type, one per + # slot — "{step}:{slot}" disambiguates which one a slotted ref + # means; an unslotted ref (the whole step) keeps the plain + # "{step}" key, unchanged from before slots existed. + key = str(ref.step) if ref.slot is None else f"{ref.step}:{ref.slot}" + declared_type = declared_types.get(key) type_ok = declared_type in allowed_input_types or ( "DashAIValue" in allowed_input_types and declared_type in _DASHAI_VALUE_TYPE_NAMES diff --git a/DashAI/back/converters/base_converter.py b/DashAI/back/converters/base_converter.py index 7805d8d0b..79306a3dc 100644 --- a/DashAI/back/converters/base_converter.py +++ b/DashAI/back/converters/base_converter.py @@ -42,6 +42,15 @@ class BaseConverter(ConfigObject, ABC): COLOR: Final[str] = "rgb(255, 255, 255)" SUPERVISED: bool = False CHANGES_ROW_COUNT: bool = False + # True for converters that never transform values, only keep or drop + # whole columns as-is (feature selection, variance thresholding): the + # output type of a surviving column is always exactly its input type, no + # arithmetic involved. Lets a caller that already knows the real input + # type (e.g. the Models-module wizard, once a real scope is chosen) use + # that instead of this class's own best-effort get_output_type() guess, + # which — called on a bare unfitted instance — has no idea what column + # it will actually run on. + PRESERVES_INPUT_TYPE: bool = False SCHEMA: BaseConverterSchema @classmethod @@ -74,6 +83,7 @@ def get_metadata(cls) -> Dict[str, Any]: meta["download_size_bytes"] = getattr(cls, "DOWNLOAD_SIZE_BYTES", None) meta["supervised"] = cls.SUPERVISED meta["changes_row_count"] = cls.CHANGES_ROW_COUNT + meta["preserves_input_type"] = cls.PRESERVES_INPUT_TYPE meta["n_components_features_bounded"] = getattr( cls, "N_COMPONENTS_FEATURES_BOUNDED", False ) diff --git a/DashAI/back/converters/category/feature_selection.py b/DashAI/back/converters/category/feature_selection.py index f090ae69a..a8c9919e2 100644 --- a/DashAI/back/converters/category/feature_selection.py +++ b/DashAI/back/converters/category/feature_selection.py @@ -34,6 +34,7 @@ class FeatureSelectionConverter(BaseConverter): ) ICON: Final[str] = Icon.FilterList.value COLOR: Final[str] = "rgb(255, 206, 86)" + PRESERVES_INPUT_TYPE = True def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/converters/scikit_learn/variance_threshold.py b/DashAI/back/converters/scikit_learn/variance_threshold.py index 216e755d7..335893904 100644 --- a/DashAI/back/converters/scikit_learn/variance_threshold.py +++ b/DashAI/back/converters/scikit_learn/variance_threshold.py @@ -92,6 +92,7 @@ class VarianceThreshold( de="Varianz-Schwellenwert", zh="方差阈值", ) + PRESERVES_INPUT_TYPE = True def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/job/model_job.py b/DashAI/back/job/model_job.py index 5db3fe021..2fe6a7047 100644 --- a/DashAI/back/job/model_job.py +++ b/DashAI/back/job/model_job.py @@ -70,7 +70,9 @@ def apply_persisted_preprocessing(model_session: ModelSession, x, y): preprocessor = pickle.load(f) transformed = preprocessor.transform_only(split_dict) - resolved_input_columns = resolve_refs(input_refs, preprocessor.resolved_columns) + resolved_input_columns = resolve_refs( + input_refs, preprocessor.resolved_columns, preprocessor.resolved_slots + ) fold_x = { split_name: dataset.select_columns(resolved_input_columns) diff --git a/DashAI/back/job/preprocessing_job.py b/DashAI/back/job/preprocessing_job.py index be34995db..993d997d9 100644 --- a/DashAI/back/job/preprocessing_job.py +++ b/DashAI/back/job/preprocessing_job.py @@ -154,7 +154,9 @@ def run(self) -> None: with open(os.path.join(tmp_dir, "final.pkl"), "wb") as f: pickle.dump(final_preprocessor, f) - resolved_input_columns = resolve_refs(input_refs, final_resolved) + resolved_input_columns = resolve_refs( + input_refs, final_resolved, final_preprocessor.resolved_slots + ) self.report_progress(0.95, "Validating against the task") task = component_registry[model_session.task_name]["class"]() diff --git a/DashAI/back/preprocessing/column_ref.py b/DashAI/back/preprocessing/column_ref.py index ad20a2a4f..652860a64 100644 --- a/DashAI/back/preprocessing/column_ref.py +++ b/DashAI/back/preprocessing/column_ref.py @@ -8,7 +8,7 @@ PreprocessingJob has fit the sequence (see session_preprocessor.py). """ -from typing import Any, Dict, List, Literal, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel, Field, TypeAdapter from typing_extensions import Annotated @@ -22,6 +22,15 @@ class RawColumnRef(BaseModel): class GroupColumnRef(BaseModel): kind: Literal["group"] = "group" step: int + # None (the default) means "every column step produced" — unchanged, + # backward-compatible behavior. A converter step's real output columns + # aren't always one homogeneous type (e.g. SimpleImputer with a scope + # that mixes categorical and numeric columns just preserves each one's + # own type), so `slot` lets a ref pick out only the columns of one + # declared type from that step's output, once a real fit has classified + # them (see SessionPreprocessor._classify_by_type). The slot name is a + # DashAI type's display_name(), e.g. "Categorical" or "Integer". + slot: Optional[str] = None ColumnRef = Annotated[Union[RawColumnRef, GroupColumnRef], Field(discriminator="kind")] @@ -57,6 +66,7 @@ def validate_scopes(self) -> None: def resolve_refs( refs: List[Union[RawColumnRef, GroupColumnRef]], resolved_columns: Dict[int, List[str]], + resolved_slots: Optional[Dict[int, Dict[str, List[str]]]] = None, ) -> List[str]: """Flatten a list of ColumnRef into concrete column names. @@ -67,25 +77,33 @@ def resolve_refs( resolved_columns : dict Maps a ConverterSequence step index to the concrete column names that step produced in one specific fit (see SessionPreprocessor). + resolved_slots : dict, optional + Maps a step index to {type_name: [column names]}, the same step + output classified by real per-column type (see SessionPreprocessor. + _classify_by_type). Required only if some ref has a non-None `slot`. Returns ------- list of str - Concrete column names, in order. A GroupColumnRef expands to every - column its step produced. + Concrete column names, in order. A GroupColumnRef with `slot=None` + expands to every column its step produced; with a `slot` set, only + to that step's columns of that declared type. Raises ------ KeyError If a GroupColumnRef names a step with no entry in resolved_columns - (the step has not been fit yet). + (or, for a slot ref, no matching entry in resolved_slots) — the step + has not been fit yet, or produced no column of that type. """ names: List[str] = [] for ref in refs: if ref.kind == "raw": names.append(ref.name) - else: + elif ref.slot is None: names.extend(resolved_columns[ref.step]) + else: + names.extend((resolved_slots or {})[ref.step][ref.slot]) return names diff --git a/DashAI/back/preprocessing/session_preprocessor.py b/DashAI/back/preprocessing/session_preprocessor.py index 8dddf0224..7ca6ff801 100644 --- a/DashAI/back/preprocessing/session_preprocessor.py +++ b/DashAI/back/preprocessing/session_preprocessor.py @@ -33,11 +33,39 @@ def __init__(self, sequence: ConverterSequence, component_registry: Any): self.component_registry = component_registry self.fitted_converters: List[Any] = [] self.resolved_columns: Dict[int, List[str]] = {} + self.resolved_slots: Dict[int, Dict[str, List[str]]] = {} def _instantiate(self, step) -> Any: converter_class = self.component_registry[step.converter]["class"] return converter_class(**step.params) + @staticmethod + def _classify_by_type( + converter: Any, column_names: List[str] + ) -> Dict[str, List[str]]: + """Group a step's real output columns by their real, per-column type. + + Calls the now-fitted converter's own get_output_type(column_name) — + already implemented by every converter, and already accurate once + fitted (e.g. SimpleImputer's preserves the input column's own type + for "most_frequent"/"constant"). Most converters produce one + homogeneous type, so this is a single slot; a converter whose scope + mixed column types (e.g. SimpleImputer imputing a categorical and a + numeric column together) naturally splits into one slot per type, + with no converter-specific code needed here or in the converter + itself. + """ + slots: Dict[str, List[str]] = {} + for name in column_names: + output_type = converter.get_output_type(name) + type_name = ( + output_type.display_name() + if output_type is not None and hasattr(output_type, "display_name") + else "unknown" + ) + slots.setdefault(type_name, []).append(name) + return slots + @staticmethod def _transform_split(converter, dataset, scope_names, train_transformed): """Transform one split's scoped columns, without crashing on 0 rows. @@ -92,9 +120,12 @@ def fit_transform( current: Dict[str, "DashAIDataset"] = dict(split) self.fitted_converters = [] self.resolved_columns = {} + self.resolved_slots = {} for index, step in enumerate(self.sequence.steps): - scope_names = resolve_refs(step.scope, self.resolved_columns) + scope_names = resolve_refs( + step.scope, self.resolved_columns, self.resolved_slots + ) converter = self._instantiate(step) train_scope = current["train"].select_columns(scope_names) @@ -128,6 +159,9 @@ def fit_transform( self.resolved_columns[index] = ( new_columns if new_columns else list(train_transformed.column_names) ) + self.resolved_slots[index] = self._classify_by_type( + converter, self.resolved_columns[index] + ) new_current = {} for split_name, dataset in current.items(): @@ -160,7 +194,9 @@ def transform_only( current: Dict[str, "DashAIDataset"] = dict(split) for index, converter in enumerate(self.fitted_converters): scope_names = resolve_refs( - self.sequence.steps[index].scope, self.resolved_columns + self.sequence.steps[index].scope, + self.resolved_columns, + self.resolved_slots, ) train_transformed = None diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx index 280848a3a..089fa436c 100644 --- a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx @@ -22,7 +22,11 @@ import { useTableLocalization } from "../../../utils/useTableLocalization"; import DeleteConfirmationModal from "../../threeSectionLayout/DeleteConfirmationModal"; import ItemsToDeleteList from "../../notebooks/converter/ItemsToDeleteList"; import { useExplorersAndConverters } from "../../notebooks/context/ExplorersAndConvertersContext"; -import { buildColumnKeysAndTypes, refToKey } from "./sessionColumnRefs"; +import { + buildColumnKeysAndTypes, + groupKey, + refToKey, +} from "./sessionColumnRefs"; function TypeChip({ type }) { if (!type) return null; @@ -75,8 +79,7 @@ function SessionConverterParametersTable({ step, optionLabels, columnTypes, - outputKey, - outputLabel, + outputEntries, t, localization, }) { @@ -99,13 +102,21 @@ function SessionConverterParametersTable({ .join(", "), }, { + // Usually one chip; more than one when this step's scope mixed + // column types (e.g. SimpleImputer preserving both a categorical and + // a numeric column), so each declared slot gets its own chip. key: t("datasets:label.converterOutput"), value: ( - + + {outputEntries.map(({ key, label }) => ( + + ))} + ), }, ]; @@ -132,8 +143,12 @@ SessionConverterParametersTable.propTypes = { step: PropTypes.object.isRequired, optionLabels: PropTypes.object.isRequired, columnTypes: PropTypes.object.isRequired, - outputKey: PropTypes.string.isRequired, - outputLabel: PropTypes.string.isRequired, + outputEntries: PropTypes.arrayOf( + PropTypes.shape({ + key: PropTypes.string.isRequired, + label: PropTypes.string.isRequired, + }), + ).isRequired, t: PropTypes.func.isRequired, localization: PropTypes.object.isRequired, }; @@ -142,10 +157,11 @@ SessionConverterParametersTable.propTypes = { * A single applied-converter card, styled after the notebook's own * ConverterBox (icon + real component display name + description + * parameters table), but built against the session's preprocessing step - * shape (`{converter, scope, outputType}`) instead of the notebook's + * shape (`{converter, scope, outputSlots}`) instead of the notebook's * (`{parameters: {scope, target}}`), and with a "Salida" row showing the - * converter's declared output type, since that group doesn't exist as a - * real column until the session is created. + * converter's declared output type(s) — usually one chip, more than one + * when the step's scope mixed column types — since that group doesn't + * exist as a real column until the session is created. */ function SessionConverterCard({ step, @@ -165,15 +181,29 @@ function SessionConverterCard({ datasetTypes, preprocessing, }); - // Every earlier step's group label uses its resolved display name (e.g. - // "Bag of Words: output"), matching this card's own output label below — - // not the raw registry name buildColumnKeysAndTypes falls back to when it - // has no display-name lookup of its own. - const optionLabels = Object.fromEntries( - stepDisplayNames.map((name, i) => [`__group__${i}`, `${name}: output`]), - ); - const outputKey = `__group__${index}`; - const outputLabel = `${displayName}: output`; + // Every earlier step's group label(s) use its resolved display name (e.g. + // "Bag of Words: output"), matching this card's own output label(s) + // below — not the raw registry name buildColumnKeysAndTypes falls back + // to when it has no display-name lookup of its own. One label per + // declared slot, so a step whose scope mixed types (more than one slot) + // gets one distinguishable label per slot. + const optionLabels = {}; + preprocessing.forEach((s, i) => { + const name = stepDisplayNames[i]; + const slots = s.outputSlots?.length > 0 ? s.outputSlots : [{ slot: null }]; + slots.forEach(({ slot }) => { + const key = groupKey(i, slot); + optionLabels[key] = slot + ? `${name}: output (${slot})` + : `${name}: output`; + }); + }); + const ownSlots = + step.outputSlots?.length > 0 ? step.outputSlots : [{ slot: null }]; + const outputEntries = ownSlots.map(({ slot }) => { + const key = groupKey(index, slot); + return { key, label: optionLabels[key] || `${displayName}: output` }; + }); return ( diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx index 5723802fd..a1865f03c 100644 --- a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx @@ -66,7 +66,7 @@ describe("AppliedConvertersView", () => { converter: "Binarizer", params: { threshold: 0.5 }, scope: [{ kind: "raw", name: "age" }], - outputType: "Integer", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; @@ -86,13 +86,13 @@ describe("AppliedConvertersView", () => { converter: "BagOfWordsConverter", params: {}, scope: [{ kind: "raw", name: "text" }], - outputType: "Integer", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, { converter: "Binarizer", params: {}, scope: [{ kind: "group", step: 0 }], - outputType: "Integer", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; @@ -104,6 +104,43 @@ describe("AppliedConvertersView", () => { expect(screen.getAllByText("Bag of Words: output").length).toBe(2); }); + it("shows one chip per declared slot when a step's scope mixed column types", async () => { + const newExp = { + preprocessing: [ + { + converter: "SimpleImputer", + params: { strategy: "most_frequent" }, + scope: [ + { kind: "raw", name: "age" }, + { kind: "raw", name: "text" }, + ], + outputSlots: [ + { slot: "Integer", type: "Integer", dtype: "int64" }, + { slot: "Categorical", type: "Categorical", dtype: null }, + ], + }, + ], + }; + + renderView({ + newExp, + setNewExp: () => {}, + datasetTypes: { + ...datasetTypes, + // "SimpleImputer" isn't in AVAILABLE_CONVERTERS, so its display + // name falls back to the raw converter name — fine here, this + // test only cares about the output chips. + }, + }); + + expect( + await screen.findByText("SimpleImputer: output (Integer)"), + ).toBeInTheDocument(); + expect( + screen.getByText("SimpleImputer: output (Categorical)"), + ).toBeInTheDocument(); + }); + it("cascades deletion to every converter configured after the deleted one", async () => { const setNewExp = jest.fn(); const newExp = { @@ -112,13 +149,13 @@ describe("AppliedConvertersView", () => { converter: "BagOfWordsConverter", params: {}, scope: [{ kind: "raw", name: "text" }], - outputType: "Integer", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, { converter: "Binarizer", params: {}, scope: [{ kind: "group", step: 0 }], - outputType: "Integer", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; diff --git a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx index 5c5a9178a..1ff3be744 100644 --- a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx +++ b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx @@ -3,6 +3,7 @@ import PropTypes from "prop-types"; import { Box } from "@mui/material"; import ScopeStepSessionConverter from "./ScopeStepSessionConverter"; import ParameterStepConverter from "../../notebooks/converterCreation/ParameterStepConverter"; +import { resolveDeclaredOutputSlots } from "./sessionColumnRefs"; /** * "Add a converter" form for the session wizard's preprocessing step — @@ -27,12 +28,18 @@ export default function FormSessionConverterSection({ const [scope, setScope] = useState([]); const handleSaveConverter = async (params) => { + const outputSlots = resolveDeclaredOutputSlots({ + tool, + params, + scope, + datasetTypes, + preprocessing: newExp.preprocessing, + }); const newStep = { converter: tool.name, params: params || {}, scope, - outputType: tool?.metadata?.output_type || null, - outputDtype: tool?.metadata?.output_dtype || null, + outputSlots, }; setNewExp({ ...newExp, diff --git a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx index ab825916c..6eb54bfea 100644 --- a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx +++ b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx @@ -129,12 +129,21 @@ function SelectColumnsStep({ return; } const hasGroupRef = inputSelection.some(isGroupKey); - const converterOutputTypes = Object.fromEntries( - (newExp.preprocessing || []).map((step, index) => [ - String(index), - step.outputType || null, - ]), - ); + // Keyed "{step}" for a step with one declared type (unslotted, the + // common case) or "{step}:{slot}" for one of several — matches the + // key scheme validate_columns looks declared_type up by backend-side + // (see model_sessions.py), which mirrors GroupColumnRef.slot. + const converterOutputTypes = {}; + (newExp.preprocessing || []).forEach((step, index) => { + const slots = + step.outputSlots?.length > 0 + ? step.outputSlots + : [{ slot: null, type: step.outputType || null }]; + slots.forEach(({ slot, type }) => { + const key = slot == null ? String(index) : `${index}:${slot}`; + converterOutputTypes[key] = type || null; + }); + }); const validation = await validateColumnsRequest( newExp.task_name, dataset.id, diff --git a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx index 88c9db8cf..c5128d2ff 100644 --- a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx +++ b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx @@ -16,6 +16,7 @@ import ToolGrid from "../../notebooks/tool/ToolGrid"; import { getComponents } from "../../../api/component"; import { evaluateColumnEligibility } from "../../../utils/columnEligibility"; import FormSessionConverterSection from "./FormSessionConverterSection"; +import { buildColumnKeysAndTypes } from "./sessionColumnRefs"; /** * Converters-only sidebar for the session wizard's preprocessing step, @@ -55,21 +56,34 @@ export default function SessionConvertersRightBar({ }; }, [t]); - // Only the raw dataset columns are relevant to "is this converter usable - // at all" gating — a converter that can only ever be scoped on a group - // atom (never offered as scope-eligible on its own) is not a case this - // gate needs to handle, since every converter's very first use is always - // against raw columns. + // A new converter can be scoped on any raw dataset column OR the output + // group (any declared slot) of any converter already configured — it + // always lands at the end of the sequence, so every existing step is + // "before" it and fair game to chain off. Gating this on raw columns + // alone wrongly blocked, e.g., PCA on a text-only dataset (no raw + // Integer/Float columns) even after adding Bag of Words, whose Integer + // output group PCA could legitimately scope on. + const { columnTypes: allColumnTypes } = useMemo( + () => + buildColumnKeysAndTypes({ + datasetTypes, + preprocessing: newExp.preprocessing, + }), + [datasetTypes, newExp.preprocessing], + ); + const datasetColumns = useMemo( () => - Object.entries(datasetTypes || {}).map(([columnName, typeInfo], idx) => ({ - id: idx, - columnName, - valueType: typeInfo.type || t("common:unknown"), - dataType: typeInfo.dtype || t("common:unknown"), - order: idx, - })), - [datasetTypes, t], + Object.entries(allColumnTypes || {}).map( + ([columnName, typeInfo], idx) => ({ + id: idx, + columnName, + valueType: typeInfo.type || t("common:unknown"), + dataType: typeInfo.dtype || t("common:unknown"), + order: idx, + }), + ), + [allColumnTypes, t], ); const validateConverter = (converter) => { diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js index 38eb9ce26..395881d3f 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js @@ -6,34 +6,84 @@ * knows how to work with a flat `{name: string}` column list (Autocomplete * options, MaterialReactTable rows, columnTypes maps) can represent a group * without knowing groups exist at all. + * + * A step's output isn't always one homogeneous type — e.g. SimpleImputer + * with "most_frequent"/"constant" just preserves each scope column's own + * type, so a scope mixing a categorical and a numeric column produces both + * kinds of columns. `slot` (a DashAI type's display_name(), e.g. + * "Categorical") picks out just the columns of one declared type from a + * step's output, mirroring the backend's GroupColumnRef.slot / + * SessionPreprocessor.resolved_slots. A step with only one declared type — + * the common case, and everything before slots existed — has no slot + * (`slot: null`), which means "the whole group," unchanged from before. */ const GROUP_KEY_PREFIX = "__group__"; +const SLOT_SEPARATOR = "__slot__"; -export const groupKey = (step) => `${GROUP_KEY_PREFIX}${step}`; +export const groupKey = (step, slot = null) => + slot == null + ? `${GROUP_KEY_PREFIX}${step}` + : `${GROUP_KEY_PREFIX}${step}${SLOT_SEPARATOR}${slot}`; export const isGroupKey = (key) => typeof key === "string" && key.startsWith(GROUP_KEY_PREFIX); -export const stepFromGroupKey = (key) => - Number(key.slice(GROUP_KEY_PREFIX.length)); +export const stepFromGroupKey = (key) => { + const withoutPrefix = key.slice(GROUP_KEY_PREFIX.length); + const separatorIndex = withoutPrefix.indexOf(SLOT_SEPARATOR); + const stepPart = + separatorIndex === -1 + ? withoutPrefix + : withoutPrefix.slice(0, separatorIndex); + return Number(stepPart); +}; + +export const slotFromGroupKey = (key) => { + const withoutPrefix = key.slice(GROUP_KEY_PREFIX.length); + const separatorIndex = withoutPrefix.indexOf(SLOT_SEPARATOR); + return separatorIndex === -1 + ? null + : withoutPrefix.slice(separatorIndex + SLOT_SEPARATOR.length); +}; /** ColumnRef -> synthetic key */ export const refToKey = (ref) => - ref.kind === "raw" ? ref.name : groupKey(ref.step); + ref.kind === "raw" ? ref.name : groupKey(ref.step, ref.slot ?? null); /** synthetic key -> ColumnRef */ -export const keyToRef = (key) => - isGroupKey(key) - ? { kind: "group", step: stepFromGroupKey(key) } - : { kind: "raw", name: key }; +export const keyToRef = (key) => { + if (!isGroupKey(key)) return { kind: "raw", name: key }; + const step = stepFromGroupKey(key); + const slot = slotFromGroupKey(key); + return slot == null ? { kind: "group", step } : { kind: "group", step, slot }; +}; + +// A step's declared output, normalized to a list of slots — even a +// homogeneous step (the common case) is one "slot" with slot: null, so +// every caller iterates the same shape regardless of how many there are. +// Falls back defensively for a step that predates outputSlots. +function stepOutputSlots(step) { + if (Array.isArray(step?.outputSlots) && step.outputSlots.length > 0) { + return step.outputSlots; + } + return [ + { + slot: null, + type: step?.outputType ?? null, + dtype: step?.outputDtype ?? null, + }, + ]; +} /** * Every column key a session's preprocessing sequence can be scoped over, * up to (and not including) `uptoStep`: every raw dataset column, plus one - * group key per converter step before it. Passing no `uptoStep` includes - * every configured step (used once a sequence is final and being displayed, - * e.g. in SelectColumnsStep, where every step is already "before" the + * group key per declared slot of every converter step before it (usually + * one key per step; more than one only when that step's scope mixed + * column types). Passing no `uptoStep` includes every configured step + * (used once a sequence is final and being displayed, e.g. in + * SelectColumnsStep, where every step is already "before" the * column-selection step that comes after all of them). */ export function buildColumnKeysAndTypes({ @@ -50,14 +100,102 @@ export function buildColumnKeysAndTypes({ for (let index = 0; index < limit; index += 1) { const step = steps[index]; - const key = groupKey(index); - columnTypes[key] = { - type: step.outputType || null, - dtype: step.outputDtype || null, - }; - optionLabels[key] = `${step.converter}: output`; - allKeys.push(key); + stepOutputSlots(step).forEach(({ slot, type, dtype }) => { + const key = groupKey(index, slot); + columnTypes[key] = { type: type || null, dtype: dtype || null }; + optionLabels[key] = slot + ? `${step.converter}: output (${slot})` + : `${step.converter}: output`; + allKeys.push(key); + }); } return { allKeys, columnTypes, optionLabels }; } + +// SimpleImputer only preserves its input column's type for the +// "most_frequent"/"constant" strategies (no arithmetic performed); "mean"/ +// "median" always produce something numeric, but which exact numeric type +// depends on a statistic computed from real data, so there's nothing to +// resolve ahead of a fit — tool.metadata.output_type ("Float") already +// covers that case correctly, since every allowed_types list in this +// codebase that includes Integer also includes Float (they're always +// interchangeable for compatibility checks). +function simpleImputerPreservesType(toolName, params) { + if (toolName !== "SimpleImputer") return false; + const strategy = params?.strategy; + return strategy === "most_frequent" || strategy === "constant"; +} + +/** + * The output slot(s) to record for a converter step once its scope is + * known — always an array of {slot, type, dtype}, even when there's just + * one (slot: null, meaning "the whole group", exactly like before slots + * existed). + * + * Most converters' declared output_type/output_dtype (metadata computed + * backend-side from a bare, never-fitted instance) is the best available + * answer regardless of scope, and stays a single slot. But a converter + * that only keeps or drops whole columns unchanged (backend + * PRESERVES_INPUT_TYPE, e.g. feature selection, VarianceThreshold, or + * SimpleImputer with "most_frequent"/"constant") has an output type + * that's actually already knowable — it's just whatever the chosen + * scope's own column type already is. When that scope mixes distinct + * types, this splits into one slot per distinct type instead of falling + * back to a single guess, matching exactly what SessionPreprocessor. + * _classify_by_type computes for real once the session is created. + */ +export function resolveDeclaredOutputSlots({ + tool, + params, + scope, + datasetTypes, + preprocessing, +}) { + const fallback = [ + { + slot: null, + type: tool?.metadata?.output_type || null, + dtype: tool?.metadata?.output_dtype || null, + }, + ]; + + const preservesInputType = + Boolean(tool?.metadata?.preserves_input_type) || + simpleImputerPreservesType(tool?.name, params); + + if (!preservesInputType || !scope?.length) return fallback; + + const { columnTypes } = buildColumnKeysAndTypes({ + datasetTypes, + preprocessing, + }); + const scopeEntries = scope.map((ref) => columnTypes[refToKey(ref)]); + if (scopeEntries.some((entry) => !entry?.type)) return fallback; + + const distinctTypes = [...new Set(scopeEntries.map((entry) => entry.type))]; + + const dtypeFor = (type) => { + const dtypes = [ + ...new Set( + scopeEntries + .filter((entry) => entry.type === type) + .map((entry) => entry.dtype) + .filter(Boolean), + ), + ]; + return dtypes.length === 1 ? dtypes[0] : null; + }; + + if (distinctTypes.length === 1) { + return [ + { slot: null, type: distinctTypes[0], dtype: dtypeFor(distinctTypes[0]) }, + ]; + } + + return distinctTypes.map((type) => ({ + slot: type, + type, + dtype: dtypeFor(type), + })); +} diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js index ff43497e1..27f9b6f55 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -2,9 +2,11 @@ import { groupKey, isGroupKey, stepFromGroupKey, + slotFromGroupKey, refToKey, keyToRef, buildColumnKeysAndTypes, + resolveDeclaredOutputSlots, } from "./sessionColumnRefs"; describe("sessionColumnRefs", () => { @@ -18,22 +20,36 @@ describe("sessionColumnRefs", () => { expect(keyToRef(refToKey(ref))).toEqual(ref); }); + it("round-trips a slotted group ColumnRef through a key", () => { + const ref = { kind: "group", step: 0, slot: "Categorical" }; + expect(keyToRef(refToKey(ref))).toEqual(ref); + }); + it("recognizes group keys and extracts their step", () => { expect(isGroupKey(groupKey(3))).toBe(true); expect(isGroupKey("age")).toBe(false); expect(stepFromGroupKey(groupKey(3))).toBe(3); }); + it("extracts both step and slot from a slotted group key", () => { + const key = groupKey(2, "Categorical"); + expect(stepFromGroupKey(key)).toBe(2); + expect(slotFromGroupKey(key)).toBe("Categorical"); + expect(slotFromGroupKey(groupKey(2))).toBeNull(); + }); + it("builds keys/types/labels for raw columns plus every prior step", () => { const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ datasetTypes: { age: { type: "Integer" }, text: { type: "Text" } }, preprocessing: [ { converter: "BagOfWordsConverter", - outputType: "Integer", - outputDtype: "int64", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + { + converter: "Binarizer", + outputSlots: [{ slot: null, type: "Integer", dtype: null }], }, - { converter: "Binarizer", outputType: "Integer" }, ], uptoStep: 1, }); @@ -49,11 +65,195 @@ describe("sessionColumnRefs", () => { const { allKeys } = buildColumnKeysAndTypes({ datasetTypes: { age: { type: "Integer" } }, preprocessing: [ - { converter: "BagOfWordsConverter", outputType: "Integer" }, - { converter: "Binarizer", outputType: "Integer" }, + { + converter: "BagOfWordsConverter", + outputSlots: [{ slot: null, type: "Integer" }], + }, + { + converter: "Binarizer", + outputSlots: [{ slot: null, type: "Integer" }], + }, ], }); expect(allKeys).toEqual(["age", "__group__0", "__group__1"]); }); + + it("offers one key per declared slot for a step with a heterogeneous scope", () => { + const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ + datasetTypes: {}, + preprocessing: [ + { + converter: "SimpleImputer", + outputSlots: [ + { slot: "Integer", type: "Integer", dtype: "int64" }, + { slot: "Categorical", type: "Categorical", dtype: null }, + ], + }, + ], + }); + + const integerKey = groupKey(0, "Integer"); + const categoricalKey = groupKey(0, "Categorical"); + expect(allKeys).toEqual([integerKey, categoricalKey]); + expect(columnTypes[integerKey]).toEqual({ + type: "Integer", + dtype: "int64", + }); + expect(columnTypes[categoricalKey]).toEqual({ + type: "Categorical", + dtype: null, + }); + expect(optionLabels[integerKey]).toBe("SimpleImputer: output (Integer)"); + expect(optionLabels[categoricalKey]).toBe( + "SimpleImputer: output (Categorical)", + ); + }); + + it("falls back to a single unslotted key for a step predating outputSlots", () => { + const { allKeys, columnTypes } = buildColumnKeysAndTypes({ + datasetTypes: {}, + preprocessing: [ + { converter: "Binarizer", outputType: "Integer", outputDtype: "int64" }, + ], + }); + + expect(allKeys).toEqual(["__group__0"]); + expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); + }); + + describe("resolveDeclaredOutputSlots", () => { + const datasetTypes = { + age: { type: "Integer", dtype: "int64" }, + score: { type: "Float", dtype: "float64" }, + name: { type: "Text", dtype: "string" }, + }; + + it("falls back to the converter's declared metadata by default", () => { + const tool = { + name: "Binarizer", + metadata: { output_type: "Integer", output_dtype: "int64" }, + }; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [{ kind: "raw", name: "age" }], + datasetTypes, + preprocessing: [], + }); + expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); + }); + + it("uses the scope's real type when the converter preserves input type", () => { + const tool = { + name: "SelectKBest", + metadata: { output_type: "Float", preserves_input_type: true }, + }; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [{ kind: "raw", name: "age" }], + datasetTypes, + preprocessing: [], + }); + expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); + }); + + it("splits into one slot per distinct type when a preserving converter's scope mixes types", () => { + const tool = { + name: "VarianceThreshold", + metadata: { output_type: "Float", preserves_input_type: true }, + }; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [ + { kind: "raw", name: "age" }, + { kind: "raw", name: "name" }, + ], + datasetTypes, + preprocessing: [], + }); + expect(result).toEqual( + expect.arrayContaining([ + { slot: "Integer", type: "Integer", dtype: "int64" }, + { slot: "Text", type: "Text", dtype: "string" }, + ]), + ); + expect(result).toHaveLength(2); + }); + + it("treats SimpleImputer as type-preserving only for most_frequent/constant", () => { + const tool = { + name: "SimpleImputer", + metadata: { output_type: "Float", output_dtype: null }, + }; + const scope = [{ kind: "raw", name: "name" }]; + + const mostFrequent = resolveDeclaredOutputSlots({ + tool, + params: { strategy: "most_frequent" }, + scope, + datasetTypes, + preprocessing: [], + }); + expect(mostFrequent).toEqual([ + { slot: null, type: "Text", dtype: "string" }, + ]); + + const mean = resolveDeclaredOutputSlots({ + tool, + params: { strategy: "mean" }, + scope, + datasetTypes, + preprocessing: [], + }); + expect(mean).toEqual([{ slot: null, type: "Float", dtype: null }]); + }); + + it("resolves a preserving converter's scope through a chained group ref", () => { + const tool = { + name: "SelectKBest", + metadata: { output_type: "Float", preserves_input_type: true }, + }; + const preprocessing = [ + { + converter: "BagOfWordsConverter", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + ]; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [{ kind: "group", step: 0 }], + datasetTypes, + preprocessing, + }); + expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); + }); + + it("resolves a preserving converter's scope through one specific slot of a chained group ref", () => { + const tool = { + name: "SelectKBest", + metadata: { output_type: "Float", preserves_input_type: true }, + }; + const preprocessing = [ + { + converter: "SimpleImputer", + outputSlots: [ + { slot: "Integer", type: "Integer", dtype: "int64" }, + { slot: "Categorical", type: "Categorical", dtype: null }, + ], + }, + ]; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [{ kind: "group", step: 0, slot: "Integer" }], + datasetTypes, + preprocessing, + }); + expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); + }); + }); }); diff --git a/tests/back/api/test_model_session_api.py b/tests/back/api/test_model_session_api.py index c6167a959..d7f018847 100644 --- a/tests/back/api/test_model_session_api.py +++ b/tests/back/api/test_model_session_api.py @@ -166,6 +166,42 @@ def test_validate_columns_rejects_a_group_ref_with_an_incompatible_type( assert response.json()["dataset_status"] == "invalid" +def test_validate_columns_accepts_a_slotted_group_ref_matching_the_task_type( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": [], + "outputs_columns": ["Species"], + "input_refs": [{"kind": "group", "step": 0, "slot": "Integer"}], + "converter_output_types": {"0:Integer": "Integer", "0:Categorical": "Text"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "valid" + + +def test_validate_columns_rejects_a_slotted_group_ref_with_an_incompatible_type( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": [], + "outputs_columns": ["Species"], + "input_refs": [{"kind": "group", "step": 0, "slot": "Categorical"}], + "converter_output_types": {"0:Integer": "Integer", "0:Categorical": "Text"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "invalid" + + def test_bulk_delete_model_sessions(client: TestClient, dataset_1: Dataset) -> None: created_ids = [] for name in ["bulk_delete_session_1", "bulk_delete_session_2"]: diff --git a/tests/back/converters/test_base_converter_metadata.py b/tests/back/converters/test_base_converter_metadata.py index 58ed056e7..1191b67fd 100644 --- a/tests/back/converters/test_base_converter_metadata.py +++ b/tests/back/converters/test_base_converter_metadata.py @@ -186,3 +186,39 @@ def transform(self, x, y=None): meta = _RequiresArgConverter.get_metadata() assert meta["output_type"] is None assert meta["output_dtype"] is None + + +def test_get_metadata_preserves_input_type_defaults_to_false(): + meta = _FloatIntConverter.get_metadata() + assert meta["preserves_input_type"] is False + + +def test_get_metadata_reports_preserves_input_type_when_declared(): + class _SelectionLikeConverter(BaseConverter): + SCHEMA = None + metadata = {"allowed_types": [Float, Integer]} + PRESERVES_INPUT_TYPE = True + + def get_output_type(self, column_name=None): + return None + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + meta = _SelectionLikeConverter.get_metadata() + assert meta["preserves_input_type"] is True + + +def test_feature_selection_and_variance_threshold_declare_preserves_input_type(): + from DashAI.back.converters.category.feature_selection import ( + FeatureSelectionConverter, + ) + from DashAI.back.converters.scikit_learn.variance_threshold import ( + VarianceThreshold, + ) + + assert FeatureSelectionConverter.get_metadata()["preserves_input_type"] is True + assert VarianceThreshold.get_metadata()["preserves_input_type"] is True diff --git a/tests/back/preprocessing/test_column_ref.py b/tests/back/preprocessing/test_column_ref.py index 84b32538a..18510c20a 100644 --- a/tests/back/preprocessing/test_column_ref.py +++ b/tests/back/preprocessing/test_column_ref.py @@ -25,6 +25,49 @@ def test_group_ref_to_an_unfit_step_raises_key_error(): resolve_refs([GroupColumnRef(step=0)], {}) +def test_slotted_group_ref_resolves_to_only_that_types_columns(): + refs = [GroupColumnRef(step=0, slot="Categorical")] + resolved_columns = {0: ["most_frequent_city", "imputed_age"]} + resolved_slots = { + 0: { + "Categorical": ["most_frequent_city"], + "Integer": ["imputed_age"], + } + } + assert resolve_refs(refs, resolved_columns, resolved_slots) == [ + "most_frequent_city" + ] + + +def test_slotted_group_ref_with_no_resolved_slots_raises_key_error(): + with pytest.raises(KeyError): + resolve_refs([GroupColumnRef(step=0, slot="Categorical")], {0: ["x"]}) + + +def test_slotted_group_ref_to_a_type_the_step_never_produced_raises_key_error(): + with pytest.raises(KeyError): + resolve_refs( + [GroupColumnRef(step=0, slot="Text")], + {0: ["x"]}, + {0: {"Integer": ["x"]}}, + ) + + +def test_unslotted_group_ref_still_resolves_to_every_column_when_slots_exist(): + refs = [GroupColumnRef(step=0)] + resolved_columns = {0: ["most_frequent_city", "imputed_age"]} + resolved_slots = { + 0: { + "Categorical": ["most_frequent_city"], + "Integer": ["imputed_age"], + } + } + assert resolve_refs(refs, resolved_columns, resolved_slots) == [ + "most_frequent_city", + "imputed_age", + ] + + def test_sequence_rejects_a_step_referencing_itself_or_later(): sequence = ConverterSequence( steps=[ @@ -63,6 +106,12 @@ def test_parse_column_refs_round_trips_json_dicts(): assert parsed == [RawColumnRef(name="age"), GroupColumnRef(step=2)] +def test_parse_column_refs_round_trips_a_slotted_group_ref(): + raw = [{"kind": "group", "step": 0, "slot": "Categorical"}] + parsed = parse_column_refs(raw) + assert parsed == [GroupColumnRef(step=0, slot="Categorical")] + + def test_converter_sequence_model_dump_round_trips(): sequence = ConverterSequence( steps=[ diff --git a/tests/back/preprocessing/test_session_preprocessor.py b/tests/back/preprocessing/test_session_preprocessor.py index 6fbdec08c..e2bd22c6c 100644 --- a/tests/back/preprocessing/test_session_preprocessor.py +++ b/tests/back/preprocessing/test_session_preprocessor.py @@ -104,6 +104,44 @@ def transform(self, x, y=None): return to_dashai_dataset(frame, types=types) +class _FakeTypeObj: + """Minimal stand-in for a DashAIDataType — only display_name() matters + to SessionPreprocessor._classify_by_type.""" + + def __init__(self, name): + self._name = name + + def display_name(self): + return self._name + + +class _MixedTypeConverter(BaseConverter): + """Mirrors SimpleImputer's most_frequent/FeatureSelectionConverter: + keeps every scope column's own value and declared type unchanged. Used + to test that fit_transform classifies real output columns into one slot + per distinct type when a scope mixes them, without needing a real + Categorical dataset column — the per-column type is just a hardcoded + mapping passed in as a param, exactly like a real converter would derive + it from its own fitted state. + """ + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + + def __init__(self, type_by_column=None, **kwargs): + self._type_by_column = type_by_column or {} + + def fit(self, x, y=None): + return self + + def get_output_type(self, column_name=None): + return _FakeTypeObj(self._type_by_column.get(column_name, "Integer")) + + def transform(self, x, y=None): + return x + + class _FakeRegistry: def __init__(self, classes): self._classes = classes @@ -208,6 +246,110 @@ def test_chained_step_can_reference_the_previous_steps_group(): assert len(resolved[1]) == 2 +def test_fit_transform_classifies_a_steps_real_output_columns_by_type(): + registry = _FakeRegistry({"MixedType": _MixedTypeConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="MixedType", + params={"type_by_column": {"age": "Integer", "city": "Categorical"}}, + scope=[RawColumnRef(name="age"), RawColumnRef(name="city")], + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + train = _dataset( + {"age": [1, 2, 3], "city": [10, 20, 10]}, + { + "age": {"type": "Integer", "dtype": "int64"}, + "city": {"type": "Integer", "dtype": "int64"}, + }, + ) + preprocessor.fit_transform({"train": train}) + + assert preprocessor.resolved_slots[0] == { + "Integer": ["age"], + "Categorical": ["city"], + } + + +def test_chained_step_can_reference_a_specific_slot_of_an_earlier_steps_group(): + registry = _FakeRegistry( + {"MixedType": _MixedTypeConverter, "Doubler": _DoublingConverter} + ) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="MixedType", + params={"type_by_column": {"age": "Integer", "city": "Categorical"}}, + scope=[RawColumnRef(name="age"), RawColumnRef(name="city")], + ), + ConverterStep( + converter="Doubler", + params={}, + scope=[GroupColumnRef(step=0, slot="Integer")], + ), + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + train = _dataset( + {"age": [1, 2, 3], "city": [10, 20, 10]}, + { + "age": {"type": "Integer", "dtype": "int64"}, + "city": {"type": "Integer", "dtype": "int64"}, + }, + ) + transformed, _ = preprocessor.fit_transform({"train": train}) + + # Doubler was scoped only to the "Integer" slot (age); city, the + # "Categorical" slot, is untouched. + assert transformed["train"].to_pandas()["age"].tolist() == [2, 4, 6] + assert transformed["train"].to_pandas()["city"].tolist() == [10, 20, 10] + + +def test_transform_only_resolves_a_slotted_scope_using_persisted_resolved_slots(): + registry = _FakeRegistry( + {"MixedType": _MixedTypeConverter, "Doubler": _DoublingConverter} + ) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="MixedType", + params={"type_by_column": {"age": "Integer", "city": "Categorical"}}, + scope=[RawColumnRef(name="age"), RawColumnRef(name="city")], + ), + ConverterStep( + converter="Doubler", + params={}, + scope=[GroupColumnRef(step=0, slot="Integer")], + ), + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + train = _dataset( + {"age": [1, 2, 3], "city": [10, 20, 10]}, + { + "age": {"type": "Integer", "dtype": "int64"}, + "city": {"type": "Integer", "dtype": "int64"}, + }, + ) + preprocessor.fit_transform({"train": train}) + + new_data = _dataset( + {"age": [5], "city": [99]}, + { + "age": {"type": "Integer", "dtype": "int64"}, + "city": {"type": "Integer", "dtype": "int64"}, + }, + ) + result = preprocessor.transform_only({"train": new_data}) + + assert result["train"].to_pandas()["age"].tolist() == [10] + assert result["train"].to_pandas()["city"].tolist() == [99] + + def test_transform_only_applies_an_already_fitted_sequence_without_refitting(): registry = _FakeRegistry({"Doubler": _DoublingConverter}) sequence = ConverterSequence( From ec786724eb934c8cae395ff7d9f389ee354ccc9a Mon Sep 17 00:00:00 2001 From: Creylay Date: Thu, 10 Sep 2026 13:31:56 -0300 Subject: [PATCH 12/39] feat: Implement rawColumnsNeededFor function to resolve dataset column references for manual predictions --- .../models/ManualPredictionsTable.jsx | 29 ++++++-- .../models/modelSession/sessionColumnRefs.js | 29 ++++++++ .../modelSession/sessionColumnRefs.test.js | 69 +++++++++++++++++++ 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/DashAI/front/src/components/models/ManualPredictionsTable.jsx b/DashAI/front/src/components/models/ManualPredictionsTable.jsx index 76646adcb..84054d0f4 100644 --- a/DashAI/front/src/components/models/ManualPredictionsTable.jsx +++ b/DashAI/front/src/components/models/ManualPredictionsTable.jsx @@ -38,6 +38,7 @@ import { } from "../../api/predict"; import { enqueuePredictionJob } from "../../api/job"; import { getModelSessionById } from "../../api/modelSession"; +import { rawColumnsNeededFor } from "./modelSession/sessionColumnRefs"; import { startJobPolling } from "../../utils/jobPoller"; import { getTargetDecimals, @@ -245,13 +246,33 @@ export default function ManualPredictionsTable({ }; }, [run, session]); - const inputColumns = modelSession?.input_columns ?? EMPTY_ARRAY; + // Manual prediction submits raw values that the backend runs through the + // session's persisted preprocessor before predicting — it only ever + // accepts real dataset columns (BaseTask.process_manual_input rejects + // anything else), never a converter's already-resolved output name like + // "pca_1". A session with no preprocessing has input_column_refs already + // wrapping input_columns 1:1, so this resolves to the same thing then. + const inputColumns = useMemo(() => { + if (!modelSession) return EMPTY_ARRAY; + const refs = modelSession.input_column_refs; + if (!refs || refs.length === 0) + return modelSession.input_columns ?? EMPTY_ARRAY; + return rawColumnsNeededFor(refs, modelSession.preprocessing?.steps || []); + }, [modelSession]); const createEmptyRow = useCallback(() => { if (!inputSample || inputColumns.length === 0) return {}; - const randomIndex = Math.floor( - Math.random() * inputSample[inputColumns[0]].length, + // Defensive: a column missing from the sample (shouldn't happen once + // inputColumns is raw-keyed to match inputSample/inputTypes, but this is + // exactly the shape of bug that used to crash the whole tab) falls back + // to an empty prefill instead of throwing — the user can still type a + // value by hand. + const sampleColumn = inputColumns.find( + (col) => inputSample[col]?.length > 0, ); + const randomIndex = sampleColumn + ? Math.floor(Math.random() * inputSample[sampleColumn].length) + : 0; const row = {}; inputColumns.forEach((col) => { const typeInfo = inputTypes[col]; @@ -264,7 +285,7 @@ export default function ManualPredictionsTable({ row[col] = typeInfo.categories[randomIndex % typeInfo.categories.length]; } else { - row[col] = inputSample[col][randomIndex]; + row[col] = inputSample[col]?.[randomIndex]; } }); return row; diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js index 395881d3f..0c505ac1f 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js @@ -199,3 +199,32 @@ export function resolveDeclaredOutputSlots({ dtype: dtypeFor(type), })); } + +/** + * Every RAW dataset column name needed to compute a list of ColumnRef, + * walking group refs back to their step's own scope recursively (a group + * ref's step may itself reference an earlier group, chained arbitrarily + * deep). This is what a caller must actually supply values for — e.g. + * manual prediction, where the backend only ever accepts real dataset + * columns as input (see BaseTask.process_manual_input), never a + * converter's resolved output name like "pca_1": it runs the raw values + * through the session's persisted preprocessor itself before predicting. + */ +export function rawColumnsNeededFor(refs, steps) { + const needed = []; + const seen = new Set(); + const visit = (ref) => { + if (ref.kind === "raw") { + if (!seen.has(ref.name)) { + seen.add(ref.name); + needed.push(ref.name); + } + return; + } + const step = (steps || [])[ref.step]; + if (!step) return; + (step.scope || []).forEach(visit); + }; + (refs || []).forEach(visit); + return needed; +} diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js index 27f9b6f55..234524de1 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -7,6 +7,7 @@ import { keyToRef, buildColumnKeysAndTypes, resolveDeclaredOutputSlots, + rawColumnsNeededFor, } from "./sessionColumnRefs"; describe("sessionColumnRefs", () => { @@ -256,4 +257,72 @@ describe("sessionColumnRefs", () => { expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); }); }); + + describe("rawColumnsNeededFor", () => { + it("returns raw ref names as-is", () => { + const refs = [ + { kind: "raw", name: "age" }, + { kind: "raw", name: "score" }, + ]; + expect(rawColumnsNeededFor(refs, [])).toEqual(["age", "score"]); + }); + + it("resolves a group ref to its step's own raw scope", () => { + const refs = [{ kind: "group", step: 0 }]; + const steps = [ + { + converter: "BagOfWordsConverter", + scope: [{ kind: "raw", name: "text" }], + }, + ]; + expect(rawColumnsNeededFor(refs, steps)).toEqual(["text"]); + }); + + it("resolves a chained group ref recursively through an earlier step", () => { + const refs = [{ kind: "group", step: 1 }]; + const steps = [ + { + converter: "BagOfWordsConverter", + scope: [{ kind: "raw", name: "text" }], + }, + { converter: "PCA", scope: [{ kind: "group", step: 0 }] }, + ]; + expect(rawColumnsNeededFor(refs, steps)).toEqual(["text"]); + }); + + it("ignores slot when walking a group ref back to raw columns", () => { + const refs = [{ kind: "group", step: 0, slot: "Integer" }]; + const steps = [ + { + converter: "SimpleImputer", + scope: [ + { kind: "raw", name: "age" }, + { kind: "raw", name: "city" }, + ], + }, + ]; + // The slot only narrows which OUTPUT columns you get; fitting the + // step still needs every raw column in its scope, regardless. + expect(rawColumnsNeededFor(refs, steps)).toEqual(["age", "city"]); + }); + + it("de-duplicates a raw column needed by more than one ref", () => { + const refs = [ + { kind: "raw", name: "age" }, + { kind: "group", step: 0 }, + ]; + const steps = [ + { converter: "Doubler", scope: [{ kind: "raw", name: "age" }] }, + ]; + expect(rawColumnsNeededFor(refs, steps)).toEqual(["age"]); + }); + + it("skips a group ref pointing at a step that doesn't exist", () => { + const refs = [ + { kind: "raw", name: "age" }, + { kind: "group", step: 5 }, + ]; + expect(rawColumnsNeededFor(refs, [])).toEqual(["age"]); + }); + }); }); From de0fd15d9e661e9e1ed6710cddcbd3aa3e0b0daf Mon Sep 17 00:00:00 2001 From: Creylay Date: Thu, 10 Sep 2026 14:02:33 -0300 Subject: [PATCH 13/39] feat: Default input selection to last preprocessing step's output groups in SelectColumnsStep --- .../models/modelSession/SelectColumnsStep.jsx | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx index 6eb54bfea..1bf182567 100644 --- a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx +++ b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx @@ -19,6 +19,7 @@ import { isGroupKey, refToKey, keyToRef, + groupKey, buildColumnKeysAndTypes, } from "./sessionColumnRefs"; @@ -77,11 +78,26 @@ function SelectColumnsStep({ inputSelection.length === 0 && (!newExp.input_columns || newExp.input_columns.length === 0) ) { - setInputSelection( - rawColumnNames.length > 1 - ? rawColumnNames.slice(0, -1) - : [rawColumnNames[0]], - ); + const steps = newExp.preprocessing || []; + if (steps.length > 0) { + // With preprocessing configured, default to just the last step's + // output group(s) — the point of building a chain is usually to + // end up using its final result, not the raw columns it started + // from. Every raw column and every other group stays available to + // pick instead; this is only the starting default. + const lastIndex = steps.length - 1; + const slots = + steps[lastIndex].outputSlots?.length > 0 + ? steps[lastIndex].outputSlots + : [{ slot: null }]; + setInputSelection(slots.map(({ slot }) => groupKey(lastIndex, slot))); + } else { + setInputSelection( + rawColumnNames.length > 1 + ? rawColumnNames.slice(0, -1) + : [rawColumnNames[0]], + ); + } } if ( outputColumnNames.length === 0 && From 137b2490f44b214b61e0383fd1775e0daffae3d5 Mon Sep 17 00:00:00 2001 From: Creylay Date: Fri, 11 Sep 2026 08:45:54 -0300 Subject: [PATCH 14/39] feat(i18n): update translations for experiments and models --- .../src/utils/i18n/locales/de/datasets.json | 139 +++++------ .../utils/i18n/locales/de/experiments.json | 37 +-- .../src/utils/i18n/locales/de/models.json | 215 +++++++++-------- .../src/utils/i18n/locales/pt/datasets.json | 151 ++++++------ .../utils/i18n/locales/pt/experiments.json | 37 +-- .../src/utils/i18n/locales/pt/models.json | 227 +++++++++--------- .../src/utils/i18n/locales/zh/datasets.json | 133 +++++----- .../utils/i18n/locales/zh/experiments.json | 37 +-- .../src/utils/i18n/locales/zh/models.json | 213 ++++++++-------- 9 files changed, 614 insertions(+), 575 deletions(-) diff --git a/DashAI/front/src/utils/i18n/locales/de/datasets.json b/DashAI/front/src/utils/i18n/locales/de/datasets.json index 7e422e5be..5f259e2cb 100644 --- a/DashAI/front/src/utils/i18n/locales/de/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/de/datasets.json @@ -52,6 +52,7 @@ "uploadFile": "Datei hochladen" }, "error": { + "cannotSaveEmptyDataset": "Dataset kann nicht gespeichert werden: Alle Spalten wurden entfernt.", "converterFailed": "Beim Verarbeiten des Konverters ist ein Fehler aufgetreten.", "converterFailedWithInfo": "Fehler beim Verarbeiten des Konverters: {{error}}", "createConverterError": "Konverter konnte nicht erstellt werden", @@ -68,68 +69,52 @@ "failedToCreateDataset": "Der Datensatz konnte nicht erstellt werden. Bitte pruefe deine Datei und Konfiguration und versuche es erneut.", "failedToCreateDatasetFromNotebook": "Datensatz konnte nicht aus Notizbuch erstellt werden", "failedToCreateExplorer": "Explorer konnte nicht erstellt werden", + "failedToCreateFolder": "Ordner konnte nicht erstellt werden", "failedToDeleteDataset": "Datensatz konnte nicht gelöscht werden", "failedToDeleteDatasets": "Die ausgewählten Datensätze konnten nicht gelöscht werden", - "failedToCreateFolder": "Ordner konnte nicht erstellt werden", - "failedToUpdateFolder": "Ordner konnte nicht aktualisiert werden", "failedToDeleteFolder": "Ordner konnte nicht gelöscht werden", - "folderNameExists": "Ein Ordner mit diesem Namen existiert bereits", - "failedToMoveDataset": "Datensatz konnte nicht verschoben werden", "failedToDeleteNotebook": "Notizbuch konnte nicht gelöscht werden", "failedToDeleteNotebooks": "Die ausgewählten Notizbücher konnten nicht gelöscht werden", "failedToFetchNotebooks": "Notizbücher konnten nicht abgerufen werden", "failedToLoadDatasetInfo": "Datensatzinformationen konnten nicht abgerufen werden", + "failedToMoveDataset": "Datensatz konnte nicht verschoben werden", + "failedToResetExplorerResults": "Ursprüngliches Diagramm konnte nicht wiederhergestellt werden", "failedToUpdateDataset": "Datensatz konnte nicht aktualisiert werden", "failedToUpdateExplorerResults": "Explorer-Ergebnisse konnten nicht aktualisiert werden", - "failedToResetExplorerResults": "Ursprüngliches Diagramm konnte nicht wiederhergestellt werden", + "failedToUpdateFolder": "Ordner konnte nicht aktualisiert werden", "failedToUpdateNotebook": "Notizbuch konnte nicht aktualisiert werden", + "fetchingConverters": "Fehler beim Abrufen der verfügbaren Converter.", "fetchingDataloaders": "Fehler beim Abrufen kompatibler Datenlader.", "fetchingDatasetColumns": "Fehler beim Abrufen der Datensatzspalten.", "fetchingExplorersConverters": "Explorer/Konverter konnten nicht abgerufen werden", "fileTypeNotAllowed": "Dateityp für den ausgewählten Datenlader nicht zulässig.", + "folderNameExists": "Ein Ordner mit diesem Namen existiert bereits", "loadingDatasetPreview": "Fehler beim Laden der Datensatzvorschau", "noDatasetDataAvailable": "Keine Datensatzdaten verfügbar", "noDatasetFileAvailable": "Keine Datensatzdatei verfügbar", - "notebookNameEmpty": "Notizbuchname darf nicht leer sein", "noValidColumnsForExplorer": "Keine gültigen Spalten für diesen Explorer verfügbar.", "noValidColumnsWithDtypesMentioned": "Dieser Datensatz hat keine Spalten mit den erforderlichen Typen ({{dtypes}}).", + "notebookNameEmpty": "Notizbuchname darf nicht leer sein", "processConverterError": "Konverter konnte nicht verarbeitet werden", "requiredFieldsMissing": "Pflichtfelder fehlen", "requiresExactColumns_one": "Erfordert genau {{required}} gültige Spalte, aber {{available}} verfügbar.", "requiresExactColumns_other": "Erfordert genau {{required}} gültige Spalten, aber {{available}} verfügbar.", "requiresMinColumns_one": "Erfordert mindestens {{required}} gültige Spalte, aber nur {{available}} verfügbar.", "requiresMinColumns_other": "Erfordert mindestens {{required}} gültige Spalten, aber nur {{available}} verfügbar.", - "zipContentsNotCompatible": "ZIP enthält keine mit dem ausgewählten Datenlader kompatiblen Dateien", - "cannotSaveEmptyDataset": "Dataset kann nicht gespeichert werden: Alle Spalten wurden entfernt." + "zipContentsNotCompatible": "ZIP enthält keine mit dem ausgewählten Datenlader kompatiblen Dateien" }, "label": { - "task": "Aufgabe", + "aboutTool": "Über {{toolName}}", "all": "alle", + "allTypeChangesValid": "Alle Typänderungen sind gültig und können sicher angewendet werden.", "allowedDataTypes": "Zulässige Datentypen: <1><0>", "allowedValueTypes": "Zulässige Wertetypen: <1><0>", - "excludedDataTypes": "Ausgeschlossene Datentypen: {{dtypes}}", - "allTypeChangesValid": "Alle Typänderungen sind gültig und können sicher angewendet werden.", "analysisTools": "Analysewerkzeuge", "appearance": "Erscheinungsbild", "appliedTransformations": "Angewendete Transformationen:", "associatedDataset": "Verknüpfter Datensatz", "atLeastTwoColorStopsRequired": "Keine Farbstopps definiert. Bitte fügen Sie mindestens zwei Farbstopps hinzu.", "availableDatasets": "Verfügbare Datensätze", - "confirmDeleteFolder": "Ordner löschen", - "confirmDeleteFolderContent": "Möchten Sie den Ordner \"{{name}}\" wirklich löschen? Datensätze darin werden in keinen Ordner verschoben.", - "noFolder": "Kein Ordner", - "newFolder": "Neuer Ordner", - "folderName": "Ordnername", - "confirmDeleteDataset": "Sind Sie sicher, dass Sie den Datensatz \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmDeleteDatasetLinkedWarning": "Alle mit diesem Datensatz verknüpften Notizbücher und Sitzungen werden ebenfalls gelöscht.", - "confirmBulkDeleteDatasets_one": "Sind Sie sicher, dass Sie den ausgewählten Datensatz löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmBulkDeleteDatasets_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Datensätze löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "selectDatasetsToDelete": "Datensätze zum Löschen auswählen", - "confirmDeleteNotebook": "Sind Sie sicher, dass Sie das Notizbuch \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmBulkDeleteNotebooks": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Notizbücher löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmBulkDeleteNotebooks_one": "Sind Sie sicher, dass Sie das ausgewählte Notizbuch löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmBulkDeleteNotebooks_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Notizbücher löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "selectNotebooksToDelete": "Notizbücher zum Löschen auswählen", "avg": "Durchschn.", "avgLength": "Durchschnittliche Länge", "avgWordCount": "Durchschnittliche Wortanzahl", @@ -154,6 +139,7 @@ "classTargetColumn": "Klassen-/Zielspalte", "clickToUpload": "Zum Hochladen klicken", "color": "Farbe", + "colorStops": "Farbstopps", "colorbarBorderColor": "Farbleisten-Rahmenfarbe", "colorbarBorderWidth": "Farbleisten-Rahmenbreite", "colorbarTickFontColor": "Farbleisten-Skalierungsschriftfarbe", @@ -161,7 +147,7 @@ "colors": "Farben", "colorscale": "Farbskala", "colorscaleMode": "Farbskalenmodus", - "colorStops": "Farbstopps", + "columnInsights": "Spalteneinblicke", "columnName": "Spaltenname", "columnTypesDistribution": "Spaltentypenverteilung", "configureAndUpload": "Konfigurieren & Hochladen", @@ -170,10 +156,20 @@ "configureParametersStep": "Schritt {{step}}: Parameter konfigurieren", "configureScope": "Bereich konfigurieren", "configureToolTitle": "{{toolType}} konfigurieren: {{toolName}}", - "columnInsights": "Spalteneinblicke", + "confirmBulkDeleteDatasets_one": "Sind Sie sicher, dass Sie den ausgewählten Datensatz löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmBulkDeleteDatasets_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Datensätze löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmBulkDeleteNotebooks": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Notizbücher löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmBulkDeleteNotebooks_one": "Sind Sie sicher, dass Sie das ausgewählte Notizbuch löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmBulkDeleteNotebooks_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Notizbücher löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmDeleteDataset": "Sind Sie sicher, dass Sie den Datensatz \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmDeleteDatasetLinkedWarning": "Alle mit diesem Datensatz verknüpften Notizbücher und Sitzungen werden ebenfalls gelöscht.", + "confirmDeleteFolder": "Ordner löschen", + "confirmDeleteFolderContent": "Möchten Sie den Ordner \"{{name}}\" wirklich löschen? Datensätze darin werden in keinen Ordner verschoben.", + "confirmDeleteNotebook": "Sind Sie sicher, dass Sie das Notizbuch \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", "constantColumns": "Konstante Spalten", "convert": "Konvertieren", "converter": "Konverter", + "converterOutput": "Ausgabe", "correlation": "Korrelation", "correlationAnalysis": "Korrelationsanalyse", "correlations": "Korrelationen", @@ -181,43 +177,46 @@ "createNewNotebook": "Notizbuch", "createNewNotebookDescription": "Starten Sie eine neue Analysesitzung mit einem vorhandenen Datensatz.", "customArray": "Benutzerdefiniertes Array", - "dataloaderConfiguration": "Datenlader-Konfiguration", "dataQuality": "Datenqualität", "dataQualityScoreTooltip": "Der Datenqualitätswert wird auf Basis verschiedener Faktoren berechnet, einschließlich fehlender Werte, doppelter Zeilen und Datenkonsistenz. Ein höherer Wert zeigt bessere Datenqualität an.", "dataQualitySummary": "Datenqualitätszusammenfassung", + "dataType": "Datentyp", + "dataloaderConfiguration": "Datenlader-Konfiguration", "datasetDescription": "{{rows}} Zeilen, {{columns}} Spalten", "datasetLoading": "Datensatz wird geladen...", "datasetModule": "Datensatz-Modul", "datasetModuleSubtitle": "Laden Sie Ihre Datensätze hoch: Erkunden, analysieren und transformieren Sie Ihre Daten mit erweiterten explorativen Analysewerkzeugen. Erstellen Sie interaktive Notizbücher, generieren Sie Visualisierungen und wenden Sie Datentransformationen intuitiv an.", "datasetName": "Datensatzname", "datasetPreview": "Datensatzvorschau", - "aboutTool": "Über {{toolName}}", "datasetPreviewFor": "Notizbuch: {{name}} Vorschau", - "dataType": "Datentyp", + "decreasingColor": "Farbe für Rückgang", "deleteConverterConfirmation": "Sind Sie sicher, dass Sie den Konverter \"{{converter}}\" löschen möchten? Das Löschen dieses Konverters entfernt auch alle nachfolgenden Konverter und Explorer. Diese Aktion kann nicht rückgängig gemacht werden.", "deleteExplorerConfirmation": "Sind Sie sicher, dass Sie den Explorer \"{{explorer}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", "detailsForExplorer": "Details für Explorer: {{name}}", "dimensionIdx": "Dimension {{idx}}: {{label}}", - "dimensionsLabels": "Dimensionsbeschriftungen", "dimensionTitle": "Dimensionstitel", + "dimensionsLabels": "Dimensionsbeschriftungen", "distributionMetrics": "Verteilungsmetriken", "dragAndDropFileHere": "Datei hier ablegen", "dragToResize": "Ziehen zum Vergrößern und Anzeigen weiterer Inhalte", + "dropToolHere": "Hier ablegen zum Hinzufügen", "duplicatedRows": "Doppelte Zeilen", "editPlotLayout": "Plot-Layout bearbeiten", "endIndex": "Endindex", + "excludedDataTypes": "Ausgeschlossene Datentypen: {{dtypes}}", "explorationPath": "Explorationspfad", "explorationType": "Explorationstyp", "explore": "Erkunden", "explorer": "Explorer", + "exportCSV": "Als CSV herunterladen", "exportCardImage": "Karte als Bild", "exportChartImage": "Diagramm als Bild", - "exportCSV": "Als CSV herunterladen", "exportImage": "Herunterladen", "exportJSON": "Als JSON herunterladen", "exportMetrics": "Metriken als JSON exportieren", "fileSizeMB": "Dateigröße (MB)", "firstLastStopsAtExtremes": "Farbstopps müssen Positionen 0 am Anfang und 1 am Ende enthalten.", + "folderName": "Ordnername", "fontFamily": "Schriftfamilie", "foundDuplicateRows_one": "{{count}} doppelte Zeile im Datensatz gefunden", "foundDuplicateRows_other": "{{count}} doppelte Zeilen im Datensatz gefunden", @@ -228,17 +227,20 @@ "highCardinality": "Hohe Kardinalität", "highCardinalityDetected": "Hohe Kardinalität erkannt in: {{columns}}", "ifYourDatasetHaveSplits": "Wenn Ihr Datensatz Aufteilungen hat, laden Sie ihn als ZIP-Datei hoch", + "importFromHub": "Datei-Hub", + "importFromHubDescription": "Datensätze aus externen Quellen wie HuggingFace und OpenML durchsuchen und herunterladen.", + "increasingColor": "Farbe für Anstieg", "indices": "Indizes (kommagetrennt oder 'all' eingeben)", "inferenceRows": "Inferenzzeilen", "inferenceRowsDescription": "Anzahl der Zeilen für Vorschau/Typinferenz (Minimum 2).", "inferredConfiguration": "Typinferenz-Konfiguration", "insightConstantColumn": "Diese Spalte hat nur einen eindeutigen Wert. Sie liefert keine Informationen für die Analyse.", - "insightHighCardinality": "Diese kategorische Spalte hat mehr als 100 eindeutige Werte. Erwägen Sie Gruppierung oder Kodierung.", "insightEmptyMessage": "Keine Probleme erkannt. Spalteneinblicke erscheinen hier, wenn potenzielle Beobachtungen gefunden werden.", + "insightHighCardinality": "Diese kategorische Spalte hat mehr als 100 eindeutige Werte. Erwägen Sie Gruppierung oder Kodierung.", "insightHighNanRatio": "{{value}}% der Werte fehlen in dieser Spalte. Erwägen Sie Imputation oder Entfernung.", "insightLowUniqueness": "Niedrige Eindeutigkeitsrate ({{value}}%). Könnte eine kategorische Variable sein, die fälschlicherweise als Text klassifiziert wurde.", - "insightOutliers_one": "{{count}} potenzieller Ausreißerwert erkannt (IQR-Methode).", "insightOutliers_many": "{{count}} potenzielle Ausreißerwerte erkannt (IQR-Methode).", + "insightOutliers_one": "{{count}} potenzieller Ausreißerwert erkannt (IQR-Methode).", "insightOutliers_other": "{{count}} potenzielle Ausreißerwerte erkannt (IQR-Methode).", "insightPossibleId": "Alle Werte sind eindeutig. Diese Spalte könnte ein Bezeichner sein und für die Modellierung nicht nützlich.", "insightSkewed": "Rechtsschiefe Verteilung (Schiefe: {{value}}). Erwägen Sie eine Logarithmustransformation.", @@ -260,19 +262,15 @@ "legendYPosition": "Legenden-Y-Position", "lengthDistribution": "Längenverteilung", "lengthMetrics": "Längenmetriken", - "lowerBound": "Untergrenze", + "lineColor": "Linienfarbe", "lowUniquenessWarning": "Warnung: Diese Textspalte hat eine sehr niedrige Eindeutigkeitsrate. Dies könnte eine kategorische Variable sein, die fälschlicherweise als Text klassifiziert wurde.", + "lowerBound": "Untergrenze", "marginBottom": "Unterer Rand", "marginLeft": "Linker Rand", "marginRight": "Rechter Rand", - "margins": "Ränder", "marginTop": "Oberer Rand", + "margins": "Ränder", "markerColor": "Markierungsfarbe", - "lineColor": "Linienfarbe", - "increasingColor": "Farbe für Anstieg", - "decreasingColor": "Farbe für Rückgang", - "totalsColor": "Farbe für Summen", - "transparent": "Transparent", "max": "Max", "maxLength": "Maximale Länge", "mean": "Mittelwert", @@ -286,29 +284,28 @@ "missingValuesDetected": "Fehlende Werte in Spalten erkannt: {{columns}}", "missingValuesOverview": "Übersicht fehlender Werte", "mostFrequent": "Am häufigsten", - "nameYourNotebook": "Notizbuch benennen", "nComponentsColumnInfo": "Sie haben {{n}} Spalte(n) ausgewählt. n_components muss kleiner oder gleich {{n}} sein. Reduzieren Sie n_components oder wählen Sie mehr Spalten aus, um Fehler zu vermeiden.", + "nameYourNotebook": "Notizbuch benennen", "newDatasetCreatedWithTransformations": "Ein neuer Datensatz wird mit diesen Transformationen erstellt. Er kann mit anderen Modulen verwendet werden, ohne das Original zu beeinflussen.", + "newFolder": "Neuer Ordner", "noDataQualityIssuesDetected": "Keine Datenqualitätsprobleme erkannt", - "noDuplicateRows": "Keine doppelten Zeilen im Datensatz gefunden.", - "qualityIssuesFound_one": "{{count}} Problem erkannt", - "qualityIssuesFound_other": "{{count}} Probleme erkannt", "noDataset": "Kein Datensatz", "noDatasetsAvailable": "Keine Datensätze verfügbar", + "noDuplicateRows": "Keine doppelten Zeilen im Datensatz gefunden.", "noExplorersOrConverters": "Beginnen Sie mit der Exploration, indem Sie Ihren ersten Explorer oder Konverter hinzufügen!", + "noFolder": "Kein Ordner", "noMissingValues": "Keine fehlenden Werte im Datensatz gefunden.", - "none": "keine", "noStrongCorrelationsFound": "Keine starken Korrelationen erkannt", + "noToolsMatched": "Keine Werkzeuge gefunden, die Ihrer Suche entsprechen.", + "noTransformationsApplied": "Keine Transformationen angewendet.", + "noTransformationsAppliedYet": "Noch keine Transformationen angewendet.", + "none": "keine", "notebookCreationNote": "Eine Kopie des ausgewählten Datensatzes wird erstellt, um im Notizbuch zu arbeiten, ohne das Original zu verändern.", "notebookDescription": "Notizbuchbeschreibung", "notebookHistory": "Notizbuchverlauf: {{notebook}}", "notebookInformation": "Notizbuchinformationen", "notebookName": "Notizbuchname", "notebooks": "Notizbücher", - "noToolsMatched": "Keine Werkzeuge gefunden, die Ihrer Suche entsprechen.", - "dropToolHere": "Hier ablegen zum Hinzufügen", - "noTransformationsApplied": "Keine Transformationen angewendet.", - "noTransformationsAppliedYet": "Noch keine Transformationen angewendet.", "numericalAnalysis": "Numerische Analyse", "outliers": "Ausreißer", "overview": "Übersicht", @@ -316,11 +313,15 @@ "position": "Position", "possibleIDColumns": "Mögliche ID-Spalten", "presetScale": "Voreingestellte Skala", - "processingTitle": "Ihr Dataset wird verarbeitet...", + "previewRows": "Vorschauzeilen", + "previewRowsDescription": "Anzahl der Zeilen für die Vorschau (Minimum 2).", "processingMessage": "Dies kann je nach Größe Ihrer Daten einige Momente dauern.", + "processingTitle": "Ihr Dataset wird verarbeitet...", "proportion": "Anteil", "q1": "Q1", "q3": "Q3", + "qualityIssuesFound_one": "{{count}} Problem erkannt", + "qualityIssuesFound_other": "{{count}} Probleme erkannt", "qualityScore": "Qualitätswert: {{value}}{{unit}}", "range": "Bereich", "requiredColumns": "Erforderliche Spalten", @@ -336,6 +337,7 @@ "saveProcessedDataset": "Verarbeiteten Datensatz speichern", "scopeColumns": "Bereich - Spalten", "scopeRows": "Bereich - Zeilen", + "searchConverters": "Converter suchen", "searchDatasetsNotebooks": "Datensätze und Notizbücher suchen", "searchExplorersConverters": "Explorer/Konverter suchen", "selectColorscale": "Farbskala auswählen", @@ -345,6 +347,14 @@ "selectDataset": "Datensatz auswählen", "selectDatasetFirst": "Zuerst einen Datensatz auswählen", "selectDatasetForNotebook": "Datensatz für das Notizbuch auswählen", + "selectDatasetsToDelete": "Datensätze zum Löschen auswählen", + "selectNotebookToAccessAnalysisTools": "Wählen Sie ein Notizbuch aus, um auf Analysewerkzeuge zuzugreifen.", + "selectNotebooksToDelete": "Notizbücher zum Löschen auswählen", + "selectScopeDescriptionColumns": "Hier konfigurieren Sie, auf welche Spalten der Konverter angewendet werden soll.", + "selectScopeDescriptionRows": "Hier konfigurieren Sie, auf welche Zeilen der Konverter angewendet werden soll.", + "selectScopeStep": "Schritt {{step}}: Bereich auswählen", + "selectTargetColumnDescription": "Wählen Sie eine Spalte als Zielvariable für überwachtes Lernen aus.", + "selectUploadMethod": "Wählen Sie eine Methode zum Hochladen Ihrer Daten", "selectedColumns_one": "{{count}} Spalte ausgewählt", "selectedColumns_other": "{{count}} Spalten ausgewählt", "selectedDataloaderConfiguration": "{{dataloader}}-Konfiguration", @@ -352,27 +362,22 @@ "selectedOrder": "Ausgewählte Reihenfolge", "selectedRows": "Ausgewählte Zeilen: {{value}}", "selectionMode": "Auswahlmodus", - "selectNotebookToAccessAnalysisTools": "Wählen Sie ein Notizbuch aus, um auf Analysewerkzeuge zuzugreifen.", - "selectScopeDescriptionColumns": "Hier konfigurieren Sie, auf welche Spalten der Konverter angewendet werden soll.", - "selectScopeDescriptionRows": "Hier konfigurieren Sie, auf welche Zeilen der Konverter angewendet werden soll.", - "selectScopeStep": "Schritt {{step}}: Bereich auswählen", - "selectTargetColumnDescription": "Wählen Sie eine Spalte als Zielvariable für überwachtes Lernen aus.", - "selectUploadMethod": "Wählen Sie eine Methode zum Hochladen Ihrer Daten", "shapeIndicators": "Formindikatoren", "showGrid": "{{axis}}-Achse Gitter anzeigen", - "showingRowsInference": "{{sampleLength}} von {{previewRowCount}} Zeilen für die Typinferenz analysiert.", - "showingRowsPreview": "{{sampleLength}} von {{previewRowCount}} Zeilen werden angezeigt.", "showLegend": "Legende anzeigen", "showMore": "Mehr anzeigen ({{count}} verbleibend)", "showMore_one": "Mehr anzeigen ({{count}} verbleibend)", "showMore_other": "Mehr anzeigen ({{count}} verbleibend)", "showZeroLine": "{{axis}}-Achse Nulllinie anzeigen", + "showingRowsInference": "{{sampleLength}} von {{previewRowCount}} Zeilen für die Typinferenz analysiert.", + "showingRowsPreview": "{{sampleLength}} von {{previewRowCount}} Zeilen werden angezeigt.", "skewness": "Schiefe", "someTypeChangesCannotBeApplied": "Einige Typänderungen können nicht angewendet werden:", "startIndex": "Startindex", "stdDev": "Standardabweichung", "strongCorrelations": "Starke Korrelationen", "targetColumn": "Zielspalte", + "task": "Aufgabe", "text": "Text", "tickLabels": "Skalierungsbeschriftungen", "tickLabelsHelper": "Eine Beschriftung pro Zeile", @@ -387,35 +392,33 @@ "totalPercentage": "{{percentage}}% des Gesamten", "totalRows": "Gesamtzeilen", "totalRowsCount": "Gesamtzeilen: {{total}}", + "totalsColor": "Farbe für Summen", "tourDisabledMessage": "Zurück zur Startseite, um die Tour zu starten", "tourDisabledMessageNotebook": "Zurück zur Datensatzvisualisierung, um die Tour zu starten", "traceIdx": "Spur {{index}}: {{trace}}", + "transparent": "Transparent", "typeChangeWarnings": "Warnungen zu Typänderungen:", "typeToSearchDatasets": "Tippen Sie, um Datensätze zu suchen...", "unique": "Eindeutig", - "uniquenessFormula": "Eindeutigkeit = (Eindeutig ÷ Gesamt) x 100", "uniquePercentage": "{{percentage}}% Eindeutig", "uniqueValues": "Eindeutige Werte", + "uniquenessFormula": "Eindeutigkeit = (Eindeutig ÷ Gesamt) x 100", "unknownDataset": "Unbekannter Datensatz", "uploadAndConfigure": "Laden Sie Ihren Datensatz hoch und konfigurieren Sie Parameter", - "importFromHub": "Datei-Hub", - "importFromHubDescription": "Datensätze aus externen Quellen wie HuggingFace und OpenML durchsuchen und herunterladen.", "uploadDataset": "Datensatz", "uploadDatasetBeforeCreatingSession": "Sie müssen einen Datensatz hochladen, bevor Sie eine Sitzung erstellen. Gehen Sie zum Datensatz-Modul, um Ihre Daten hochzuladen.", "uploadDatasetDescription": "Importieren Sie Ihre Daten aus verschiedenen Quellen und Formaten", "uploadYourDataset": "Laden Sie Ihren Datensatz hoch", "upperBound": "Obergrenze", + "useNativeTypes": "Native Typen verwenden", + "useNativeTypesDescription": "Spaltentypen aus der Datei verwenden statt statistische Inferenz. Schneller und exakt für selbstbeschreibende Formate.", "validateTypeChanges": "Typänderungen validieren", "validatingTypeChanges": "Typänderungen werden validiert...", "valueDistribution": "Werteverteilung", "valueType": "Wertetyp", "viewMode": "Ansichtsmodus", "xAxis": "X-Achse", - "yAxis": "Y-Achse", - "previewRows": "Vorschauzeilen", - "previewRowsDescription": "Anzahl der Zeilen für die Vorschau (Minimum 2).", - "useNativeTypes": "Native Typen verwenden", - "useNativeTypesDescription": "Spaltentypen aus der Datei verwenden statt statistische Inferenz. Schneller und exakt für selbstbeschreibende Formate." + "yAxis": "Y-Achse" }, "message": { "columnTypesUpdated": "Spaltentypen erfolgreich aktualisiert", diff --git a/DashAI/front/src/utils/i18n/locales/de/experiments.json b/DashAI/front/src/utils/i18n/locales/de/experiments.json index 3f091f917..71a2580a8 100644 --- a/DashAI/front/src/utils/i18n/locales/de/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/de/experiments.json @@ -34,53 +34,57 @@ "label": { "addModelsToExperiment": "Modelle zum Experiment hinzufügen", "addOptimizersToExperiment": "Optimierer zum Experiment hinzufügen", + "cardinalityAtLeast": "mindestens {{min}}", + "cardinalityBetween": "{{min}} bis {{max}}", "columnsInvalidRequirements": "Aktuelle Ein- und Ausgabespalten erfüllen nicht die Anforderungen von {{taskName}}", "columnsValidRequirements": "Aktuelle Ein- und Ausgabespalten erfüllen die Anforderungen von {{taskName}}", "configureExperimentsSubtitle": "Experimente konfigurieren, um Modelle zu trainieren.", "configureModels": "Modelle konfigurieren", "configureOptimizer": "Hyperparameter-Optimierung konfigurieren", + "crossValidation": "Kreuzvalidierung", "currentExperiments": "Aktuelle Experimente", "currentOptimizerSettings": "Aktuelle Optimierereinstellungen {{optimizer}}", + "cvType": "Art der Kreuzvalidierung", "datasetInputColumnRequirements": "<0>Die Eingabespalten müssen folgende Typen haben<1><2>, und eine Kardinalität von <3>{{cardinality}}.", "datasetOutputColumnRequirements": "<0>Die Ausgabespalten müssen folgende Typen haben<1><2>, und eine Kardinalität von <3>{{cardinality}}.", - "cardinalityAtLeast": "mindestens {{min}}", - "cardinalityBetween": "{{min}} bis {{max}}", "duration": "Dauer", "endTime": "Endzeit", "experimentName": "Experimentname", "experimentsModuleTitle": "Experiment-Modul", + "groupColumn": "Spalte zum Gruppieren", + "groupColumnDescription": "Spalte, die zur Gruppierung verwendet wird. Stichproben derselben Gruppe bleiben im selben Fold zusammen, sodass eine Gruppe nicht gleichzeitig in Trainings- und Testmenge erscheint.", + "holdout": "Holdout", "manual": "Manuell", "metricToOptimize": "Zu optimierende Metrik", "missingValues": "fehlende Werte", "missingValuesDetected": "Der Datensatz enthält fehlende Werte (NaN) in den Spalten:", "modelName": "Modellname", "modelsInExperiment": "Aktuelle Modelle im Experiment", + "noConverterAdded": "Es wurde kein Converter hinzugefügt.", "noDatasetsAvailable": "Keine Datensätze verfügbar", "noDatasetsAvailableGoToDataTab": "Gehen Sie zum <1>Daten-Tab, um zuerst einen hochzuladen.", "noModelsAvailable": "Keine Modelle verfügbar", "noOptimizersNoMetric": "Keine Hyperparameter-Optimierung", + "numFolds": "Anzahl der Folds (k)", + "numFoldsDescription": "Anzahl der Folds bei der Kreuzvalidierung. Mehr Folds können die Stabilität der Bewertung verbessern, erhöhen aber die Verarbeitungszeit. Muss eine ganze Zahl größer als 1 sein.", + "numRepeats": "Anzahl der Wiederholungen", + "numRepeatsDescription": "Anzahl der Wiederholungen des Kreuzvalidierungsprozesses. Ein höherer Wert liefert präzisere Werte, erfordert aber mehr Trainingszeit. Muss eine ganze Zahl größer als 1 sein.", "optimizer": "Optimierer", "optimizerMetric": "Optimierungsmetrik", "parameterModification": "Parametermodifikation", - "prepareDataset": "Datensatz vorbereiten", "predefined": "Vordefiniert", + "prepareDataset": "Datensatz vorbereiten", "random": "Zufällig", "recommendPreprocessMissingValues": "Es wird empfohlen, den Datensatz vorzuverarbeiten, um diese fehlenden Werte vor dem Training eines Modells zu behandeln.", "rowIndexes": "Zeilenindizes", "rowIndexesDescription": "Geben Sie Zeilenbereiche für jede Aufteilung mit kommaseparierten Werten oder Bereichen an (z.B. 0-100, 200).", "seed": "Seed", + "selectAColumn": "Eine Spalte auswählen", + "selectDataset": "Datensatz auswählen", "selectDatasetColumns": "Geben Sie an, welche Spalten des Datensatzes als Ein- und Ausgabe verwendet werden.", "selectDatasetTitle": "Datensatz für die ausgewählte Aufgabe auswählen", - "selectHowToDivideDataset": "Wählen Sie, wie der Datensatz in Trainings-, Validierungs- und Testmengen aufgeteilt werden soll.", "selectEvaluationStrategy": "Bewertungsstrategie", - "holdout": "Holdout", - "crossValidation": "Kreuzvalidierung", - "cvType": "Art der Kreuzvalidierung", - "numFolds": "Anzahl der Folds (k)", - "numFoldsDescription": "Anzahl der Folds bei der Kreuzvalidierung. Mehr Folds können die Stabilität der Bewertung verbessern, erhöhen aber die Verarbeitungszeit. Muss eine ganze Zahl größer als 1 sein.", - "numRepeats": "Anzahl der Wiederholungen", - "numRepeatsDescription": "Anzahl der Wiederholungen des Kreuzvalidierungsprozesses. Ein höherer Wert liefert präzisere Werte, erfordert aber mehr Trainingszeit. Muss eine ganze Zahl größer als 1 sein.", - "groupColumn": "Spalte zum Gruppieren", + "selectHowToDivideDataset": "Wählen Sie, wie der Datensatz in Trainings-, Validierungs- und Testmengen aufgeteilt werden soll.", "selectInputOutputColumnsDescription": "Spaltenbezeichnungen aus den Listen auswählen.", "selectMetrics": "Metriken auswählen", "selectModelFirst": "Zuerst ein Modell auswählen", @@ -88,20 +92,17 @@ "setNameAndTask": "Name und Aufgabe festlegen", "shuffle": "Mischen", "shuffleDescription": "Bestimmt, ob die Daten beim Definieren der Mengen gemischt werden. Muss wahr sein, um die Daten zu mischen, andernfalls falsch.", + "splitType": "Aufteilungstyp", "splits": "Aufteilungen", "splitsDescription": "Anteil der jedem Teilbereich zugewiesenen Daten. Werte müssen zwischen 0 und 1 liegen und 1 ergeben.", - "splitType": "Aufteilungstyp", "startTime": "Startzeit", "stratify": "Stratifizieren", "stratifyDescription": "Legt fest, ob die Daten proportional gemäß der Klassenverteilung in jeder Menge aufgeteilt werden. Mischen muss wahr sein, um die Daten zu stratifizieren.", + "stratifyRequiresShuffle": "Erfordert aktiviertes Mischen", "useManualSplittingBySpecifyingRowIndexes": "Manuelle Aufteilung durch Angabe der Zeilenindizes jeder Teilmenge", "usePredefinedSplitsFromDataset": "Vordefinierte Aufteilungen aus dem Datensatz verwenden", "usePredefinedSplitsFromDatasetNotAvailable": "Vordefinierte Aufteilungen aus dem Datensatz verwenden (nicht verfügbar)", - "useRandomRowsBySpecifyingPortion": "Zufällige Zeilen durch Angabe des Anteils des Datensatzes für jede Teilmenge", - "selectAColumn": "Eine Spalte auswählen", - "groupColumnDescription": "Spalte, die zur Gruppierung verwendet wird. Stichproben derselben Gruppe bleiben im selben Fold zusammen, sodass eine Gruppe nicht gleichzeitig in Trainings- und Testmenge erscheint.", - "stratifyRequiresShuffle": "Erfordert aktiviertes Mischen", - "selectDataset": "Datensatz auswählen" + "useRandomRowsBySpecifyingPortion": "Zufällige Zeilen durch Angabe des Anteils des Datensatzes für jede Teilmenge" }, "message": { "confirmDeleteRun": "Sind Sie sicher, dass Sie diesen Durchlauf löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", diff --git a/DashAI/front/src/utils/i18n/locales/de/models.json b/DashAI/front/src/utils/i18n/locales/de/models.json index e8a76d581..aaf9a16a9 100644 --- a/DashAI/front/src/utils/i18n/locales/de/models.json +++ b/DashAI/front/src/utils/i18n/locales/de/models.json @@ -1,22 +1,22 @@ { "button": { + "addConverter": "Converter hinzufügen", + "addNewPrediction": "Neue Vorhersage hinzufügen", "createExplainer": "Erklärungsmodell erstellen", "createGlobalExplainer": "Neues globales Erklärungsmodell", "createLocalExplainer": "Neues lokales Erklärungsmodell", "createPrediction": "Vorhersage erstellen", - "addNewPrediction": "Neue Vorhersage hinzufügen", - "uploadNewDataset": "Neuen Datensatz hochladen", "createSession": "Sitzung erstellen", "deleteAndRetrain": "Löschen und neu trainieren", "deleteRun": "Durchlauf löschen", "hideOperations": "Operationen ausblenden", "hideParameters": "Parameter ausblenden", "modelsHub": "Modell-Zentrale", - "newSession": "Neue Sitzung", "modifyParameters": "Parameter bearbeiten", - "newPrediction": "Neue Vorhersage", "newDatasetPrediction": "Neue Datensatz-Vorhersage", "newManualPrediction": "Neue manuelle Vorhersage", + "newPrediction": "Neue Vorhersage", + "newSession": "Neue Sitzung", "retrain": "Neu trainieren", "runAll": "Alle ausführen", "runAllModels": "Alle Modelle ausführen", @@ -24,7 +24,8 @@ "saveAndRunModel": "Modell speichern und ausführen", "showOperations": "Operationen anzeigen", "showParameters": "Parameter anzeigen", - "updateAndRetrain": "Aktualisieren und neu trainieren" + "updateAndRetrain": "Aktualisieren und neu trainieren", + "uploadNewDataset": "Neuen Datensatz hochladen" }, "error": { "completeRequiredFields": "Bitte füllen Sie alle Pflichtfelder aus", @@ -71,104 +72,33 @@ }, "label": { "addModelToSession": "Modell zur Sitzung hinzufügen", + "allRepetitions": "Alle Wiederholungen", + "alternativeHypothesis": "Alternativhypothese", + "applyPreprocessing": "Vorverarbeitung anwenden", + "applyPreprocessingDescription": "Die ausgewählten Converter werden nur anhand der Trainingsdaten angepasst und auf den Rest angewendet, um Datenlecks zu vermeiden. Wird dies nicht angewendet, trainiert das Modell mit den Daten wie sie sind.", "availableExplainers": "Verfügbare Erklärungsmodelle", "availableModels": "Verfügbare Modelle", - "noSavedTests": "Noch keine gespeicherten Tests.", - "name": "Name", - "metric": "Metrik", - "metricSplit": "Aufteilung", - "significantsCount_one": "{{count}} / {{total}} signifikant", - "significantsCount_other": "{{count}} / {{total}} signifikanten", - "bestModel": "Bestes", - "score": "Bewertung", - "scoreProfile": "Bewertungsprofil", - "scoreHeaderTooltip": "Gewichtete Bewertung von 0–100 basierend auf dem ausgewählten Profil. Fehlermetriken ohne Obergrenze (MAE, RMSE, TER, …) werden relativ zum schlechtesten Modell in diesem Vergleich normalisiert. Bewegen Sie den Mauszeiger über eine Zelle, um die Aufschlüsselung anzuzeigen.", - "profile_balanced": "Ausgewogen", - "profile_detectPositives": "Positive Erkennen", - "profile_avoidFalseAlarms": "Falschalarme Vermeiden", - "profile_probabilityQuality": "Qualität der Wahrscheinlichkeiten", - "profile_regression_fit": "Modellanpassung", - "profile_regression_error": "Ausgewogener Fehler", - "profile_translation_quality": "Übersetzungsqualität", - "profile_translation_balanced": "Ausgewogene Übersetzung", - "profile_text_balanced": "Ausgewogen", - "profile_text_detectPositives": "Positive Erkennen", - "profile_text_avoidFalseAlarms": "Falschalarme Vermeiden", - "profile_text_probabilityQuality": "Qualität der Wahrscheinlichkeiten", - "nestedCrossValidation": "Verschachtelte Kreuzvalidierung", - "outerSplitterInherited": "Die ursprüngliche Splitter-Konfiguration der Sitzung wird für die äußere Schleife der verschachtelten Kreuzvalidierung übernommen", - "innerLoopConfiguration": "Konfigurieren Sie die Datenaufteilung für die innere Schleife der verschachtelten Kreuzvalidierung", - "outerSplitter": "Splitter für die äußere Schleife", - "outerFolds": "Anzahl der Folds", - "innerSplitter": "Splitter für die innere Schleife", - "innerFolds": "Anzahl der Folds", - "statisticalTests": "Statistische Tests", - "modelsToCompare": "Zu vergleichende Modelle", - "significanceLevel": "Signifikanzniveau", - "noFinishedRuns": "Keine abgeschlossenen Durchläufe verfügbar", - "result": "Ergebnis", - "statistic": "Statistik", - "significant": "Signifikant", - "notSignificant": "Nicht Signifikant", - "technicalDetails": "Technische Details", - "model1": "Modell 1", - "model2": "Modell 2", - "nemenyiPairwiseComparisons": "Nemenyi-Post-hoc-Vergleiche nach dem Friedman-Test", - "tukeyPairwiseComparisons": "Tukey-Post-hoc-Vergleiche nach dem ANOVA-Test", - "wilcoxonPairwiseComparisons": "Paarweise Wilcoxon-Tests mit Holm-Korrektur", - "alternativeHypothesis": "Alternativhypothese", - "correctionMethod": "Korrekturmethode", - "repetition": "Wiederholung", - "allRepetitions": "Alle Wiederholungen", "averaged": "Gemittelt", - "lines": "Linien", - "histogramPlot": "Histogramm", - "foldNumber": "Fold-Nummer", - "theoricalQuantiles": "Theoretische Quantile", - "sampleQuantiles": "Stichprobenquantile", - "metricValue": "Metrikwert", - "frequency": "Häufigkeit", - "graphs": "Grafiken", - "helperTests": "Voraussetzungstests", - "selectAtLeastRuns": "Wählen Sie mindestens {{min}} Modelle aus", - "selectExactlyRuns_one": "Wählen Sie genau {{count}} Modell aus", - "selectExactlyRuns_other": "Wählen Sie genau {{count}} Modelle aus", - "normalityByRunSummary": "{{normal}} von {{total}} Durchläufen scheinen einer Normalverteilung zu folgen (α = {{alpha}}).", - "notNormal": "Nicht normal", - "normal": "Normal", - "selectBetweenRuns": "Wählen Sie zwischen {{min}} und {{max}} Modellen aus", - "saveDetails": "Details zum Speichern", - "testName": "Testname (optional)", - "testNameHelp": "Wenn Sie dieses Feld leer lassen, wird der Testname verwendet.", - "testDescription": "Beschreibung (optional)", - "resultSaved": "Gespeichert", - "saveResult": "Ergebnis speichern", - "withHpo": "Mit HPO", - "withoutHpo": "Ohne HPO", - "nestedCv": "Verschachtelte Kreuzvalidierung", + "bar": "Balken", + "bestModel": "Bestes", + "chartType": "Diagrammtyp", "chooseTaskForSessionWithDataset": "Wählen Sie die ML-Aufgabe für Ihre Sitzung mit dem Datensatz \"{{datasetName}}\".", "configuration": "Konfiguration", "configureModel": "Modell konfigurieren", "configureOptimizer": "Optimierer konfigurieren", "configureSession": "Sitzung konfigurieren", "configureTasksTrainCompareModels": "Aufgaben konfigurieren, Modelle trainieren und vergleichen. Wählen Sie eine Aufgabe, um Ihren Modellierungs-Workflow zu beginnen.", - "confirmDeleteSession": "Sind Sie sicher, dass Sie die Sitzung \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "configureTest": "Test konfigurieren", "confirmBulkDeleteSessions_one": "Sind Sie sicher, dass Sie die ausgewählte Sitzung löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", "confirmBulkDeleteSessions_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Sitzungen löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "selectSessionsToDelete": "Sitzungen zum Löschen auswählen", + "confirmDeleteSession": "Sind Sie sicher, dass Sie die Sitzung \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmParameterUpdate": "Parameteraktualisierung bestätigen", + "correctionMethod": "Korrekturmethode", "customMetrics": "Benutzerdefinierte Metriken", "datasetPredictions": "Datensatz-Vorhersagen", + "divideColumnsAndSplits": "Spalten aufteilen und Datensatz-Splits konfigurieren", "dropModelHere": "Hier ablegen zum Hinzufügen", "editRun": "Durchlauf bearbeiten", - "chartType": "Diagrammtyp", - "savedTests": "Gespeicherte Tests", - "foldGraphs": "Fold-Grafiken", - "nestedCvResults": "Verschachtelte Ergebnisse", - "outerFoldFetch": "Äußere Folds", - "finalFoldFetch": "Finale Folds", - "twoSided": "Beidseitig (Modell 1 ≠ Modell 2)", - "greater": "Größer (Modell 1 > Modell 2)", - "less": "Kleiner (Modell 1 < Modell 2)", "editRunParameters": "Parameter bearbeiten und Modell neu ausführen", "epoch": "Epoche", "exitModelDetailToAddModels": "Kehren Sie zur Sitzungsübersicht zurück, um weitere Modelle hinzuzufügen.", @@ -176,39 +106,64 @@ "explainability": "Erklärbarkeit", "explainersCount_one": "• <1>{{count}} Erklärungsmodell", "explainersCount_other": "• <1>{{count}} Erklärungsmodelle", + "finalFoldFetch": "Finale Folds", + "foldGraphs": "Fold-Grafiken", + "foldNumber": "Fold-Nummer", + "frequency": "Häufigkeit", "generalMetrics": "Allgemeine Metriken", "globalExplainer": "Globales Erklärungsmodell", "globalExplainers": "Globale Erklärungsmodelle", "goalMetric": "Zielmetrik", + "graphs": "Grafiken", + "greater": "Größer (Modell 1 > Modell 2)", "heatmap": "Heatmap", + "helperTests": "Voraussetzungstests", "hideResults": "Ergebnisse ausblenden", "higherIsBetter": "Höher ist besser", + "histogramPlot": "Histogramm", "hyperparameterOptimizationPlots": "Hyperparameter-Optimierungsplots", "hyperparameterOptimizerConfiguration": "Hyperparameter-Optimierer-Konfiguration", "hyperparameters": "Hyperparameter", + "innerFolds": "Anzahl der Folds", + "innerLoopConfiguration": "Konfigurieren Sie die Datenaufteilung für die innere Schleife der verschachtelten Kreuzvalidierung", + "innerSplitter": "Splitter für die innere Schleife", "inputColumns": "Eingabespalten", + "less": "Kleiner (Modell 1 < Modell 2)", + "lines": "Linien", "liveMetrics": "Live-Metriken", "localExplainer": "Lokales Erklärungsmodell", "localExplainers": "Lokale Erklärungsmodelle", "lowerIsBetter": "Niedriger ist besser", - "manualPredictions": "Manuelle Vorhersagen", "manual": "Manuell", + "manualPredictions": "Manuelle Vorhersagen", + "metric": "Metrik", + "metricSplit": "Aufteilung", + "metricToOptimize": "Zu optimierende Metrik", + "metricValue": "Metrikwert", "metrics": "Metriken", "metricsEmptyForDisplaySet": "Die Ergebnismetriken für {{set}} sind leer.", - "metricToOptimize": "Zu optimierende Metrik", + "minTwoRunsRequired": "Wählen Sie mindestens 2 abgeschlossene Durchläufe aus, um statistische Tests durchzuführen", + "model1": "Modell 1", + "model2": "Modell 2", "modelComparison": "Modellvergleich", "modelConfiguration": "Modellkonfiguration", "modelCount_one": "{{count}} Modell", "modelCount_other": "{{count}} Modelle", - "bar": "Balken", "modelsModule": "Modell-Modul", + "modelsToCompare": "Zu vergleichende Modelle", + "name": "Name", "nameYourSession": "Sitzung benennen", + "nemenyiPairwiseComparisons": "Nemenyi-Post-hoc-Vergleiche nach dem Friedman-Test", + "nestedCrossValidation": "Verschachtelte Kreuzvalidierung", + "nestedCv": "Verschachtelte Kreuzvalidierung", + "nestedCvResults": "Verschachtelte Ergebnisse", "noCompatibleExplainersFound": "Keine kompatiblen Erklärungsmodelle gefunden", "noCompatibleModelsFound": "Keine kompatiblen Modelle gefunden", "noCompletedRuns": "Noch keine abgeschlossenen Durchläufe", "noConfigurationAvailable": "Keine Konfiguration für diesen Durchlauf verfügbar", "noDatasetPredictionsYet": "Noch keine Datensatz-Vorhersagen", "noExplainersMatchSearch": "Keine Erklärungsmodelle entsprechen Ihrer Suche", + "noFinishedRuns": "Keine abgeschlossenen Durchläufe verfügbar", "noGlobalExplainersYet": "Noch keine globalen Erklärungsmodelle", "noHyperparameterPlotsAvailable": "Keine Hyperparameter-Plots verfügbar. Nur Durchläufe mit mindestens einem optimierbaren Parameter erzeugen Plots.", "noLocalExplainersYet": "Noch keine lokalen Erklärungsmodelle", @@ -218,19 +173,49 @@ "noModelsMatchSearch": "Keine Modelle entsprechen Ihrer Suche", "noPredictionsYet": "Noch keine Vorhersagen", "noRunsYet": "Noch keine Durchläufe. Fügen Sie Modelle aus dem rechten Panel hinzu.", + "noSavedTests": "Noch keine gespeicherten Tests.", "noSessionSelected": "Keine Sitzung ausgewählt", + "noTestsMatch": "Kein Test entspricht Ihrer Suche", + "nonParametricTests": "Nichtparametrische Tests", + "normal": "Normal", + "normalityByRunSummary": "{{normal}} von {{total}} Durchläufen scheinen einer Normalverteilung zu folgen (α = {{alpha}}).", + "notNormal": "Nicht normal", + "notSignificant": "Nicht Signifikant", "operations": "Vorgänge", "operationsWillBeDeletedWarning": "Diese Operationen werden dauerhaft gelöscht und können nicht wiederhergestellt werden. Sind Sie sicher?", + "optimizer": "Optimierer", "optimizerConfiguration": "Einstellungen des Hyperparameter-Optimierers konfigurieren", "optimizerParameters": "Optimiererparameter", - "optimizer": "Optimierer", + "outerFoldFetch": "Äußere Folds", + "outerFolds": "Anzahl der Folds", + "outerSplitter": "Splitter für die äußere Schleife", + "outerSplitterInherited": "Die ursprüngliche Splitter-Konfiguration der Sitzung wird für die äußere Schleife der verschachtelten Kreuzvalidierung übernommen", "outputColumns": "Ausgabespalten", + "parametricTests": "Parametrische Tests", "pleaseSelectMetricToOptimize": "Bitte wählen Sie eine Metrik zur Optimierung aus.", "predictions": "Vorhersagen", "predictionsCount_one": "• <1>{{count}} Vorhersage", "predictionsCount_other": "• <1>{{count}} Vorhersagen", - "divideColumnsAndSplits": "Spalten aufteilen und Datensatz-Splits konfigurieren", "prepareDataset": "Datensatz vorbereiten", + "preprocessingFailed": "Vorverarbeitung fehlgeschlagen", + "preprocessingInProgress": "Verarbeitung läuft...", + "preprocessingOptional": "Vorverarbeitung (optional)", + "preprocessingOptionalDescription": "Die ausgewählten Converter werden nur anhand der Trainingsdaten angepasst und auf den Rest angewendet, um Datenlecks zu vermeiden.", + "profile_avoidFalseAlarms": "Falschalarme Vermeiden", + "profile_balanced": "Ausgewogen", + "profile_detectPositives": "Positive Erkennen", + "profile_probabilityQuality": "Qualität der Wahrscheinlichkeiten", + "profile_regression_error": "Ausgewogener Fehler", + "profile_regression_fit": "Modellanpassung", + "profile_text_avoidFalseAlarms": "Falschalarme Vermeiden", + "profile_text_balanced": "Ausgewogen", + "profile_text_detectPositives": "Positive Erkennen", + "profile_text_probabilityQuality": "Qualität der Wahrscheinlichkeiten", + "profile_translation_balanced": "Ausgewogene Übersetzung", + "profile_translation_quality": "Übersetzungsqualität", + "repetition": "Wiederholung", + "result": "Ergebnis", + "resultSaved": "Gespeichert", "retrainConfirmDetails": "Sind Sie sicher, dass Sie den Durchlauf \"<1>{{runName}}\" neu trainieren möchten?", "retrainModel": "Modell neu trainieren?", "retrainWillDeleteOperations": "Dieser Durchlauf hat bestehende Operationen, die gelöscht werden", @@ -240,46 +225,70 @@ "runFailedNoHyperparameterPlots": "Durchlauf fehlgeschlagen. Keine Hyperparameter-Plots verfügbar.", "runInProgressCannotEdit": "Der Durchlauf wird gerade ausgeführt und kann nicht bearbeitet werden.", "runName": "Durchlaufname", - "confirmParameterUpdate": "Parameteraktualisierung bestätigen", "runNotFound": "Durchlauf nicht gefunden", "runNotStartedNoHyperparameterPlots": "Durchlauf nicht gestartet. Keine Hyperparameter-Plots verfügbar.", + "runsSelected": "ausgewählte Durchläufe", + "sampleQuantiles": "Stichprobenquantile", "saveConfirmDetails": "Das Speichern von \"<1>{{runName}}\" setzt seinen Status auf 'Nicht gestartet' zurück und löscht seine aktuellen Metriken und Ergebnisse. Möchten Sie wirklich fortfahren?", + "saveDetails": "Details zum Speichern", "saveParameterChanges": "Parameteränderungen speichern?", + "saveResult": "Ergebnis speichern", "saveWillDeleteOperationsDetails": "Das Speichern von \"<1>{{runName}}\" setzt den Durchlauf zurück. Folgendes wird beim erneuten Training gelöscht:", + "savedTests": "Gespeicherte Tests", + "score": "Bewertung", + "scoreHeaderTooltip": "Gewichtete Bewertung von 0–100 basierend auf dem ausgewählten Profil. Fehlermetriken ohne Obergrenze (MAE, RMSE, TER, …) werden relativ zum schlechtesten Modell in diesem Vergleich normalisiert. Bewegen Sie den Mauszeiger über eine Zelle, um die Aufschlüsselung anzuzeigen.", + "scoreProfile": "Bewertungsprofil", "searchDatasetsSessions": "Datensätze und Sitzungen suchen", "searchMetric": "Metrik suchen...", "searchModels": "Modelle suchen...", "searchTests": "Tests suchen...", - "parametricTests": "Parametrische Tests", - "nonParametricTests": "Nichtparametrische Tests", - "minTwoRunsRequired": "Wählen Sie mindestens 2 abgeschlossene Durchläufe aus, um statistische Tests durchzuführen", - "noTestsMatch": "Kein Test entspricht Ihrer Suche", - "runsSelected": "ausgewählte Durchläufe", - "configureTest": "Test konfigurieren", + "selectAtLeastRuns": "Wählen Sie mindestens {{min}} Modelle aus", + "selectBetweenRuns": "Wählen Sie zwischen {{min}} und {{max}} Modellen aus", + "selectColumnsDescription": "Wählen Sie aus, welche Spalten Eingaben sind und welche die Ausgabe für diese Sitzung ist.", + "selectColumnsTitle": "Spalten auswählen", "selectDataset": "Datensatz auswählen", - "selectDatasetForSession": "Datensatz für Ihre Sitzung auswählen", "selectDatasetAndPrepare": "Benennen Sie Ihre Sitzung, wählen Sie einen Datensatz aus und konfigurieren Sie Spalten und Aufteilungen.", "selectDatasetFirst": "Wählen Sie einen Datensatz aus, um Spalten und Zeilenaufteilungen zu konfigurieren.", + "selectDatasetForSession": "Datensatz für Ihre Sitzung auswählen", + "selectExactlyRuns_one": "Wählen Sie genau {{count}} Modell aus", + "selectExactlyRuns_other": "Wählen Sie genau {{count}} Modelle aus", "selectSessionToViewModels": "Wählen Sie eine Sitzung, um verfügbare Modelle anzuzeigen.", + "selectSessionsToDelete": "Sitzungen zum Löschen auswählen", "selectTask": "Aufgabe auswählen", "selectTaskForSession": "Eine Aufgabe für Ihre Sitzung auswählen", "sessionConfiguration": "Sitzungskonfiguration", "sessionName": "Sitzungsname", "showResults": "Ergebnisse anzeigen", + "significanceLevel": "Signifikanzniveau", + "significant": "Signifikant", + "significantsCount_one": "{{count}} / {{total}} signifikant", + "significantsCount_other": "{{count}} / {{total}} signifikanten", + "statistic": "Statistik", + "statisticalTests": "Statistische Tests", "step": "Schritt", + "technicalDetails": "Technische Details", "test": "Test", + "testDescription": "Beschreibung (optional)", "testMetrics": "Testmetriken", + "testName": "Testname (optional)", + "testNameHelp": "Wenn Sie dieses Feld leer lassen, wird der Testname verwendet.", "testSet": "Testmenge", + "theoricalQuantiles": "Theoretische Quantile", "thereAreNoMetricsForThisRun": "Es gibt keine Metriken für {{set}} in diesem Durchlauf", "tourDisabledMessage": "Zurück zur Startseite, um die Tour zu starten", "train": "Training", - "trainingMetrics": "Trainingsmetriken", "trainSet": "Trainingsmenge", + "trainingMetrics": "Trainingsmetriken", "trial": "Versuch", + "tukeyPairwiseComparisons": "Tukey-Post-hoc-Vergleiche nach dem ANOVA-Test", + "twoSided": "Beidseitig (Modell 1 ≠ Modell 2)", "validation": "Validierung", "validationMetrics": "Validierungsmetriken", "validationSet": "Validierungsmenge", - "viewResultsAs": "Ergebnisse als Spalten oder Graphen anzeigen" + "viewResultsAs": "Ergebnisse als Spalten oder Graphen anzeigen", + "wilcoxonPairwiseComparisons": "Paarweise Wilcoxon-Tests mit Holm-Korrektur", + "withHpo": "Mit HPO", + "withoutHpo": "Ohne HPO" }, "message": { "allRunsCompleted": "{{experiment}} hat alle Durchläufe abgeschlossen.", diff --git a/DashAI/front/src/utils/i18n/locales/pt/datasets.json b/DashAI/front/src/utils/i18n/locales/pt/datasets.json index b671ad4ac..822903d69 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/pt/datasets.json @@ -54,6 +54,7 @@ "uploadFile": "Enviar um arquivo" }, "error": { + "cannotSaveEmptyDataset": "Não é possível salvar o dataset: todas as colunas foram removidas.", "converterFailed": "Ocorreu um erro durante o processamento do conversor.", "converterFailedWithInfo": "Erro ao processar conversor: {{error}}", "createConverterError": "Falha ao criar conversor", @@ -70,72 +71,54 @@ "failedToCreateDataset": "Nao foi possivel criar o conjunto de dados. Verifique seu arquivo e a configuracao e tente novamente.", "failedToCreateDatasetFromNotebook": "Falha ao criar conjunto de dados a partir do caderno", "failedToCreateExplorer": "Falha ao criar explorador", + "failedToCreateFolder": "Falha ao criar pasta", "failedToDeleteDataset": "Falha ao excluir conjunto de dados", "failedToDeleteDatasets": "Falha ao excluir os conjuntos de dados selecionados", - "failedToCreateFolder": "Falha ao criar pasta", - "failedToUpdateFolder": "Falha ao atualizar pasta", "failedToDeleteFolder": "Falha ao excluir pasta", - "folderNameExists": "Já existe uma pasta com este nome", - "failedToMoveDataset": "Falha ao mover conjunto de dados", "failedToDeleteNotebook": "Falha ao excluir caderno", "failedToDeleteNotebooks": "Falha ao excluir os cadernos selecionados", "failedToFetchNotebooks": "Falha ao obter cadernos", "failedToLoadDatasetInfo": "Falha ao obter informações do conjunto de dados", + "failedToMoveDataset": "Falha ao mover conjunto de dados", + "failedToResetExplorerResults": "Falha ao restaurar o gráfico original", "failedToUpdateDataset": "Falha ao atualizar conjunto de dados", "failedToUpdateExplorerResults": "Falha ao atualizar resultados do explorador", - "failedToResetExplorerResults": "Falha ao restaurar o gráfico original", + "failedToUpdateFolder": "Falha ao atualizar pasta", "failedToUpdateNotebook": "Falha ao atualizar caderno", + "fetchingConverters": "Erro ao obter os conversores disponíveis.", "fetchingDataloaders": "Erro ao tentar obter dataloaders compatíveis.", "fetchingDatasetColumns": "Erro ao tentar obter as colunas do conjunto de dados.", "fetchingExplorersConverters": "Falha ao obter exploradores/conversores", "fileTypeNotAllowed": "Tipo de arquivo não permitido para o dataloader selecionado.", + "folderNameExists": "Já existe uma pasta com este nome", "loadingDatasetPreview": "Erro ao carregar visualização prévia do conjunto de dados", "noDatasetDataAvailable": "Não há dados do conjunto de dados disponíveis", "noDatasetFileAvailable": "Não há arquivo do conjunto de dados disponível", - "notebookNameEmpty": "O nome do caderno não pode estar vazio", "noValidColumnsForExplorer": "Não há colunas válidas disponíveis para este explorador.", "noValidColumnsWithDtypesMentioned": "Este conjunto de dados não possui colunas com os tipos necessários ({{dtypes}}).", + "notebookNameEmpty": "O nome do caderno não pode estar vazio", "processConverterError": "Falha ao processar conversor", "requiredFieldsMissing": "Campos obrigatórios ausentes", - "requiresExactColumns_one": "Requer exatamente {{required}} coluna válida, mas {{available}} disponível.", "requiresExactColumns_many": "Requer exatamente {{required}} colunas válidas, mas {{available}} disponíveis.", + "requiresExactColumns_one": "Requer exatamente {{required}} coluna válida, mas {{available}} disponível.", "requiresExactColumns_other": "Requer exatamente {{required}} colunas válidas, mas {{available}} disponíveis.", - "requiresMinColumns_one": "Requer pelo menos {{required}} coluna válida, mas apenas {{available}} disponível.", "requiresMinColumns_many": "Requer pelo menos {{required}} colunas válidas, mas apenas {{available}} disponíveis.", + "requiresMinColumns_one": "Requer pelo menos {{required}} coluna válida, mas apenas {{available}} disponível.", "requiresMinColumns_other": "Requer pelo menos {{required}} colunas válidas, mas apenas {{available}} disponíveis.", - "zipContentsNotCompatible": "O arquivo ZIP não contém arquivos compatíveis com o dataloader selecionado", - "cannotSaveEmptyDataset": "Não é possível salvar o dataset: todas as colunas foram removidas." + "zipContentsNotCompatible": "O arquivo ZIP não contém arquivos compatíveis com o dataloader selecionado" }, "label": { - "task": "Tarefa", + "aboutTool": "Sobre {{toolName}}", "all": "todos", + "allTypeChangesValid": "Todas as alterações de tipo são válidas e podem ser aplicadas com segurança.", "allowedDataTypes": "Tipos de dados permitidos: <1><0>", "allowedValueTypes": "Tipos de valores permitidos: <1><0>", - "excludedDataTypes": "Tipos de dados excluídos: {{dtypes}}", - "allTypeChangesValid": "Todas as alterações de tipo são válidas e podem ser aplicadas com segurança.", "analysisTools": "Ferramentas de Análise", "appearance": "Aparência", "appliedTransformations": "Transformações Aplicadas:", "associatedDataset": "Conjunto de Dados Associado", "atLeastTwoColorStopsRequired": "Nenhum ponto de cor definido. Por favor, adicione pelo menos dois pontos de cor.", "availableDatasets": "Conjuntos de Dados Disponíveis", - "confirmDeleteFolder": "Excluir pasta", - "confirmDeleteFolderContent": "Tem certeza que deseja excluir a pasta \"{{name}}\"? Os conjuntos de dados dentro serão movidos para sem pasta.", - "noFolder": "Sem pasta", - "newFolder": "Nova pasta", - "folderName": "Nome da pasta", - "confirmDeleteDataset": "Tem certeza de que deseja excluir o conjunto de dados \"{{name}}\"? Esta ação não pode ser desfeita.", - "confirmDeleteDatasetLinkedWarning": "Todos os cadernos e sessões associados a este conjunto de dados também serão excluídos.", - "confirmBulkDeleteDatasets_one": "Tem certeza de que deseja excluir o conjunto de dados selecionado? Esta ação não pode ser desfeita.", - "confirmBulkDeleteDatasets_many": "Tem certeza de que deseja excluir os {{count}} conjuntos de dados selecionados? Esta ação não pode ser desfeita.", - "confirmBulkDeleteDatasets_other": "Tem certeza de que deseja excluir os {{count}} conjuntos de dados selecionados? Esta ação não pode ser desfeita.", - "selectDatasetsToDelete": "Selecionar conjuntos de dados para excluir", - "confirmDeleteNotebook": "Tem certeza de que deseja excluir o caderno \"{{name}}\"? Esta ação não pode ser desfeita.", - "confirmBulkDeleteNotebooks": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", - "confirmBulkDeleteNotebooks_one": "Tem certeza de que deseja excluir o caderno selecionado? Esta ação não pode ser desfeita.", - "confirmBulkDeleteNotebooks_many": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", - "confirmBulkDeleteNotebooks_other": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", - "selectNotebooksToDelete": "Selecionar cadernos para excluir", "avg": "Média", "avgLength": "Comprimento Médio", "avgWordCount": "Contagem Média de Palavras", @@ -160,6 +143,7 @@ "classTargetColumn": "Coluna de Classe/Alvo", "clickToUpload": "Clique para enviar", "color": "Cor", + "colorStops": "Pontos de Cor", "colorbarBorderColor": "Cor da Borda da Barra de Cores", "colorbarBorderWidth": "Largura da Borda da Barra de Cores", "colorbarTickFontColor": "Cor da Fonte dos Números da Barra de Cores", @@ -167,7 +151,7 @@ "colors": "Cores", "colorscale": "Escala de Cores", "colorscaleMode": "Modo da Escala de Cores", - "colorStops": "Pontos de Cor", + "columnInsights": "Observações por Coluna", "columnName": "Nome da Coluna", "columnTypesDistribution": "Distribuição dos Tipos de Coluna", "configureAndUpload": "Configurar e Enviar", @@ -176,10 +160,22 @@ "configureParametersStep": "Etapa {{step}}: Configurar Parâmetros", "configureScope": "Configurar Escopo", "configureToolTitle": "Configurar {{toolType}}: {{toolName}}", - "columnInsights": "Observações por Coluna", + "confirmBulkDeleteDatasets_many": "Tem certeza de que deseja excluir os {{count}} conjuntos de dados selecionados? Esta ação não pode ser desfeita.", + "confirmBulkDeleteDatasets_one": "Tem certeza de que deseja excluir o conjunto de dados selecionado? Esta ação não pode ser desfeita.", + "confirmBulkDeleteDatasets_other": "Tem certeza de que deseja excluir os {{count}} conjuntos de dados selecionados? Esta ação não pode ser desfeita.", + "confirmBulkDeleteNotebooks": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", + "confirmBulkDeleteNotebooks_many": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", + "confirmBulkDeleteNotebooks_one": "Tem certeza de que deseja excluir o caderno selecionado? Esta ação não pode ser desfeita.", + "confirmBulkDeleteNotebooks_other": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", + "confirmDeleteDataset": "Tem certeza de que deseja excluir o conjunto de dados \"{{name}}\"? Esta ação não pode ser desfeita.", + "confirmDeleteDatasetLinkedWarning": "Todos os cadernos e sessões associados a este conjunto de dados também serão excluídos.", + "confirmDeleteFolder": "Excluir pasta", + "confirmDeleteFolderContent": "Tem certeza que deseja excluir a pasta \"{{name}}\"? Os conjuntos de dados dentro serão movidos para sem pasta.", + "confirmDeleteNotebook": "Tem certeza de que deseja excluir o caderno \"{{name}}\"? Esta ação não pode ser desfeita.", "constantColumns": "Colunas Constantes", "convert": "Converter", "converter": "Conversor", + "converterOutput": "Saída", "correlation": "Correlação", "correlationAnalysis": "Análise de Correlação", "correlations": "Correlações", @@ -187,46 +183,49 @@ "createNewNotebook": "Caderno", "createNewNotebookDescription": "Inicia uma nova sessão de análise com um conjunto de dados existente.", "customArray": "Array Personalizado", - "dataloaderConfiguration": "Configuração do Dataloader", "dataQuality": "Qualidade dos Dados", "dataQualityScoreTooltip": "A qualidade dos dados é calculada com base na presença de valores ausentes, duplicatas e diversidade de tipos de dados nas colunas.", "dataQualitySummary": "Resumo da Qualidade dos Dados", + "dataType": "Tipo de Dado", + "dataloaderConfiguration": "Configuração do Dataloader", "datasetDescription": "{{rows}} Linhas, {{columns}} Colunas", "datasetLoading": "Carregando conjunto de dados...", "datasetModule": "Módulo de Conjuntos de Dados", "datasetModuleSubtitle": "Envie seus conjuntos de dados: Explore, analise e transforme seus dados com ferramentas avançadas de análise exploratória. Crie cadernos interativos, gere visualizações e aplique transformações de dados de forma intuitiva.", "datasetName": "Nome do Conjunto de Dados", "datasetPreview": "Visualização Prévia do Conjunto de Dados", - "aboutTool": "Sobre {{toolName}}", "datasetPreviewFor": "Caderno: Visualização Prévia de {{name}}", - "dataType": "Tipo de Dado", + "decreasingColor": "Cor de Diminuição", "deleteConverterConfirmation": "Tem certeza de que deseja excluir o conversor \"{{converter}}\"? Excluir este conversor também excluirá todos os conversores e exploradores posteriores aplicados em seguida. Esta ação não pode ser desfeita.", "deleteExplorerConfirmation": "Tem certeza de que deseja excluir o explorador \"{{explorer}}\"? Esta ação não pode ser desfeita.", "detailsForExplorer": "Detalhes do Explorador: {{name}}", "dimensionIdx": "Dimensão {{idx}}: {{label}}", - "dimensionsLabels": "Rótulos de Dimensões", "dimensionTitle": "Título da Dimensão", + "dimensionsLabels": "Rótulos de Dimensões", "distributionMetrics": "Métricas de Distribuição", "dragAndDropFileHere": "Arraste e solte o arquivo aqui", "dragToResize": "Arraste para redimensionar e ver mais conteúdo", + "dropToolHere": "Solte aqui para adicionar", "duplicatedRows": "Linhas Duplicadas", "editPlotLayout": "Editar Layout do Gráfico", "endIndex": "Índice Final", + "excludedDataTypes": "Tipos de dados excluídos: {{dtypes}}", "explorationPath": "Caminho de Exploração", "explorationType": "Tipo de Exploração", "explore": "Explorar", "explorer": "Explorador", + "exportCSV": "Baixar como CSV", "exportCardImage": "Cartão como imagem", "exportChartImage": "Gráfico como imagem", - "exportCSV": "Baixar como CSV", "exportImage": "Baixar", "exportJSON": "Baixar como JSON", "exportMetrics": "Exportar métricas como JSON", "fileSizeMB": "Tamanho do Arquivo (MB)", "firstLastStopsAtExtremes": "Os pontos de cor devem incluir as posições 0 no início e 1 no final.", + "folderName": "Nome da pasta", "fontFamily": "Família de Fonte", - "foundDuplicateRows_one": "{{count}} linha duplicada encontrada no conjunto de dados", "foundDuplicateRows_many": "{{count}} linhas duplicadas encontradas no conjunto de dados", + "foundDuplicateRows_one": "{{count}} linha duplicada encontrada no conjunto de dados", "foundDuplicateRows_other": "{{count}} linhas duplicadas encontradas no conjunto de dados", "fromDataset": "do conjunto de dados {{datasetName}}", "generalSettings": "Configurações Gerais", @@ -235,19 +234,20 @@ "highCardinality": "Alta Cardinalidade", "highCardinalityDetected": "Alta cardinalidade detectada em: {{columns}}", "ifYourDatasetHaveSplits": "Se seu conjunto de dados tiver divisões, envie-o como arquivo zip", + "importFromHub": "Central de Arquivos", + "importFromHubDescription": "Navegue e baixe conjuntos de dados de fontes externas como HuggingFace e OpenML.", + "increasingColor": "Cor de Aumento", "indices": "Índices (separados por vírgula, ou digite 'all')", "inferenceRows": "Linhas de Inferência", "inferenceRowsDescription": "Número de linhas utilizadas para visualização prévia/inferência de tipo (mínimo 2).", - "previewRows": "Linhas de Pré-visualização", - "previewRowsDescription": "Número de linhas a mostrar na pré-visualização (mínimo 2).", "inferredConfiguration": "Configuração de Inferência de Tipo", "insightConstantColumn": "Esta coluna possui apenas um valor único. Não contribui com informação para a análise.", - "insightHighCardinality": "Esta coluna categórica tem mais de 100 valores únicos. Considere agrupar ou codificar.", "insightEmptyMessage": "Nenhum problema detectado. As observações por coluna aparecerão aqui quando forem encontradas possíveis observações nos dados.", + "insightHighCardinality": "Esta coluna categórica tem mais de 100 valores únicos. Considere agrupar ou codificar.", "insightHighNanRatio": "{{value}}% dos valores estão ausentes nesta coluna. Considere imputação ou remoção.", "insightLowUniqueness": "Baixa unicidade ({{value}}%). Pode ser uma variável categórica classificada incorretamente como texto.", - "insightOutliers_one": "{{count}} valor atípico detectado (método IQR).", "insightOutliers_many": "{{count}} valores atípicos detectados (método IQR).", + "insightOutliers_one": "{{count}} valor atípico detectado (método IQR).", "insightOutliers_other": "{{count}} valores atípicos detectados (método IQR).", "insightPossibleId": "Todos os valores são únicos. Esta coluna pode ser um identificador e não é útil para modelagem.", "insightSkewed": "Distribuição assimétrica à direita (assimetria: {{value}}). Considere aplicar uma transformação logarítmica.", @@ -269,19 +269,15 @@ "legendYPosition": "Posição Y da Legenda", "lengthDistribution": "Distribuição de Comprimento", "lengthMetrics": "Métricas de Comprimento", - "lowerBound": "Limite Inferior", + "lineColor": "Cor da Linha", "lowUniquenessWarning": "Aviso: Esta coluna de texto tem uma proporção de unicidade muito baixa. Pode ser uma variável categórica classificada incorretamente como texto, o que pode causar problemas de análise.", + "lowerBound": "Limite Inferior", "marginBottom": "Margem Inferior", "marginLeft": "Margem Esquerda", "marginRight": "Margem Direita", - "margins": "Margens", "marginTop": "Margem Superior", + "margins": "Margens", "markerColor": "Cor do Marcador", - "lineColor": "Cor da Linha", - "increasingColor": "Cor de Aumento", - "decreasingColor": "Cor de Diminuição", - "totalsColor": "Cor dos Totais", - "transparent": "Transparente", "max": "Máximo", "maxLength": "Comprimento Máximo", "mean": "Média", @@ -295,30 +291,28 @@ "missingValuesDetected": "Valores ausentes detectados nas colunas: {{columns}}", "missingValuesOverview": "Resumo de Valores Ausentes", "mostFrequent": "Mais Frequente", - "nameYourNotebook": "Nomeie seu Caderno", "nComponentsColumnInfo": "Você selecionou {{n}} coluna(s). n_components deve ser menor ou igual a {{n}}. Reduza n_components ou adicione mais colunas para evitar erros.", + "nameYourNotebook": "Nomeie seu Caderno", "newDatasetCreatedWithTransformations": "Um novo conjunto de dados será criado com estas transformações. Pode ser usado com outros módulos sem afetar o original.", + "newFolder": "Nova pasta", "noDataQualityIssuesDetected": "Nenhum problema de qualidade de dados detectado", - "noDuplicateRows": "Nenhuma linha duplicada detectada no conjunto de dados.", - "qualityIssuesFound_one": "{{count}} problema detectado", - "qualityIssuesFound_many": "{{count}} problemas detectados", - "qualityIssuesFound_other": "{{count}} problemas detectados", "noDataset": "Sem conjunto de dados", "noDatasetsAvailable": "Nenhum Conjunto de Dados Disponível", + "noDuplicateRows": "Nenhuma linha duplicada detectada no conjunto de dados.", "noExplorersOrConverters": "Comece a explorar adicionando seu primeiro explorador ou conversor!", + "noFolder": "Sem pasta", "noMissingValues": "Nenhum valor ausente detectado no conjunto de dados.", - "none": "nenhum", "noStrongCorrelationsFound": "Nenhuma correlação forte detectada", + "noToolsMatched": "Nenhuma ferramenta encontrada que corresponda à sua busca.", + "noTransformationsApplied": "Nenhuma transformação aplicada.", + "noTransformationsAppliedYet": "Nenhuma transformação foi aplicada ainda.", + "none": "nenhum", "notebookCreationNote": "Uma cópia do conjunto de dados selecionado será criada para trabalhar no caderno sem alterar o original.", "notebookDescription": "Descrição do Caderno", "notebookHistory": "Histórico do Caderno: {{notebook}}", "notebookInformation": "Informações do Caderno", "notebookName": "Nome do Caderno", "notebooks": "Cadernos", - "noToolsMatched": "Nenhuma ferramenta encontrada que corresponda à sua busca.", - "dropToolHere": "Solte aqui para adicionar", - "noTransformationsApplied": "Nenhuma transformação aplicada.", - "noTransformationsAppliedYet": "Nenhuma transformação foi aplicada ainda.", "numericalAnalysis": "Análise Numérica", "outliers": "Valores Atípicos", "overview": "Visão Geral", @@ -326,11 +320,16 @@ "position": "Posição", "possibleIDColumns": "Possíveis Colunas de ID", "presetScale": "Escala Predefinida", - "processingTitle": "Processando seu dataset...", + "previewRows": "Linhas de Pré-visualização", + "previewRowsDescription": "Número de linhas a mostrar na pré-visualização (mínimo 2).", "processingMessage": "Isso pode levar alguns minutos dependendo do tamanho do conjunto de dados.", + "processingTitle": "Processando seu dataset...", "proportion": "Proporção", "q1": "Q1", "q3": "Q3", + "qualityIssuesFound_many": "{{count}} problemas detectados", + "qualityIssuesFound_one": "{{count}} problema detectado", + "qualityIssuesFound_other": "{{count}} problemas detectados", "qualityScore": "Pontuação de Qualidade: {{value}}{{unit}}", "range": "Intervalo", "requiredColumns": "Colunas necessárias", @@ -346,6 +345,7 @@ "saveProcessedDataset": "Salvar Conjunto de Dados Processado", "scopeColumns": "Escopo - Colunas", "scopeRows": "Escopo - Linhas", + "searchConverters": "Buscar conversores", "searchDatasetsNotebooks": "Buscar Conjuntos de Dados e Cadernos", "searchExplorersConverters": "Buscar exploradores/conversores", "selectColorscale": "Selecionar Escala de Cores", @@ -355,36 +355,39 @@ "selectDataset": "Selecionar um conjunto de dados", "selectDatasetFirst": "Selecione primeiro um conjunto de dados", "selectDatasetForNotebook": "Selecione um conjunto de dados para o caderno", - "selectedColumns_one": "{{count}} coluna selecionada", + "selectDatasetsToDelete": "Selecionar conjuntos de dados para excluir", + "selectNotebookToAccessAnalysisTools": "Selecione um caderno para acessar as ferramentas de análise.", + "selectNotebooksToDelete": "Selecionar cadernos para excluir", + "selectScopeDescriptionColumns": "Aqui você configurará em quais colunas aplicar o conversor.", + "selectScopeDescriptionRows": "Aqui você configurará em quais linhas aplicar o conversor.", + "selectScopeStep": "Etapa {{step}}: Selecionar Escopo", + "selectTargetColumnDescription": "Selecione uma coluna para ser usada como variável alvo para aprendizado supervisionado.", + "selectUploadMethod": "Selecione uma forma de enviar seus dados", "selectedColumns_many": "{{count}} colunas selecionadas", + "selectedColumns_one": "{{count}} coluna selecionada", "selectedColumns_other": "{{count}} colunas selecionadas", "selectedDataloaderConfiguration": "Configuração de {{dataloader}}", "selectedDataset": "Conjunto de dados selecionado", "selectedOrder": "Ordem Selecionada", "selectedRows": "Linhas selecionadas: {{value}}", "selectionMode": "Modo de Seleção", - "selectNotebookToAccessAnalysisTools": "Selecione um caderno para acessar as ferramentas de análise.", - "selectScopeDescriptionColumns": "Aqui você configurará em quais colunas aplicar o conversor.", - "selectScopeDescriptionRows": "Aqui você configurará em quais linhas aplicar o conversor.", - "selectScopeStep": "Etapa {{step}}: Selecionar Escopo", - "selectTargetColumnDescription": "Selecione uma coluna para ser usada como variável alvo para aprendizado supervisionado.", - "selectUploadMethod": "Selecione uma forma de enviar seus dados", "shapeIndicators": "Indicadores de Forma", "showGrid": "Mostrar Grade do Eixo {{axis}}", - "showingRowsInference": "Mostrando {{sampleLength}} de {{previewRowCount}} linhas analisadas para inferência de tipo.", "showLegend": "Mostrar Legenda", "showMore": "Mostrar mais ({{count}} restantes)", - "showMore_one": "Mostrar mais ({{count}} restante)", "showMore_many": "Mostrar mais ({{count}} restantes)", + "showMore_one": "Mostrar mais ({{count}} restante)", "showMore_other": "Mostrar mais ({{count}} restantes)", - "showingRowsPreview": "Mostrando {{sampleLength}} de {{previewRowCount}} linhas.", "showZeroLine": "Mostrar Linha Zero do Eixo {{axis}}", + "showingRowsInference": "Mostrando {{sampleLength}} de {{previewRowCount}} linhas analisadas para inferência de tipo.", + "showingRowsPreview": "Mostrando {{sampleLength}} de {{previewRowCount}} linhas.", "skewness": "Assimetria", "someTypeChangesCannotBeApplied": "Algumas alterações de tipo não podem ser aplicadas:", "startIndex": "Índice Inicial", "stdDev": "Desvio Padrão", "strongCorrelations": "Correlações Fortes", "targetColumn": "Coluna Alvo", + "task": "Tarefa", "text": "Texto", "tickLabels": "Rótulos das Marcas", "tickLabelsHelper": "Um rótulo por linha", @@ -392,27 +395,27 @@ "title": "Título", "titleColor": "Cor do Título", "titleFontSize": "Tamanho de Fonte do Título", - "toolsCount_one": "{{count}} ferramenta", "toolsCount_many": "{{count}} ferramentas", + "toolsCount_one": "{{count}} ferramenta", "toolsCount_other": "{{count}} ferramentas", "topValueCount": "Contagem dos Valores Principais", "totalColumns": "Colunas Totais", "totalPercentage": "{{percentage}}% do total", "totalRows": "Linhas Totais", "totalRowsCount": "Linhas totais: {{total}}", + "totalsColor": "Cor dos Totais", "tourDisabledMessage": "Volte ao início para começar o tour", "tourDisabledMessageNotebook": "Volte à visualização do conjunto de dados para começar o tour", "traceIdx": "Rastreamento {{index}}: {{trace}}", + "transparent": "Transparente", "typeChangeWarnings": "Avisos sobre alterações de tipo:", "typeToSearchDatasets": "Digite para buscar conjuntos de dados...", "unique": "Único", - "uniquenessFormula": "Unicidade = (Único ÷ Total) x 100", "uniquePercentage": "{{percentage}}% Único", "uniqueValues": "Valores Únicos", + "uniquenessFormula": "Unicidade = (Único ÷ Total) x 100", "unknownDataset": "Conjunto de dados desconhecido", "uploadAndConfigure": "Envie seu conjunto de dados e configure os parâmetros", - "importFromHub": "Central de Arquivos", - "importFromHubDescription": "Navegue e baixe conjuntos de dados de fontes externas como HuggingFace e OpenML.", "uploadDataset": "Conjunto de Dados", "uploadDatasetBeforeCreatingSession": "Você precisa enviar um conjunto de dados antes de criar uma sessão. Por favor, vá ao Módulo de Conjuntos de Dados para enviar seus dados.", "uploadDatasetDescription": "Importe seus dados de várias fontes e formatos", diff --git a/DashAI/front/src/utils/i18n/locales/pt/experiments.json b/DashAI/front/src/utils/i18n/locales/pt/experiments.json index 5e68b14b8..86ef9313a 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/pt/experiments.json @@ -34,53 +34,57 @@ "label": { "addModelsToExperiment": "Adicionar Modelos ao seu Experimento", "addOptimizersToExperiment": "Adicionar Otimizadores ao seu Experimento", + "cardinalityAtLeast": "pelo menos {{min}}", + "cardinalityBetween": "{{min}} a {{max}}", "columnsInvalidRequirements": "As colunas de Entrada e Saída atuais não correspondem aos requisitos de {{taskName}}", "columnsValidRequirements": "As colunas de Entrada e Saída atuais correspondem aos requisitos de {{taskName}}", "configureExperimentsSubtitle": "Configure experimentos para treinar modelos.", "configureModels": "Configurar modelos", "configureOptimizer": "Configurar otimização de hiperparâmetros", + "crossValidation": "Validação Cruzada", "currentExperiments": "Experimentos Atuais", "currentOptimizerSettings": "Configuração Atual do Otimizador {{optimizer}}", + "cvType": "Tipo de Validação Cruzada", "datasetInputColumnRequirements": "<0>As colunas de entrada devem ser dos tipos<1><2>, e devem ter uma cardinalidade de <3>{{cardinality}}.", "datasetOutputColumnRequirements": "<0>As colunas de saída devem ser dos tipos<1><2>, e devem ter uma cardinalidade de <3>{{cardinality}}.", - "cardinalityAtLeast": "pelo menos {{min}}", - "cardinalityBetween": "{{min}} a {{max}}", "duration": "Duração", "endTime": "Hora de Conclusão", "experimentName": "Nome do Experimento", "experimentsModuleTitle": "Módulo de Experimentos", + "groupColumn": "Coluna para Agrupamento", + "groupColumnDescription": "Coluna a ser usada para o agrupamento. As amostras pertencentes ao mesmo grupo permanecerão juntas na mesma dobra, evitando que um grupo apareça simultaneamente no treinamento e no teste.", + "holdout": "Holdout", "manual": "Manual", "metricToOptimize": "Métrica a Otimizar", "missingValues": "valores ausentes", "missingValuesDetected": "O conjunto de dados contém valores ausentes (NaN) nas colunas:", "modelName": "Nome do Modelo", "modelsInExperiment": "Modelos Atuais no Experimento", + "noConverterAdded": "Nenhum conversor foi adicionado.", "noDatasetsAvailable": "Nenhum Conjunto de Dados Disponível", "noDatasetsAvailableGoToDataTab": "Vá para a <1>aba de dados para enviar um primeiro.", "noModelsAvailable": "Nenhum Modelo Disponível", "noOptimizersNoMetric": "Sem otimização de hiperparâmetros", + "numFolds": "Número de Dobras (k)", + "numFoldsDescription": "Número de dobras da validação cruzada. Mais dobras podem melhorar a estabilidade da avaliação, mas aumentam o tempo de processamento. Deve ser um número inteiro maior que 1.", + "numRepeats": "Número de Repetições", + "numRepeatsDescription": "Número de vezes que o processo de validação cruzada será repetido. Um valor mais alto gera valores mais precisos, mas requer mais tempo de treinamento. Deve ser um número inteiro maior que 1.", "optimizer": "Otimizador", "optimizerMetric": "Métrica de Otimização", "parameterModification": "Modificação de Parâmetros", - "prepareDataset": "Preparar conjunto de dados", "predefined": "Predefinido", + "prepareDataset": "Preparar conjunto de dados", "random": "Aleatório", "recommendPreprocessMissingValues": "Recomenda-se pré-processar o conjunto de dados para tratar esses valores ausentes antes de treinar um modelo.", "rowIndexes": "Índices de linhas", "rowIndexesDescription": "Especifica intervalos de linhas para cada partição usando valores separados por vírgulas ou intervalos (ex.: 0-100, 200).", "seed": "Semente", + "selectAColumn": "Selecione uma coluna", + "selectDataset": "Selecionar conjunto de dados", "selectDatasetColumns": "Indique quais colunas do conjunto de dados serão usadas como entrada e saída.", "selectDatasetTitle": "Selecione um conjunto de dados para a tarefa selecionada", - "selectHowToDivideDataset": "Selecione como dividir o conjunto de dados em subconjuntos de treinamento, validação e teste.", "selectEvaluationStrategy": "Estratégia de avaliação", - "holdout": "Holdout", - "crossValidation": "Validação Cruzada", - "cvType": "Tipo de Validação Cruzada", - "numFolds": "Número de Dobras (k)", - "numFoldsDescription": "Número de dobras da validação cruzada. Mais dobras podem melhorar a estabilidade da avaliação, mas aumentam o tempo de processamento. Deve ser um número inteiro maior que 1.", - "numRepeats": "Número de Repetições", - "numRepeatsDescription": "Número de vezes que o processo de validação cruzada será repetido. Um valor mais alto gera valores mais precisos, mas requer mais tempo de treinamento. Deve ser um número inteiro maior que 1.", - "groupColumn": "Coluna para Agrupamento", + "selectHowToDivideDataset": "Selecione como dividir o conjunto de dados em subconjuntos de treinamento, validação e teste.", "selectInputOutputColumnsDescription": "Selecione nomes de colunas das listas.", "selectMetrics": "Selecionar métricas", "selectModelFirst": "Selecione um Modelo Primeiro", @@ -88,20 +92,17 @@ "setNameAndTask": "Definir nome e tarefa", "shuffle": "Embaralhar", "shuffleDescription": "Determina se os dados serão misturados ao definir os conjuntos ou não. Deve ser verdadeiro para misturar os dados, caso contrário falso.", + "splitType": "Tipo de partição", "splits": "Partições", "splitsDescription": "Proporção de dados atribuída a cada subconjunto. Os valores devem estar entre 0 e 1 e somar 1.", - "splitType": "Tipo de partição", "startTime": "Hora de Início", "stratify": "Estratificar", "stratifyDescription": "Define se os dados serão separados proporcionalmente conforme a distribuição de classes em cada conjunto. Embaralhar deve ser verdadeiro para estratificar os dados.", + "stratifyRequiresShuffle": "Requer que embaralhamento esteja ativado", "useManualSplittingBySpecifyingRowIndexes": "Usar divisão manual especificando os índices de linha de cada subconjunto", "usePredefinedSplitsFromDataset": "Usar divisões predefinidas do conjunto de dados", "usePredefinedSplitsFromDatasetNotAvailable": "Usar divisões predefinidas do conjunto de dados (não disponível)", - "useRandomRowsBySpecifyingPortion": "Usar linhas aleatórias especificando qual porção do conjunto de dados deseja usar para cada subconjunto", - "selectAColumn": "Selecione uma coluna", - "groupColumnDescription": "Coluna a ser usada para o agrupamento. As amostras pertencentes ao mesmo grupo permanecerão juntas na mesma dobra, evitando que um grupo apareça simultaneamente no treinamento e no teste.", - "stratifyRequiresShuffle": "Requer que embaralhamento esteja ativado", - "selectDataset": "Selecionar conjunto de dados" + "useRandomRowsBySpecifyingPortion": "Usar linhas aleatórias especificando qual porção do conjunto de dados deseja usar para cada subconjunto" }, "message": { "confirmDeleteRun": "Tem certeza de que deseja excluir esta execução? Esta ação não pode ser desfeita.", diff --git a/DashAI/front/src/utils/i18n/locales/pt/models.json b/DashAI/front/src/utils/i18n/locales/pt/models.json index aa2855b7b..4a4f53d67 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/models.json +++ b/DashAI/front/src/utils/i18n/locales/pt/models.json @@ -1,22 +1,22 @@ { "button": { + "addConverter": "Adicionar Conversor", + "addNewPrediction": "Adicionar Nova Previsão", "createExplainer": "Criar Explicador", "createGlobalExplainer": "Novo Explicador Global", "createLocalExplainer": "Novo Explicador Local", "createPrediction": "Criar Previsão", - "addNewPrediction": "Adicionar Nova Previsão", - "uploadNewDataset": "Enviar Novo Conjunto de Dados", "createSession": "Criar Sessão", "deleteAndRetrain": "Excluir e Retreinar", "deleteRun": "Excluir Execução", "hideOperations": "Ocultar Operações", "hideParameters": "Ocultar Parâmetros", "modelsHub": "Central de Modelos", - "newSession": "Nova Sessão", "modifyParameters": "Modificar Parâmetros", - "newPrediction": "Nova Previsão", "newDatasetPrediction": "Nova Previsão de Conjunto de Dados", "newManualPrediction": "Nova Previsão Manual", + "newPrediction": "Nova Previsão", + "newSession": "Nova Sessão", "retrain": "Retreinar", "runAll": "Executar Tudo", "runAllModels": "Executar Todos os Modelos", @@ -24,7 +24,8 @@ "saveAndRunModel": "Salvar e Executar Modelo", "showOperations": "Mostrar Operações", "showParameters": "Mostrar Parâmetros", - "updateAndRetrain": "Atualizar e Retreinar" + "updateAndRetrain": "Atualizar e Retreinar", + "uploadNewDataset": "Enviar Novo Conjunto de Dados" }, "error": { "completeRequiredFields": "Por favor, preencha todos os campos obrigatórios", @@ -71,149 +72,101 @@ }, "label": { "addModelToSession": "Adicionar Modelo à Sessão", + "allRepetitions": "Todas as Repetições", + "alternativeHypothesis": "Hipótese Alternativa", + "applyPreprocessing": "Aplicar pré-processamento", + "applyPreprocessingDescription": "Os conversores selecionados são ajustados apenas com os dados de treinamento e aplicados ao restante, para evitar vazamento de dados. Se não for aplicado, o modelo treina com os dados como estão.", "availableExplainers": "Explicadores Disponíveis", "availableModels": "Modelos Disponíveis", - "noSavedTests": "Ainda não há testes salvos.", - "name": "Nome", - "metric": "Métrica", - "metricSplit": "Divisão", - "significantsCount_one": "{{count}} / {{total}} Significativo", - "significantsCount_many": "{{count}} / {{total}} Significativos", - "significantsCount_other": "{{count}} / {{total}} Significativos", - "bestModel": "Melhor", - "score": "Pontuação", - "scoreProfile": "Perfil de Pontuação", - "scoreHeaderTooltip": "Pontuação ponderada 0–100 conforme o perfil selecionado. As métricas de erro sem limite superior (MAE, RMSE, TER, …) são normalizadas em relação ao pior modelo nesta comparação. Passe o cursor sobre uma célula para ver o detalhamento.", - "profile_balanced": "Equilíbrio", - "profile_detectPositives": "Detectar Positivos", - "profile_avoidFalseAlarms": "Evitar Falsos Alarmes", - "profile_probabilityQuality": "Qualidade de Probabilidades", - "profile_regression_fit": "Ajuste do Modelo", - "profile_regression_error": "Erro Balanceado", - "profile_translation_quality": "Qualidade de Tradução", - "profile_translation_balanced": "Tradução Balanceada", - "profile_text_balanced": "Equilíbrio", - "profile_text_detectPositives": "Detectar Positivos", - "profile_text_avoidFalseAlarms": "Evitar Falsos Alarmes", - "profile_text_probabilityQuality": "Qualidade de Probabilidades", - "optimizer": "Otimizador", - "nestedCrossValidation": "Validação Cruzada Aninhada", - "outerSplitterInherited": "Configuração do divisor inicial da sessão herdada para o loop externo de validação cruzada aninhada", - "innerLoopConfiguration": "Configure a partição de dados para o loop interno de validação cruzada aninhada", - "outerSplitter": "Divisor para o loop externo", - "outerFolds": "Número de folds", - "innerSplitter": "Divisor para o loop interno", - "innerFolds": "Número de folds", - "modelsToCompare": "Modelos a comparar", - "significanceLevel": "Nível de significância", - "noFinishedRuns": "Não há execuções concluídas disponíveis", - "model1": "Modelo 1", - "model2": "Modelo 2", - "nemenyiPairwiseComparisons": "Comparações post-hoc de Nemenyi após o teste de Friedman", - "tukeyPairwiseComparisons": "Comparações post-hoc de Tukey após o teste ANOVA", - "wilcoxonPairwiseComparisons": "Testes de Wilcoxon aos pares com correção de Holmes", - "alternativeHypothesis": "Hipótese Alternativa", - "correctionMethod": "Método de correção", - "result": "Resultado", - "statistic": "Estatística", - "significant": "Significativo", - "notSignificant": "Não Significativo", - "technicalDetails": "Detalhes Técnicos", - "repetition": "Repetição", - "allRepetitions": "Todas as Repetições", "averaged": "Média", - "lines": "Linhas", - "histogramPlot": "Histograma", - "foldNumber": "Número do Fold", - "theoricalQuantiles": "Quantis Teóricos", - "sampleQuantiles": "Quantis da Amostra", - "metricValue": "Valor da Métrica", - "frequency": "Frequência", - "graphs": "Gráficos", - "helperTests": "Testes de Pressupostos", - "selectAtLeastRuns": "Selecione pelo menos {{min}} modelos", - "selectExactlyRuns_one": "Selecione exatamente {{count}} modelo", - "selectExactlyRuns_many": "Selecione exatamente {{count}} modelos", - "selectExactlyRuns_other": "Selecione exatamente {{count}} modelos", - "normalityByRunSummary": "{{normal}} de {{total}} execuções parecem seguir uma distribuição normal (α = {{alpha}}).", - "notNormal": "Não normal", - "normal": "Normal", - "selectBetweenRuns": "Selecione entre {{min}} e {{max}} modelos", - "saveDetails": "Detalhes para salvar", - "testName": "Nome do Teste (opcional)", - "testNameHelp": "Se deixar em branco, será usado o nome do teste.", - "testDescription": "Descrição (opcional)", - "resultSaved": "Salvo", - "saveResult": "Salvar resultado", - "withHpo": "Com HPO", - "withoutHpo": "Sem HPO", - "nestedCv": "Validação Cruzada Aninhada", + "bar": "Barra", + "bestModel": "Melhor", + "chartType": "Tipo de gráfico", "chooseTaskForSessionWithDataset": "Escolha a tarefa de aprendizado de máquina para sua sessão com o conjunto de dados \"{{datasetName}}\".", "configuration": "Configuração", "configureModel": "Configurar Modelo", "configureOptimizer": "Configurar Otimizador", "configureSession": "Configurar Sessão", "configureTasksTrainCompareModels": "Configure tarefas, treine e compare modelos em sessões organizadas. Selecione uma tarefa para começar seu fluxo de trabalho de modelagem.", - "confirmDeleteSession": "Tem certeza de que deseja excluir a sessão \"{{name}}\"? Esta ação não pode ser desfeita.", - "confirmBulkDeleteSessions_one": "Tem certeza de que deseja excluir a sessão selecionada? Esta ação não pode ser desfeita.", + "configureTest": "Configurar Teste", "confirmBulkDeleteSessions_many": "Tem certeza de que deseja excluir as {{count}} sessões selecionadas? Esta ação não pode ser desfeita.", + "confirmBulkDeleteSessions_one": "Tem certeza de que deseja excluir a sessão selecionada? Esta ação não pode ser desfeita.", "confirmBulkDeleteSessions_other": "Tem certeza de que deseja excluir as {{count}} sessões selecionadas? Esta ação não pode ser desfeita.", - "selectSessionsToDelete": "Selecionar sessões para excluir", + "confirmDeleteSession": "Tem certeza de que deseja excluir a sessão \"{{name}}\"? Esta ação não pode ser desfeita.", + "confirmParameterUpdate": "Confirmar Atualização de Parâmetros", + "correctionMethod": "Método de correção", "customMetrics": "Métricas Personalizadas", "datasetPredictions": "Previsões de Conjunto de Dados", + "divideColumnsAndSplits": "Divida colunas e configure as partições do conjunto de dados", "dropModelHere": "Solte aqui para adicionar", "editRun": "Editar execução", - "chartType": "Tipo de gráfico", - "savedTests": "Testes salvos", - "foldGraphs": "Gráficos de Folds", - "nestedCvResults": "Resultados Aninhados", - "outerFoldFetch": "Folds Externos", - "finalFoldFetch": "Folds Finais", - "twoSided": "Ambos os lados (Modelo 1 ≠ Modelo 2)", - "greater": "Maior (Modelo 1 > Modelo 2)", - "less": "Menor (Modelo 1 < Modelo 2)", "editRunParameters": "Editar parâmetros e executar novamente o modelo", "epoch": "Época", "exitModelDetailToAddModels": "Volte para a visão geral da sessão para adicionar mais modelos.", "experimentResults": "Resultados do experimento {{name}}", "explainability": "Explicabilidade", - "explainersCount_one": "• <1>{{count}} explicador", "explainersCount_many": "• <1>{{count}} explicadores", + "explainersCount_one": "• <1>{{count}} explicador", "explainersCount_other": "• <1>{{count}} explicadores", + "finalFoldFetch": "Folds Finais", + "foldGraphs": "Gráficos de Folds", + "foldNumber": "Número do Fold", + "frequency": "Frequência", "generalMetrics": "Métricas Gerais", "globalExplainer": "Explicador Global", "globalExplainers": "Explicadores Globais", "goalMetric": "Métrica Objetivo", + "graphs": "Gráficos", + "greater": "Maior (Modelo 1 > Modelo 2)", "heatmap": "Mapa de calor", + "helperTests": "Testes de Pressupostos", "hideResults": "Ocultar Resultados", "higherIsBetter": "Maior é melhor", + "histogramPlot": "Histograma", "hyperparameterOptimizationPlots": "Gráficos de Otimização de Hiperparâmetros", "hyperparameterOptimizerConfiguration": "Configuração do Otimizador de Hiperparâmetros", "hyperparameters": "Hiperparâmetros", + "innerFolds": "Número de folds", + "innerLoopConfiguration": "Configure a partição de dados para o loop interno de validação cruzada aninhada", + "innerSplitter": "Divisor para o loop interno", "inputColumns": "Colunas de entrada", + "less": "Menor (Modelo 1 < Modelo 2)", + "lines": "Linhas", "liveMetrics": "Métricas ao Vivo", "localExplainer": "Explicador Local", "localExplainers": "Explicadores Locais", "lowerIsBetter": "Menor é melhor", - "manualPredictions": "Previsões Manuais", "manual": "Manuais", + "manualPredictions": "Previsões Manuais", + "metric": "Métrica", + "metricSplit": "Divisão", + "metricToOptimize": "Métrica a Otimizar", + "metricValue": "Valor da Métrica", "metrics": "Métricas", "metricsEmptyForDisplaySet": "As métricas de resultado para {{set}} estão vazias.", - "metricToOptimize": "Métrica a Otimizar", + "minTwoRunsRequired": "Selecione pelo menos 2 execuções concluídas para realizar testes estatísticos", + "model1": "Modelo 1", + "model2": "Modelo 2", "modelComparison": "Comparação de Modelos", "modelConfiguration": "Configuração do Modelo", - "modelCount_one": "{{count}} modelo", "modelCount_many": "{{count}} modelos", + "modelCount_one": "{{count}} modelo", "modelCount_other": "{{count}} modelos", - "bar": "Barra", "modelsModule": "Módulo de Modelos", + "modelsToCompare": "Modelos a comparar", + "name": "Nome", "nameYourSession": "Nomeie sua Sessão", + "nemenyiPairwiseComparisons": "Comparações post-hoc de Nemenyi após o teste de Friedman", + "nestedCrossValidation": "Validação Cruzada Aninhada", + "nestedCv": "Validação Cruzada Aninhada", + "nestedCvResults": "Resultados Aninhados", "noCompatibleExplainersFound": "Nenhum explicador compatível encontrado", "noCompatibleModelsFound": "Nenhum modelo compatível encontrado", "noCompletedRuns": "Ainda não há execuções concluídas", "noConfigurationAvailable": "Nenhuma configuração disponível para esta execução", "noDatasetPredictionsYet": "Ainda não há previsões de conjunto de dados", "noExplainersMatchSearch": "Nenhum explicador corresponde à sua busca", + "noFinishedRuns": "Não há execuções concluídas disponíveis", "noGlobalExplainersYet": "Ainda não há explicadores globais", "noHyperparameterPlotsAvailable": "Não há gráficos de hiperparâmetros disponíveis. Apenas execuções com pelo menos um parâmetro otimizável geram gráficos.", "noLocalExplainersYet": "Ainda não há explicadores locais", @@ -223,19 +176,50 @@ "noModelsMatchSearch": "Nenhum modelo corresponde à sua busca", "noPredictionsYet": "Ainda não há previsões", "noRunsYet": "Ainda não há execuções. Adicione modelos pelo painel direito.", + "noSavedTests": "Ainda não há testes salvos.", "noSessionSelected": "Nenhuma Sessão Selecionada", + "noTestsMatch": "Nenhum teste coincide com sua pesquisa", + "nonParametricTests": "Testes Não Paramétricos", + "normal": "Normal", + "normalityByRunSummary": "{{normal}} de {{total}} execuções parecem seguir uma distribuição normal (α = {{alpha}}).", + "notNormal": "Não normal", + "notSignificant": "Não Significativo", "operations": "Operações", "operationsWillBeDeletedWarning": "Estas operações serão excluídas permanentemente e não podem ser recuperadas. Tem certeza de que deseja continuar?", + "optimizer": "Otimizador", "optimizerConfiguration": "Configure as definições do otimizador de hiperparâmetros", "optimizerParameters": "Parâmetros do Otimizador", + "outerFoldFetch": "Folds Externos", + "outerFolds": "Número de folds", + "outerSplitter": "Divisor para o loop externo", + "outerSplitterInherited": "Configuração do divisor inicial da sessão herdada para o loop externo de validação cruzada aninhada", "outputColumns": "Colunas de saída", + "parametricTests": "Testes Paramétricos", "pleaseSelectMetricToOptimize": "Por favor, selecione uma métrica para otimizar.", "predictions": "Previsões", - "predictionsCount_one": "• <1>{{count}} previsão", "predictionsCount_many": "• <1>{{count}} previsões", + "predictionsCount_one": "• <1>{{count}} previsão", "predictionsCount_other": "• <1>{{count}} previsões", - "divideColumnsAndSplits": "Divida colunas e configure as partições do conjunto de dados", "prepareDataset": "Preparar Conjunto de Dados", + "preprocessingFailed": "O pré-processamento falhou", + "preprocessingInProgress": "Processando...", + "preprocessingOptional": "Pré-processamento (opcional)", + "preprocessingOptionalDescription": "Os conversores selecionados são ajustados apenas com os dados de treinamento e aplicados ao restante, para evitar vazamento de dados.", + "profile_avoidFalseAlarms": "Evitar Falsos Alarmes", + "profile_balanced": "Equilíbrio", + "profile_detectPositives": "Detectar Positivos", + "profile_probabilityQuality": "Qualidade de Probabilidades", + "profile_regression_error": "Erro Balanceado", + "profile_regression_fit": "Ajuste do Modelo", + "profile_text_avoidFalseAlarms": "Evitar Falsos Alarmes", + "profile_text_balanced": "Equilíbrio", + "profile_text_detectPositives": "Detectar Positivos", + "profile_text_probabilityQuality": "Qualidade de Probabilidades", + "profile_translation_balanced": "Tradução Balanceada", + "profile_translation_quality": "Qualidade de Tradução", + "repetition": "Repetição", + "result": "Resultado", + "resultSaved": "Salvo", "retrainConfirmDetails": "Tem certeza de que deseja retreinar a execução \"<1>{{runName}}\"?", "retrainModel": "Retreinar Modelo?", "retrainWillDeleteOperations": "Esta execução possui operações existentes que serão excluídas", @@ -245,47 +229,72 @@ "runFailedNoHyperparameterPlots": "Execução Falhou. Não há gráficos de hiperparâmetros disponíveis.", "runInProgressCannotEdit": "A execução está em andamento e não pode ser editada.", "runName": "Nome da Execução", - "confirmParameterUpdate": "Confirmar Atualização de Parâmetros", "runNotFound": "Execução não encontrada", "runNotStartedNoHyperparameterPlots": "Execução Não Iniciada. Não há gráficos de hiperparâmetros disponíveis.", + "runsSelected": "execuções selecionadas", + "sampleQuantiles": "Quantis da Amostra", "saveConfirmDetails": "Salvar \"<1>{{runName}}\" redefinirá seu status para 'Não Iniciado' e apagará suas métricas e resultados atuais. Tem certeza de que deseja continuar?", + "saveDetails": "Detalhes para salvar", "saveParameterChanges": "Salvar Alterações de Parâmetros?", + "saveResult": "Salvar resultado", "saveWillDeleteOperationsDetails": "Salvar \"<1>{{runName}}\" redefinirá a execução. O seguinte será excluído ao retreinar:", + "savedTests": "Testes salvos", + "score": "Pontuação", + "scoreHeaderTooltip": "Pontuação ponderada 0–100 conforme o perfil selecionado. As métricas de erro sem limite superior (MAE, RMSE, TER, …) são normalizadas em relação ao pior modelo nesta comparação. Passe o cursor sobre uma célula para ver o detalhamento.", + "scoreProfile": "Perfil de Pontuação", "searchDatasetsSessions": "Buscar Conjuntos de Dados e Sessões", "searchMetric": "Buscar métrica...", "searchModels": "Buscar Modelos...", "searchTests": "Buscar Testes...", - "statisticalTests": "Testes Estatísticos", - "parametricTests": "Testes Paramétricos", - "nonParametricTests": "Testes Não Paramétricos", - "minTwoRunsRequired": "Selecione pelo menos 2 execuções concluídas para realizar testes estatísticos", - "noTestsMatch": "Nenhum teste coincide com sua pesquisa", - "runsSelected": "execuções selecionadas", - "configureTest": "Configurar Teste", + "selectAtLeastRuns": "Selecione pelo menos {{min}} modelos", + "selectBetweenRuns": "Selecione entre {{min}} e {{max}} modelos", + "selectColumnsDescription": "Escolha quais colunas são de entrada e qual é a de saída para esta sessão.", + "selectColumnsTitle": "Selecionar Colunas", "selectDataset": "Selecionar Conjunto de Dados", - "selectDatasetForSession": "Selecione um conjunto de dados para sua sessão", "selectDatasetAndPrepare": "Nomeie sua sessão, selecione um conjunto de dados e configure suas colunas e partições.", "selectDatasetFirst": "Selecione um conjunto de dados para configurar suas colunas e partições de linhas.", + "selectDatasetForSession": "Selecione um conjunto de dados para sua sessão", + "selectExactlyRuns_many": "Selecione exatamente {{count}} modelos", + "selectExactlyRuns_one": "Selecione exatamente {{count}} modelo", + "selectExactlyRuns_other": "Selecione exatamente {{count}} modelos", "selectSessionToViewModels": "Selecione uma sessão para ver os modelos disponíveis.", + "selectSessionsToDelete": "Selecionar sessões para excluir", "selectTask": "Selecionar uma Tarefa", "selectTaskForSession": "Selecione uma Tarefa para sua Sessão", "sessionConfiguration": "Configuração da Sessão", "sessionName": "Nome da Sessão", "showResults": "Mostrar Resultados", + "significanceLevel": "Nível de significância", + "significant": "Significativo", + "significantsCount_many": "{{count}} / {{total}} Significativos", + "significantsCount_one": "{{count}} / {{total}} Significativo", + "significantsCount_other": "{{count}} / {{total}} Significativos", + "statistic": "Estatística", + "statisticalTests": "Testes Estatísticos", "step": "Etapa", + "technicalDetails": "Detalhes Técnicos", "test": "Teste", + "testDescription": "Descrição (opcional)", "testMetrics": "Métricas de Teste", + "testName": "Nome do Teste (opcional)", + "testNameHelp": "Se deixar em branco, será usado o nome do teste.", "testSet": "conjunto de teste", + "theoricalQuantiles": "Quantis Teóricos", "thereAreNoMetricsForThisRun": "não há métricas associadas ao {{set}} nesta execução", "tourDisabledMessage": "Volte ao início para começar o tour", "train": "Treinamento", - "trainingMetrics": "Métricas de Treinamento", "trainSet": "conjunto de treinamento", + "trainingMetrics": "Métricas de Treinamento", "trial": "Tentativa", + "tukeyPairwiseComparisons": "Comparações post-hoc de Tukey após o teste ANOVA", + "twoSided": "Ambos os lados (Modelo 1 ≠ Modelo 2)", "validation": "Validação", "validationMetrics": "Métricas de Validação", "validationSet": "conjunto de validação", - "viewResultsAs": "Ver resultados como colunas ou gráficos" + "viewResultsAs": "Ver resultados como colunas ou gráficos", + "wilcoxonPairwiseComparisons": "Testes de Wilcoxon aos pares com correção de Holmes", + "withHpo": "Com HPO", + "withoutHpo": "Sem HPO" }, "message": { "allRunsCompleted": "{{experiment}} concluiu todas as suas execuções.", diff --git a/DashAI/front/src/utils/i18n/locales/zh/datasets.json b/DashAI/front/src/utils/i18n/locales/zh/datasets.json index fd4db7f53..04b26007a 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/zh/datasets.json @@ -51,6 +51,7 @@ "datasetHub": "数据文件中心" }, "error": { + "cannotSaveEmptyDataset": "无法保存数据集：所有列已被删除。", "converterFailed": "处理转换器时发生错误。", "converterFailedWithInfo": "处理转换器时出错：{{error}}", "createConverterError": "创建转换器失败", @@ -67,68 +68,52 @@ "failedToCreateDataset": "创建数据集时出错：{{error}}", "failedToCreateDatasetFromNotebook": "从笔记本创建数据集失败", "failedToCreateExplorer": "创建探索器失败", + "failedToCreateFolder": "创建文件夹失败", "failedToDeleteDataset": "删除数据集失败", "failedToDeleteDatasets": "删除所选数据集失败", - "failedToCreateFolder": "创建文件夹失败", - "failedToUpdateFolder": "更新文件夹失败", "failedToDeleteFolder": "删除文件夹失败", - "folderNameExists": "已存在同名文件夹", - "failedToMoveDataset": "移动数据集失败", "failedToDeleteNotebook": "删除笔记本失败", "failedToDeleteNotebooks": "删除所选笔记本失败", "failedToFetchNotebooks": "获取笔记本列表失败", "failedToLoadDatasetInfo": "获取数据集信息失败", + "failedToMoveDataset": "移动数据集失败", + "failedToResetExplorerResults": "恢复原始图表失败", "failedToUpdateDataset": "更新数据集失败", "failedToUpdateExplorerResults": "更新探索器结果失败", - "failedToResetExplorerResults": "恢复原始图表失败", + "failedToUpdateFolder": "更新文件夹失败", "failedToUpdateNotebook": "更新笔记本失败", + "fetchingConverters": "获取可用转换器时出错。", "fetchingDataloaders": "获取兼容数据加载器时出错。", "fetchingDatasetColumns": "获取数据集列时出错。", "fetchingExplorersConverters": "获取探索器/转换器失败", "fileTypeNotAllowed": "所选数据加载器不允许此文件类型。", + "folderNameExists": "已存在同名文件夹", "loadingDatasetPreview": "加载数据集预览时出错", "noDatasetDataAvailable": "无可用数据集数据", "noDatasetFileAvailable": "无可用数据集文件", - "notebookNameEmpty": "笔记本名称不能为空", "noValidColumnsForExplorer": "此探索器没有有效的可用列。", "noValidColumnsWithDtypesMentioned": "此数据集没有所需类型（{{dtypes}}）的列。", + "notebookNameEmpty": "笔记本名称不能为空", "processConverterError": "处理转换器失败", "requiredFieldsMissing": "必填字段缺失", "requiresExactColumns_one": "需要恰好 {{required}} 个有效列，但只有 {{available}} 个可用。", "requiresExactColumns_other": "需要恰好 {{required}} 个有效列，但只有 {{available}} 个可用。", "requiresMinColumns_one": "至少需要 {{required}} 个有效列，但只有 {{available}} 个可用。", "requiresMinColumns_other": "至少需要 {{required}} 个有效列，但只有 {{available}} 个可用。", - "zipContentsNotCompatible": "ZIP 文件不包含与所选数据加载器兼容的文件", - "cannotSaveEmptyDataset": "无法保存数据集：所有列已被删除。" + "zipContentsNotCompatible": "ZIP 文件不包含与所选数据加载器兼容的文件" }, "label": { - "task": "任务", + "aboutTool": "关于 {{toolName}}", "all": "全部", + "allTypeChangesValid": "所有类型更改均有效，可以安全应用。", "allowedDataTypes": "允许的数据类型：<1><0>", "allowedValueTypes": "允许的值类型：<1><0>", - "excludedDataTypes": "排除的数据类型：{{dtypes}}", - "allTypeChangesValid": "所有类型更改均有效，可以安全应用。", "analysisTools": "分析工具", "appearance": "外观", "appliedTransformations": "已应用的变换：", "associatedDataset": "关联数据集", "atLeastTwoColorStopsRequired": "未定义色标点。请至少添加两个色标点。", "availableDatasets": "可用数据集", - "confirmDeleteFolder": "删除文件夹", - "confirmDeleteFolderContent": "确定要删除文件夹 \"{{name}}\" 吗？其中的数据集将移至无文件夹。", - "noFolder": "无文件夹", - "newFolder": "新建文件夹", - "folderName": "文件夹名称", - "confirmDeleteDataset": "确定要删除数据集 \"{{name}}\" 吗？此操作无法撤销。", - "confirmDeleteDatasetLinkedWarning": "所有与此数据集关联的笔记本和会话也将被删除。", - "confirmBulkDeleteDatasets_one": "确定要删除所选的数据集吗？此操作无法撤销。", - "confirmBulkDeleteDatasets_other": "确定要删除所选的 {{count}} 个数据集吗？此操作无法撤销。", - "selectDatasetsToDelete": "选择要删除的数据集", - "confirmDeleteNotebook": "确定要删除笔记本 \"{{name}}\" 吗？此操作无法撤销。", - "confirmBulkDeleteNotebooks": "确定要删除所选的 {{count}} 个笔记本吗？此操作无法撤销。", - "confirmBulkDeleteNotebooks_one": "确定要删除所选的笔记本吗？此操作无法撤销。", - "confirmBulkDeleteNotebooks_other": "确定要删除所选的 {{count}} 个笔记本吗？此操作无法撤销。", - "selectNotebooksToDelete": "选择要删除的笔记本", "avg": "平均值", "avgLength": "平均长度", "avgWordCount": "平均词数", @@ -153,6 +138,7 @@ "classTargetColumn": "类别/目标列", "clickToUpload": "点击上传", "color": "颜色", + "colorStops": "色标点", "colorbarBorderColor": "色标边框颜色", "colorbarBorderWidth": "色标边框宽度", "colorbarTickFontColor": "色标刻度字体颜色", @@ -160,7 +146,7 @@ "colors": "颜色", "colorscale": "色阶", "colorscaleMode": "色阶模式", - "colorStops": "色标点", + "columnInsights": "列分析", "columnName": "列名", "columnTypesDistribution": "列类型分布", "configureAndUpload": "配置并上传", @@ -169,10 +155,20 @@ "configureParametersStep": "步骤 {{step}}：配置参数", "configureScope": "配置范围", "configureToolTitle": "配置 {{toolType}}：{{toolName}}", - "columnInsights": "列分析", + "confirmBulkDeleteDatasets_one": "确定要删除所选的数据集吗？此操作无法撤销。", + "confirmBulkDeleteDatasets_other": "确定要删除所选的 {{count}} 个数据集吗？此操作无法撤销。", + "confirmBulkDeleteNotebooks": "确定要删除所选的 {{count}} 个笔记本吗？此操作无法撤销。", + "confirmBulkDeleteNotebooks_one": "确定要删除所选的笔记本吗？此操作无法撤销。", + "confirmBulkDeleteNotebooks_other": "确定要删除所选的 {{count}} 个笔记本吗？此操作无法撤销。", + "confirmDeleteDataset": "确定要删除数据集 \"{{name}}\" 吗？此操作无法撤销。", + "confirmDeleteDatasetLinkedWarning": "所有与此数据集关联的笔记本和会话也将被删除。", + "confirmDeleteFolder": "删除文件夹", + "confirmDeleteFolderContent": "确定要删除文件夹 \"{{name}}\" 吗？其中的数据集将移至无文件夹。", + "confirmDeleteNotebook": "确定要删除笔记本 \"{{name}}\" 吗？此操作无法撤销。", "constantColumns": "常量列", "convert": "转换", "converter": "转换器", + "converterOutput": "输出", "correlation": "相关性", "correlationAnalysis": "相关性分析", "correlations": "相关性", @@ -180,43 +176,46 @@ "createNewNotebook": "笔记本", "createNewNotebookDescription": "使用现有数据集开始新的分析会话。", "customArray": "自定义数组", - "dataloaderConfiguration": "数据加载器配置", "dataQuality": "数据质量", "dataQualityScoreTooltip": "数据质量评分基于多种因素计算，包括缺失值、重复行和数据一致性。评分越高表示数据质量越好。", "dataQualitySummary": "数据质量摘要", + "dataType": "数据类型", + "dataloaderConfiguration": "数据加载器配置", "datasetDescription": "{{rows}} 行，{{columns}} 列", "datasetLoading": "加载数据集中...", "datasetModule": "数据集模块", "datasetModuleSubtitle": "上传数据集：使用先进的探索性分析工具探索、分析和变换您的数据。创建交互式笔记本，生成可视化图表，并直观地应用数据变换。", "datasetName": "数据集名称", "datasetPreview": "数据集预览", - "aboutTool": "关于 {{toolName}}", "datasetPreviewFor": "笔记本：{{name}} 预览", - "dataType": "数据类型", + "decreasingColor": "下降颜色", "deleteConverterConfirmation": "确定要删除转换器 \"{{converter}}\" 吗？删除此转换器也会移除之后应用的所有转换器和探索器。此操作无法撤销。", "deleteExplorerConfirmation": "确定要删除探索器 \"{{explorer}}\" 吗？此操作无法撤销。", "detailsForExplorer": "探索器详情：{{name}}", "dimensionIdx": "维度 {{idx}}：{{label}}", - "dimensionsLabels": "维度标签", "dimensionTitle": "维度标题", + "dimensionsLabels": "维度标签", "distributionMetrics": "分布指标", "dragAndDropFileHere": "拖放文件到此处", "dragToResize": "拖动以调整大小并查看更多内容", + "dropToolHere": "拖放此处以添加", "duplicatedRows": "重复行", "editPlotLayout": "编辑图表布局", "endIndex": "结束索引", + "excludedDataTypes": "排除的数据类型：{{dtypes}}", "explorationPath": "探索路径", "explorationType": "探索类型", "explore": "探索", "explorer": "探索器", + "exportCSV": "下载为 CSV", "exportCardImage": "卡片导出为图像", "exportChartImage": "图表导出为图像", - "exportCSV": "下载为 CSV", "exportImage": "下载", "exportJSON": "下载为 JSON", "exportMetrics": "将指标导出为 JSON", "fileSizeMB": "文件大小（MB）", "firstLastStopsAtExtremes": "色标点必须包含起点位置 0 和终点位置 1。", + "folderName": "文件夹名称", "fontFamily": "字体", "foundDuplicateRows_one": "在数据集中发现 {{count}} 行重复数据", "foundDuplicateRows_other": "在数据集中发现 {{count}} 行重复数据", @@ -227,19 +226,20 @@ "highCardinality": "高基数", "highCardinalityDetected": "以下列中检测到高基数：{{columns}}", "ifYourDatasetHaveSplits": "如果您的数据集有划分，请以 zip 文件格式上传", + "importFromHub": "数据文件中心", + "importFromHubDescription": "从 HuggingFace 和 OpenML 等外部来源浏览和下载数据集。", + "increasingColor": "上升颜色", "indices": "索引（逗号分隔，或输入 'all'）", "inferenceRows": "推断行数", "inferenceRowsDescription": "用于预览/类型推断的行数（最少 2 行）。", - "previewRows": "预览行数", - "previewRowsDescription": "预览中显示的行数（最少 2 行）。", "inferredConfiguration": "类型推断配置", "insightConstantColumn": "此列只有一个唯一值，不提供任何分析信息。", - "insightHighCardinality": "此分类列有超过 100 个唯一值。考虑分组或编码。", "insightEmptyMessage": "未检测到问题。当发现潜在观察时，列分析将显示在此处。", + "insightHighCardinality": "此分类列有超过 100 个唯一值。考虑分组或编码。", "insightHighNanRatio": "此列 {{value}}% 的值缺失。考虑插补或删除。", "insightLowUniqueness": "唯一值比例较低（{{value}}%）。可能是被错误分类为文本的分类变量。", - "insightOutliers_one": "检测到 {{count}} 个潜在异常值（IQR 方法）。", "insightOutliers_many": "检测到 {{count}} 个潜在异常值（IQR 方法）。", + "insightOutliers_one": "检测到 {{count}} 个潜在异常值（IQR 方法）。", "insightOutliers_other": "检测到 {{count}} 个潜在异常值（IQR 方法）。", "insightPossibleId": "所有值均唯一。此列可能是标识符，对建模无用。", "insightSkewed": "右偏分布（偏度：{{value}}）。考虑应用对数变换。", @@ -261,19 +261,15 @@ "legendYPosition": "图例 Y 位置", "lengthDistribution": "长度分布", "lengthMetrics": "长度指标", - "lowerBound": "下界", + "lineColor": "线条颜色", "lowUniquenessWarning": "警告：此文本列的唯一值比例非常低。这可能是被错误分类为文本的分类变量，可能导致分析问题。", + "lowerBound": "下界", "marginBottom": "底部边距", "marginLeft": "左侧边距", "marginRight": "右侧边距", - "margins": "边距", "marginTop": "顶部边距", + "margins": "边距", "markerColor": "标记颜色", - "lineColor": "线条颜色", - "increasingColor": "上升颜色", - "decreasingColor": "下降颜色", - "totalsColor": "总计颜色", - "transparent": "透明", "max": "最大值", "maxLength": "最大长度", "mean": "均值", @@ -287,29 +283,28 @@ "missingValuesDetected": "以下列中检测到缺失值：{{columns}}", "missingValuesOverview": "缺失值概览", "mostFrequent": "最频繁", - "nameYourNotebook": "为笔记本命名", "nComponentsColumnInfo": "您选择了 {{n}} 列。n_components 必须小于或等于 {{n}}。请减少 n_components 或选择更多列以避免错误。", + "nameYourNotebook": "为笔记本命名", "newDatasetCreatedWithTransformations": "将使用这些变换创建新数据集。可在其他模块中使用，不影响原始数据。", + "newFolder": "新建文件夹", "noDataQualityIssuesDetected": "未检测到数据质量问题", - "noDuplicateRows": "数据集中未检测到重复行。", - "qualityIssuesFound_one": "检测到 {{count}} 个问题", - "qualityIssuesFound_other": "检测到 {{count}} 个问题", "noDataset": "无数据集", "noDatasetsAvailable": "无可用数据集", + "noDuplicateRows": "数据集中未检测到重复行。", "noExplorersOrConverters": "添加您的第一个探索器或转换器开始探索！", + "noFolder": "无文件夹", "noMissingValues": "数据集中未检测到缺失值。", - "none": "无", "noStrongCorrelationsFound": "未检测到强相关性", + "noToolsMatched": "未找到匹配搜索的工具。", + "noTransformationsApplied": "未应用任何变换。", + "noTransformationsAppliedYet": "尚未应用任何变换。", + "none": "无", "notebookCreationNote": "将创建所选数据集的副本，以便在笔记本中工作而不改变原始数据。", "notebookDescription": "笔记本描述", "notebookHistory": "笔记本历史：{{notebook}}", "notebookInformation": "笔记本信息", "notebookName": "笔记本名称", "notebooks": "笔记本", - "noToolsMatched": "未找到匹配搜索的工具。", - "dropToolHere": "拖放此处以添加", - "noTransformationsApplied": "未应用任何变换。", - "noTransformationsAppliedYet": "尚未应用任何变换。", "numericalAnalysis": "数值分析", "outliers": "异常值", "overview": "概览", @@ -317,11 +312,15 @@ "position": "位置", "possibleIDColumns": "可能的 ID 列", "presetScale": "预设色阶", - "processingTitle": "正在处理您的数据集...", + "previewRows": "预览行数", + "previewRowsDescription": "预览中显示的行数（最少 2 行）。", "processingMessage": "根据数据大小，这可能需要几分钟。", + "processingTitle": "正在处理您的数据集...", "proportion": "比例", "q1": "Q1", "q3": "Q3", + "qualityIssuesFound_one": "检测到 {{count}} 个问题", + "qualityIssuesFound_other": "检测到 {{count}} 个问题", "qualityScore": "质量评分：{{value}}{{unit}}", "range": "范围", "requiredColumns": "必需列", @@ -337,6 +336,7 @@ "saveProcessedDataset": "保存处理后的数据集", "scopeColumns": "范围 - 列", "scopeRows": "范围 - 行", + "searchConverters": "搜索转换器", "searchDatasetsNotebooks": "搜索数据集和笔记本", "searchExplorersConverters": "搜索探索器/转换器", "selectColorscale": "选择色阶", @@ -346,6 +346,14 @@ "selectDataset": "选择数据集", "selectDatasetFirst": "请先选择数据集", "selectDatasetForNotebook": "为笔记本选择数据集", + "selectDatasetsToDelete": "选择要删除的数据集", + "selectNotebookToAccessAnalysisTools": "选择笔记本以访问分析工具。", + "selectNotebooksToDelete": "选择要删除的笔记本", + "selectScopeDescriptionColumns": "在此配置转换器要应用到哪些列。", + "selectScopeDescriptionRows": "在此配置转换器要应用到哪些行。", + "selectScopeStep": "步骤 {{step}}：选择范围", + "selectTargetColumnDescription": "选择一列作为监督学习的目标变量。", + "selectUploadMethod": "选择上传数据的方式", "selectedColumns_one": "已选 {{count}} 列", "selectedColumns_other": "已选 {{count}} 列", "selectedDataloaderConfiguration": "{{dataloader}} 配置", @@ -353,26 +361,21 @@ "selectedOrder": "选择顺序", "selectedRows": "已选行：{{value}}", "selectionMode": "选择模式", - "selectNotebookToAccessAnalysisTools": "选择笔记本以访问分析工具。", - "selectScopeDescriptionColumns": "在此配置转换器要应用到哪些列。", - "selectScopeDescriptionRows": "在此配置转换器要应用到哪些行。", - "selectScopeStep": "步骤 {{step}}：选择范围", - "selectTargetColumnDescription": "选择一列作为监督学习的目标变量。", - "selectUploadMethod": "选择上传数据的方式", "shapeIndicators": "形状指示器", "showGrid": "{{axis}} 轴显示网格", - "showingRowsInference": "显示用于类型推断的 {{previewRowCount}} 行中的 {{sampleLength}} 行。", - "showingRowsPreview": "显示 {{previewRowCount}} 行中的 {{sampleLength}} 行。", "showLegend": "显示图例", "showMore": "显示更多（剩余 {{count}} 个）", "showMore_other": "显示更多（剩余 {{count}} 个）", "showZeroLine": "{{axis}} 轴显示零线", + "showingRowsInference": "显示用于类型推断的 {{previewRowCount}} 行中的 {{sampleLength}} 行。", + "showingRowsPreview": "显示 {{previewRowCount}} 行中的 {{sampleLength}} 行。", "skewness": "偏度", "someTypeChangesCannotBeApplied": "某些类型更改无法应用：", "startIndex": "起始索引", "stdDev": "标准差", "strongCorrelations": "强相关性", "targetColumn": "目标列", + "task": "任务", "text": "文本", "tickLabels": "刻度标签", "tickLabelsHelper": "每行一个标签", @@ -387,19 +390,19 @@ "totalPercentage": "占总量的 {{percentage}}%", "totalRows": "总行数", "totalRowsCount": "总行数：{{total}}", + "totalsColor": "总计颜色", "tourDisabledMessage": "返回首页以开始引导", "tourDisabledMessageNotebook": "返回数据集可视化以开始引导", "traceIdx": "轨迹 {{index}}：{{trace}}", + "transparent": "透明", "typeChangeWarnings": "类型更改警告：", "typeToSearchDatasets": "输入以搜索数据集...", "unique": "唯一", - "uniquenessFormula": "唯一性 = （唯一值 ÷ 总数）× 100", "uniquePercentage": "{{percentage}}% 唯一", "uniqueValues": "唯一值", + "uniquenessFormula": "唯一性 = （唯一值 ÷ 总数）× 100", "unknownDataset": "未知数据集", "uploadAndConfigure": "上传数据集并配置参数", - "importFromHub": "数据文件中心", - "importFromHubDescription": "从 HuggingFace 和 OpenML 等外部来源浏览和下载数据集。", "uploadDataset": "数据集", "uploadDatasetBeforeCreatingSession": "创建会话前需要上传数据集。请前往数据集模块上传您的数据。", "uploadDatasetDescription": "从各种来源和格式导入您的数据", diff --git a/DashAI/front/src/utils/i18n/locales/zh/experiments.json b/DashAI/front/src/utils/i18n/locales/zh/experiments.json index e1c6aa541..26091f645 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/zh/experiments.json @@ -34,53 +34,57 @@ "label": { "addModelsToExperiment": "向实验中添加模型", "addOptimizersToExperiment": "向实验中添加优化器", + "cardinalityAtLeast": "至少 {{min}}", + "cardinalityBetween": "{{min}} 到 {{max}}", "columnsInvalidRequirements": "当前输入列和输出列不满足 {{taskName}} 的要求", "columnsValidRequirements": "当前输入列和输出列满足 {{taskName}} 的要求", "configureExperimentsSubtitle": "配置实验以训练模型。", "configureModels": "配置模型", "configureOptimizer": "配置超参数优化", + "crossValidation": "交叉验证", "currentExperiments": "当前实验", "currentOptimizerSettings": "当前优化器设置 {{optimizer}}", + "cvType": "交叉验证类型", "datasetInputColumnRequirements": "<0>输入列的类型必须为<1><2>，且基数应为<3>{{cardinality}}。", "datasetOutputColumnRequirements": "<0>输出列的类型必须为<1><2>，且基数应为<3>{{cardinality}}。", - "cardinalityAtLeast": "至少 {{min}}", - "cardinalityBetween": "{{min}} 到 {{max}}", "duration": "时长", "endTime": "结束时间", "experimentName": "实验名称", "experimentsModuleTitle": "实验模块", + "groupColumn": "分组列", + "groupColumnDescription": "用于分组的列。属于同一组的样本将保持在同一折中，避免同一组同时出现在训练集和测试集中。", + "holdout": "留出法", "manual": "手动", "metricToOptimize": "待优化指标", "missingValues": "缺失值", "missingValuesDetected": "数据集在以下列中包含缺失值（NaN）：", "modelName": "模型名称", "modelsInExperiment": "实验中的当前模型", + "noConverterAdded": "尚未添加任何转换器。", "noDatasetsAvailable": "暂无可用数据集", "noDatasetsAvailableGoToDataTab": "请前往<1>数据标签页先上传数据集。", "noModelsAvailable": "暂无可用模型", "noOptimizersNoMetric": "不进行超参数优化", + "numFolds": "折数（k）", + "numFoldsDescription": "交叉验证的折数。折数越多可以提高评估的稳定性，但会增加处理时间。必须为大于 1 的整数。", + "numRepeats": "重复次数", + "numRepeatsDescription": "交叉验证过程重复的次数。数值越高结果越精确，但需要更多训练时间。必须为大于 1 的整数。", "optimizer": "优化器", "optimizerMetric": "优化指标", "parameterModification": "参数修改", - "prepareDataset": "准备数据集", "predefined": "预定义", + "prepareDataset": "准备数据集", "random": "随机", "recommendPreprocessMissingValues": "建议在训练模型前对数据集进行预处理以处理这些缺失值。", "rowIndexes": "行索引", "rowIndexesDescription": "使用逗号分隔的值或范围（如 0-100, 200）为每个划分指定行范围。", "seed": "随机种子", + "selectAColumn": "选择一列", + "selectDataset": "选择数据集", "selectDatasetColumns": "指定数据集中用作输入和输出的列。", "selectDatasetTitle": "为所选任务选择数据集", - "selectHowToDivideDataset": "选择如何将数据集划分为训练集、验证集和测试集。", "selectEvaluationStrategy": "评估策略", - "holdout": "留出法", - "crossValidation": "交叉验证", - "cvType": "交叉验证类型", - "numFolds": "折数（k）", - "numFoldsDescription": "交叉验证的折数。折数越多可以提高评估的稳定性，但会增加处理时间。必须为大于 1 的整数。", - "numRepeats": "重复次数", - "numRepeatsDescription": "交叉验证过程重复的次数。数值越高结果越精确，但需要更多训练时间。必须为大于 1 的整数。", - "groupColumn": "分组列", + "selectHowToDivideDataset": "选择如何将数据集划分为训练集、验证集和测试集。", "selectInputOutputColumnsDescription": "从列表中选择列名。", "selectMetrics": "选择指标", "selectModelFirst": "请先选择模型", @@ -88,20 +92,17 @@ "setNameAndTask": "设置名称和任务", "shuffle": "随机打乱", "shuffleDescription": "确定在定义子集时是否对数据进行随机打乱。设为 true 则打乱，否则为 false。", + "splitType": "划分类型", "splits": "划分", "splitsDescription": "分配给每个子集的数据比例。值必须在 0 到 1 之间且总和为 1。", - "splitType": "划分类型", "startTime": "开始时间", "stratify": "按类分层", "stratifyDescription": "定义是否根据每个子集中的类别分布按比例分割数据。按类分层时需启用随机打乱。", + "stratifyRequiresShuffle": "需要启用随机打乱", "useManualSplittingBySpecifyingRowIndexes": "通过指定每个子集的行索引进行手动划分", "usePredefinedSplitsFromDataset": "使用数据集中的预定义划分", "usePredefinedSplitsFromDatasetNotAvailable": "使用数据集中的预定义划分（不可用）", - "useRandomRowsBySpecifyingPortion": "通过指定每个子集使用数据集的比例进行随机行划分", - "selectAColumn": "选择一列", - "groupColumnDescription": "用于分组的列。属于同一组的样本将保持在同一折中，避免同一组同时出现在训练集和测试集中。", - "stratifyRequiresShuffle": "需要启用随机打乱", - "selectDataset": "选择数据集" + "useRandomRowsBySpecifyingPortion": "通过指定每个子集使用数据集的比例进行随机行划分" }, "message": { "confirmDeleteRun": "确定要删除此运行吗？此操作无法撤销。", diff --git a/DashAI/front/src/utils/i18n/locales/zh/models.json b/DashAI/front/src/utils/i18n/locales/zh/models.json index 47f6c9242..ac65f3f2c 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/models.json +++ b/DashAI/front/src/utils/i18n/locales/zh/models.json @@ -1,22 +1,22 @@ { "button": { + "addConverter": "添加转换器", + "addNewPrediction": "添加新预测", "createExplainer": "创建解释器", "createGlobalExplainer": "新建全局解释器", "createLocalExplainer": "新建局部解释器", "createPrediction": "创建预测", - "addNewPrediction": "添加新预测", - "uploadNewDataset": "上传新数据集", "createSession": "创建会话", "deleteAndRetrain": "删除并重新训练", "deleteRun": "删除运行", "hideOperations": "隐藏操作", "hideParameters": "隐藏参数", "modelsHub": "模型中心", - "newSession": "新建会话", "modifyParameters": "修改参数", - "newPrediction": "新建预测", "newDatasetPrediction": "新建数据集预测", "newManualPrediction": "新建手动预测", + "newPrediction": "新建预测", + "newSession": "新建会话", "retrain": "重新训练", "runAll": "全部运行", "runAllModels": "运行所有模型", @@ -24,7 +24,8 @@ "saveAndRunModel": "保存并运行模型", "showOperations": "显示操作", "showParameters": "显示参数", - "updateAndRetrain": "更新并重新训练" + "updateAndRetrain": "更新并重新训练", + "uploadNewDataset": "上传新数据集" }, "error": { "completeRequiredFields": "请填写所有必填字段", @@ -71,103 +72,33 @@ }, "label": { "addModelToSession": "将模型添加到会话", + "allRepetitions": "所有重复", + "alternativeHypothesis": "备择假设", + "applyPreprocessing": "应用预处理", + "applyPreprocessingDescription": "所选的转换器仅在训练数据上拟合，然后应用于其余数据，以避免数据泄漏。如果不应用，模型将按原样使用数据进行训练。", "availableExplainers": "可用解释器", "availableModels": "可用模型", - "noSavedTests": "暂无已保存的检验。", - "name": "名称", - "metric": "指标", - "metricSplit": "划分", - "significantsCount_other": "{{count}} / {{total}} 显著", - "bestModel": "最优", - "score": "评分", - "scoreProfile": "评分配置文件", - "scoreHeaderTooltip": "根据所选配置文件计算的 0–100 加权评分。无上限的误差指标（MAE、RMSE、TER 等）会相对于本次比较中最差的模型进行归一化。将鼠标悬停在单元格上可查看明细。", - "profile_balanced": "均衡", - "profile_detectPositives": "检测阳性", - "profile_avoidFalseAlarms": "避免误报", - "profile_probabilityQuality": "概率质量", - "profile_regression_fit": "模型拟合度", - "profile_regression_error": "均衡误差", - "profile_translation_quality": "翻译质量", - "profile_translation_balanced": "均衡翻译", - "profile_text_balanced": "均衡", - "profile_text_detectPositives": "检测阳性", - "profile_text_avoidFalseAlarms": "避免误报", - "profile_text_probabilityQuality": "概率质量", - "nestedCrossValidation": "嵌套交叉验证", - "outerSplitterInherited": "会话初始划分器配置已继承，用于嵌套交叉验证的外层循环", - "innerLoopConfiguration": "配置嵌套交叉验证内层循环的数据划分", - "outerSplitter": "外层循环划分器", - "outerFolds": "折数", - "innerSplitter": "内层循环划分器", - "innerFolds": "折数", - "statisticalTests": "统计检验", - "modelsToCompare": "待比较模型", - "significanceLevel": "显著性水平", - "noFinishedRuns": "没有可用的已完成运行", - "result": "结果", - "statistic": "统计量", - "significant": "显著", - "notSignificant": "不显著", - "technicalDetails": "技术细节", - "model1": "模型 1", - "model2": "模型 2", - "nemenyiPairwiseComparisons": "Friedman 检验后的 Nemenyi 事后成对比较", - "tukeyPairwiseComparisons": "ANOVA 检验后的 Tukey 事后成对比较", - "wilcoxonPairwiseComparisons": "带 Holm 校正的成对 Wilcoxon 检验", - "alternativeHypothesis": "备择假设", - "correctionMethod": "校正方法", - "repetition": "重复", - "allRepetitions": "所有重复", "averaged": "平均值", - "lines": "折线", - "histogramPlot": "直方图", - "foldNumber": "折编号", - "theoricalQuantiles": "理论分位数", - "sampleQuantiles": "样本分位数", - "metricValue": "指标值", - "frequency": "频率", - "graphs": "图表", - "helperTests": "假设检验", - "selectAtLeastRuns": "请至少选择 {{min}} 个模型", - "selectExactlyRuns": "请恰好选择 {{count}} 个模型", - "selectExactlyRuns_other": "请恰好选择 {{count}} 个模型", - "normalityByRunSummary": "{{total}} 个运行中有 {{normal}} 个似乎服从正态分布（α = {{alpha}}）。", - "notNormal": "非正态", - "normal": "正态", - "selectBetweenRuns": "请选择 {{min}} 到 {{max}} 个模型", - "saveDetails": "保存详情", - "testName": "检验名称（可选）", - "testNameHelp": "如果留空，将使用检验的名称。", - "testDescription": "描述（可选）", - "resultSaved": "已保存", - "saveResult": "保存结果", - "withHpo": "含 HPO", - "withoutHpo": "不含 HPO", - "nestedCv": "嵌套交叉验证", + "bar": "条形图", + "bestModel": "最优", + "chartType": "图表类型", "chooseTaskForSessionWithDataset": "为使用数据集 \"{{datasetName}}\" 的会话选择机器学习任务。", "configuration": "配置", "configureModel": "配置模型", "configureOptimizer": "配置优化器", "configureSession": "配置会话", "configureTasksTrainCompareModels": "在有组织的会话中配置任务、训练并比较模型。选择一个任务开始建模工作流。", - "confirmDeleteSession": "确定要删除会话 \"{{name}}\" 吗？此操作无法撤销。", + "configureTest": "配置检验", "confirmBulkDeleteSessions_one": "确定要删除所选的会话吗？此操作无法撤销。", "confirmBulkDeleteSessions_other": "确定要删除所选的 {{count}} 个会话吗？此操作无法撤销。", - "selectSessionsToDelete": "选择要删除的会话", + "confirmDeleteSession": "确定要删除会话 \"{{name}}\" 吗？此操作无法撤销。", + "confirmParameterUpdate": "确认参数更新", + "correctionMethod": "校正方法", "customMetrics": "自定义指标", "datasetPredictions": "数据集预测", + "divideColumnsAndSplits": "划分列并配置数据集划分", "dropModelHere": "拖放此处以添加", "editRun": "编辑运行", - "chartType": "图表类型", - "savedTests": "已保存的检验", - "foldGraphs": "折图表", - "nestedCvResults": "嵌套结果", - "outerFoldFetch": "外层折", - "finalFoldFetch": "最终折", - "twoSided": "双侧（模型 1 ≠ 模型 2）", - "greater": "更大（模型 1 > 模型 2）", - "less": "更小（模型 1 < 模型 2）", "editRunParameters": "编辑参数并重新运行模型", "epoch": "轮次", "exitModelDetailToAddModels": "返回会话概览以添加更多模型。", @@ -175,39 +106,64 @@ "explainability": "可解释性", "explainersCount_one": "• <1>{{count}} 个解释器", "explainersCount_other": "• <1>{{count}} 个解释器", + "finalFoldFetch": "最终折", + "foldGraphs": "折图表", + "foldNumber": "折编号", + "frequency": "频率", "generalMetrics": "通用指标", "globalExplainer": "全局解释器", "globalExplainers": "全局解释器", "goalMetric": "目标指标", + "graphs": "图表", + "greater": "更大（模型 1 > 模型 2）", "heatmap": "热图", + "helperTests": "假设检验", "hideResults": "隐藏结果", "higherIsBetter": "越高越好", + "histogramPlot": "直方图", "hyperparameterOptimizationPlots": "超参数优化图表", "hyperparameterOptimizerConfiguration": "超参数优化器配置", "hyperparameters": "超参数", + "innerFolds": "折数", + "innerLoopConfiguration": "配置嵌套交叉验证内层循环的数据划分", + "innerSplitter": "内层循环划分器", "inputColumns": "输入列", + "less": "更小（模型 1 < 模型 2）", + "lines": "折线", "liveMetrics": "实时指标", "localExplainer": "局部解释器", "localExplainers": "局部解释器", "lowerIsBetter": "越低越好", - "manualPredictions": "手动预测", "manual": "手动", + "manualPredictions": "手动预测", + "metric": "指标", + "metricSplit": "划分", + "metricToOptimize": "待优化指标", + "metricValue": "指标值", "metrics": "指标", "metricsEmptyForDisplaySet": "{{set}} 的结果指标为空。", - "metricToOptimize": "待优化指标", + "minTwoRunsRequired": "请至少选择 2 个已完成的运行以进行统计检验", + "model1": "模型 1", + "model2": "模型 2", "modelComparison": "模型比较", "modelConfiguration": "模型配置", "modelCount_one": "{{count}} 个模型", "modelCount_other": "{{count}} 个模型", - "bar": "条形图", "modelsModule": "模型模块", + "modelsToCompare": "待比较模型", + "name": "名称", "nameYourSession": "为会话命名", + "nemenyiPairwiseComparisons": "Friedman 检验后的 Nemenyi 事后成对比较", + "nestedCrossValidation": "嵌套交叉验证", + "nestedCv": "嵌套交叉验证", + "nestedCvResults": "嵌套结果", "noCompatibleExplainersFound": "未找到兼容的解释器", "noCompatibleModelsFound": "未找到兼容模型", "noCompletedRuns": "暂无已完成的运行", "noConfigurationAvailable": "此运行没有可用的配置", "noDatasetPredictionsYet": "暂无数据集预测", "noExplainersMatchSearch": "没有解释器匹配您的搜索", + "noFinishedRuns": "没有可用的已完成运行", "noGlobalExplainersYet": "暂无全局解释器", "noHyperparameterPlotsAvailable": "无超参数图表可用。只有至少包含一个可优化参数的运行才会生成图表。", "noLocalExplainersYet": "暂无局部解释器", @@ -217,19 +173,49 @@ "noModelsMatchSearch": "没有模型匹配您的搜索", "noPredictionsYet": "暂无预测记录", "noRunsYet": "暂无运行记录。请从右侧面板添加模型。", + "noSavedTests": "暂无已保存的检验。", "noSessionSelected": "未选择会话", + "noTestsMatch": "没有检验匹配您的搜索", + "nonParametricTests": "非参数检验", + "normal": "正态", + "normalityByRunSummary": "{{total}} 个运行中有 {{normal}} 个似乎服从正态分布（α = {{alpha}}）。", + "notNormal": "非正态", + "notSignificant": "不显著", "operations": "操作", "operationsWillBeDeletedWarning": "这些操作将被永久删除且无法恢复。确定要继续吗？", + "optimizer": "优化器", "optimizerConfiguration": "配置超参数优化器设置", "optimizerParameters": "优化器参数", - "optimizer": "优化器", + "outerFoldFetch": "外层折", + "outerFolds": "折数", + "outerSplitter": "外层循环划分器", + "outerSplitterInherited": "会话初始划分器配置已继承，用于嵌套交叉验证的外层循环", "outputColumns": "输出列", + "parametricTests": "参数检验", "pleaseSelectMetricToOptimize": "请选择要优化的指标。", "predictions": "预测", "predictionsCount_one": "• <1>{{count}} 个预测", "predictionsCount_other": "• <1>{{count}} 个预测", - "divideColumnsAndSplits": "划分列并配置数据集划分", "prepareDataset": "准备数据集", + "preprocessingFailed": "预处理失败", + "preprocessingInProgress": "正在处理…", + "preprocessingOptional": "预处理（可选）", + "preprocessingOptionalDescription": "所选的转换器仅在训练数据上拟合，然后应用于其余数据，以避免数据泄漏。", + "profile_avoidFalseAlarms": "避免误报", + "profile_balanced": "均衡", + "profile_detectPositives": "检测阳性", + "profile_probabilityQuality": "概率质量", + "profile_regression_error": "均衡误差", + "profile_regression_fit": "模型拟合度", + "profile_text_avoidFalseAlarms": "避免误报", + "profile_text_balanced": "均衡", + "profile_text_detectPositives": "检测阳性", + "profile_text_probabilityQuality": "概率质量", + "profile_translation_balanced": "均衡翻译", + "profile_translation_quality": "翻译质量", + "repetition": "重复", + "result": "结果", + "resultSaved": "已保存", "retrainConfirmDetails": "确定要重新训练运行 \"<1>{{runName}}\" 吗？", "retrainModel": "重新训练模型？", "retrainWillDeleteOperations": "此运行有将被删除的现有操作", @@ -239,46 +225,69 @@ "runFailedNoHyperparameterPlots": "运行失败。无超参数图表可用。", "runInProgressCannotEdit": "运行正在进行中，无法编辑。", "runName": "运行名称", - "confirmParameterUpdate": "确认参数更新", "runNotFound": "未找到该运行", "runNotStartedNoHyperparameterPlots": "运行未开始。无超参数图表可用。", + "runsSelected": "个已选运行", + "sampleQuantiles": "样本分位数", "saveConfirmDetails": "保存“<1>{{runName}}”将把其状态重置为“未开始”，并清除其当前的指标和结果。确定要继续吗？", + "saveDetails": "保存详情", "saveParameterChanges": "保存参数更改？", + "saveResult": "保存结果", "saveWillDeleteOperationsDetails": "保存 \"<1>{{runName}}\" 将重置该运行。再次训练时以下内容将被删除：", + "savedTests": "已保存的检验", + "score": "评分", + "scoreHeaderTooltip": "根据所选配置文件计算的 0–100 加权评分。无上限的误差指标（MAE、RMSE、TER 等）会相对于本次比较中最差的模型进行归一化。将鼠标悬停在单元格上可查看明细。", + "scoreProfile": "评分配置文件", "searchDatasetsSessions": "搜索数据集和会话", "searchMetric": "搜索指标...", "searchModels": "搜索模型...", "searchTests": "搜索检验...", - "parametricTests": "参数检验", - "nonParametricTests": "非参数检验", - "minTwoRunsRequired": "请至少选择 2 个已完成的运行以进行统计检验", - "noTestsMatch": "没有检验匹配您的搜索", - "runsSelected": "个已选运行", - "configureTest": "配置检验", + "selectAtLeastRuns": "请至少选择 {{min}} 个模型", + "selectBetweenRuns": "请选择 {{min}} 到 {{max}} 个模型", + "selectColumnsDescription": "选择哪些列作为此会话的输入，哪一列作为输出。", + "selectColumnsTitle": "选择列", "selectDataset": "选择数据集", - "selectDatasetForSession": "为会话选择数据集", "selectDatasetAndPrepare": "为会话命名，选择数据集并配置列和划分。", "selectDatasetFirst": "选择数据集以配置其列和行划分。", + "selectDatasetForSession": "为会话选择数据集", + "selectExactlyRuns": "请恰好选择 {{count}} 个模型", + "selectExactlyRuns_other": "请恰好选择 {{count}} 个模型", "selectSessionToViewModels": "选择会话以查看可用模型。", + "selectSessionsToDelete": "选择要删除的会话", "selectTask": "选择任务", "selectTaskForSession": "为会话选择任务", "sessionConfiguration": "会话配置", "sessionName": "会话名称", "showResults": "显示结果", + "significanceLevel": "显著性水平", + "significant": "显著", + "significantsCount_other": "{{count}} / {{total}} 显著", + "statistic": "统计量", + "statisticalTests": "统计检验", "step": "步骤", + "technicalDetails": "技术细节", "test": "测试", + "testDescription": "描述（可选）", "testMetrics": "测试集指标", + "testName": "检验名称（可选）", + "testNameHelp": "如果留空，将使用检验的名称。", "testSet": "测试集", + "theoricalQuantiles": "理论分位数", "thereAreNoMetricsForThisRun": "此运行中没有与 {{set}} 关联的指标", "tourDisabledMessage": "返回首页以开始引导", "train": "训练", - "trainingMetrics": "训练集指标", "trainSet": "训练集", + "trainingMetrics": "训练集指标", "trial": "试验", + "tukeyPairwiseComparisons": "ANOVA 检验后的 Tukey 事后成对比较", + "twoSided": "双侧（模型 1 ≠ 模型 2）", "validation": "验证", "validationMetrics": "验证集指标", "validationSet": "验证集", - "viewResultsAs": "以列或图表方式查看结果" + "viewResultsAs": "以列或图表方式查看结果", + "wilcoxonPairwiseComparisons": "带 Holm 校正的成对 Wilcoxon 检验", + "withHpo": "含 HPO", + "withoutHpo": "不含 HPO" }, "message": { "allRunsCompleted": "{{experiment}} 已完成所有运行。", From bfb15d603c9253065426ca9be3afdfbbc428a6d6 Mon Sep 17 00:00:00 2001 From: Creylay Date: Fri, 11 Sep 2026 09:07:26 -0300 Subject: [PATCH 15/39] fix: update down_revision to correct migration reference --- .../f4a91c62d8e7_add_preprocessing_to_model_session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py b/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py index c875af368..bd88a8d23 100644 --- a/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py +++ b/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py @@ -1,7 +1,7 @@ """add preprocessing columns to model_session Revision ID: f4a91c62d8e7 -Revises: a5f2c71e9d40 +Revises: m6n7o8p9q0r1 Create Date: 2026-09-08 10:00:00.000000 Sessions created before this column existed have no preprocessing, so an @@ -14,7 +14,7 @@ from alembic import op revision: str = "f4a91c62d8e7" -down_revision: Union[str, None] = "a5f2c71e9d40" +down_revision: Union[str, None] = "m6n7o8p9q0r1" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From 8dd8e2c4bef5dcc3a01523a66e15587171ab8674 Mon Sep 17 00:00:00 2001 From: Creylay Date: Fri, 11 Sep 2026 12:59:30 -0300 Subject: [PATCH 16/39] feat: enhance converter output display with disambiguation for duplicate types --- .../modelSession/AppliedConvertersView.jsx | 17 +++-- .../AppliedConvertersView.test.jsx | 37 ++++++++-- .../models/modelSession/SelectColumnsStep.jsx | 23 ++++++ .../models/modelSession/sessionColumnRefs.js | 39 ++++++++++- .../modelSession/sessionColumnRefs.test.js | 70 +++++++++++++++++++ 5 files changed, 172 insertions(+), 14 deletions(-) diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx index 089fa436c..266abc4ce 100644 --- a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx @@ -24,6 +24,7 @@ import ItemsToDeleteList from "../../notebooks/converter/ItemsToDeleteList"; import { useExplorersAndConverters } from "../../notebooks/context/ExplorersAndConvertersContext"; import { buildColumnKeysAndTypes, + buildStepDisplayNames, groupKey, refToKey, } from "./sessionColumnRefs"; @@ -191,11 +192,14 @@ function SessionConverterCard({ preprocessing.forEach((s, i) => { const name = stepDisplayNames[i]; const slots = s.outputSlots?.length > 0 ? s.outputSlots : [{ slot: null }]; + // No "(slot)" suffix here: RefChip already shows the slot's type as its + // own colored badge right next to this label, so repeating it as text + // would be redundant (a step with several slots gets several chips + // with identical text but different badges, which is enough to tell + // them apart). slots.forEach(({ slot }) => { const key = groupKey(i, slot); - optionLabels[key] = slot - ? `${name}: output (${slot})` - : `${name}: output`; + optionLabels[key] = `${name}: output`; }); }); const ownSlots = @@ -374,9 +378,10 @@ export default function AppliedConvertersView({ }; }, []); - const stepDisplayNames = steps.map( - (step) => convertersMeta[step.converter]?.display_name || step.converter, - ); + // Numbers duplicate converter types ("Simple Imputer" / "Simple Imputer + // (2)") so two steps of the same type never render with identical names + // — see buildStepDisplayNames. + const stepDisplayNames = buildStepDisplayNames(steps, convertersMeta); const itemsToDelete = useMemo(() => { if (deleteIndex === null) return []; diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx index a1865f03c..6b8f360bf 100644 --- a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx @@ -133,12 +133,37 @@ describe("AppliedConvertersView", () => { }, }); - expect( - await screen.findByText("SimpleImputer: output (Integer)"), - ).toBeInTheDocument(); - expect( - screen.getByText("SimpleImputer: output (Categorical)"), - ).toBeInTheDocument(); + // No "(slot)" suffix: each slot already gets its own colored type chip + // (asserted below), so both chips share the same label text. + expect(await screen.findAllByText("SimpleImputer: output")).toHaveLength(2); + expect(screen.getByText("Integer")).toBeInTheDocument(); + expect(screen.getByText("Categorical")).toBeInTheDocument(); + }); + + it("numbers two converters of the same type so their cards and output labels don't collide", async () => { + const newExp = { + preprocessing: [ + { + converter: "Binarizer", + params: { threshold: 0.5 }, + scope: [{ kind: "raw", name: "age" }], + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + { + converter: "Binarizer", + params: { threshold: 1 }, + scope: [{ kind: "raw", name: "text" }], + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + ], + }; + + renderView({ newExp, setNewExp: () => {}, datasetTypes }); + + expect(await screen.findByText("Binarizador")).toBeInTheDocument(); + expect(screen.getByText("Binarizador (2)")).toBeInTheDocument(); + expect(screen.getByText("Binarizador: output")).toBeInTheDocument(); + expect(screen.getByText("Binarizador (2): output")).toBeInTheDocument(); }); it("cascades deletion to every converter configured after the deleted one", async () => { diff --git a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx index 1bf182567..ca94e7426 100644 --- a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx +++ b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx @@ -42,6 +42,7 @@ function SelectColumnsStep({ const { t } = useTranslation(["experiments", "models", "common"]); const [taskRequirements, setTaskRequirements] = useState(null); + const [convertersMeta, setConvertersMeta] = useState({}); const rawColumnNames = datasetInfo.column_names || []; @@ -52,6 +53,7 @@ function SelectColumnsStep({ } = buildColumnKeysAndTypes({ datasetTypes, preprocessing: newExp.preprocessing, + convertersMeta, }); const [inputSelection, setInputSelection] = useState(() => @@ -213,6 +215,27 @@ function SelectColumnsStep({ getTaskRequirements(); }, []); + useEffect(() => { + // Only needed to disambiguate two steps of the same converter type in + // the option labels below (e.g. "Simple Imputer" / "Simple Imputer + // (2)") — see buildStepDisplayNames. + let cancelled = false; + getComponentsRequest({ selectTypes: ["Converter"] }) + .then((data) => { + if (cancelled) return; + const byName = Object.fromEntries( + (data || []).map((component) => [component.name, component]), + ); + setConvertersMeta(byName); + }) + .catch((error) => + console.error("Failed to fetch converter metadata:", error), + ); + return () => { + cancelled = true; + }; + }, []); + const columnGroupsOf = (side) => { const metadata = taskRequirements?.metadata ?? {}; if (Array.isArray(metadata[side]) && metadata[side].length > 0) { diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js index 0c505ac1f..97c8f7268 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js @@ -76,6 +76,33 @@ function stepOutputSlots(step) { ]; } +/** + * One display name per step, disambiguated when the sequence has more than + * one step of the same converter type (same registry name, or same + * fallback string when `convertersMeta` hasn't loaded yet) — the first + * occurrence keeps the bare name, later ones get " (2)", " (3)", etc., by + * order of appearance. Computed over the FULL `steps` array regardless of + * any later truncation (e.g. buildColumnKeysAndTypes's `uptoStep`), so a + * step's numbering never shifts depending on which view is asking. + */ +export function buildStepDisplayNames(steps, convertersMeta = {}) { + const baseNames = (steps || []).map( + (step) => convertersMeta[step?.converter]?.display_name || step?.converter, + ); + + const totalByName = {}; + baseNames.forEach((name) => { + totalByName[name] = (totalByName[name] || 0) + 1; + }); + + const seenSoFar = {}; + return baseNames.map((name) => { + if (totalByName[name] <= 1) return name; + seenSoFar[name] = (seenSoFar[name] || 0) + 1; + return seenSoFar[name] === 1 ? name : `${name} (${seenSoFar[name]})`; + }); +} + /** * Every column key a session's preprocessing sequence can be scoped over, * up to (and not including) `uptoStep`: every raw dataset column, plus one @@ -85,14 +112,21 @@ function stepOutputSlots(step) { * (used once a sequence is final and being displayed, e.g. in * SelectColumnsStep, where every step is already "before" the * column-selection step that comes after all of them). + * + * `convertersMeta` (registry name -> component, e.g. from getComponents) + * is optional: when supplied, option labels use each step's disambiguated + * display name (see buildStepDisplayNames) instead of its raw registry + * name. */ export function buildColumnKeysAndTypes({ datasetTypes, preprocessing, uptoStep, + convertersMeta = {}, }) { const steps = preprocessing || []; const limit = uptoStep === undefined ? steps.length : uptoStep; + const displayNames = buildStepDisplayNames(steps, convertersMeta); const columnTypes = { ...datasetTypes }; const optionLabels = {}; @@ -100,12 +134,13 @@ export function buildColumnKeysAndTypes({ for (let index = 0; index < limit; index += 1) { const step = steps[index]; + const name = displayNames[index]; stepOutputSlots(step).forEach(({ slot, type, dtype }) => { const key = groupKey(index, slot); columnTypes[key] = { type: type || null, dtype: dtype || null }; optionLabels[key] = slot - ? `${step.converter}: output (${slot})` - : `${step.converter}: output`; + ? `${name}: output (${slot})` + : `${name}: output`; allKeys.push(key); }); } diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js index 234524de1..43aea6c70 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -6,6 +6,7 @@ import { refToKey, keyToRef, buildColumnKeysAndTypes, + buildStepDisplayNames, resolveDeclaredOutputSlots, rawColumnsNeededFor, } from "./sessionColumnRefs"; @@ -111,6 +112,28 @@ describe("sessionColumnRefs", () => { ); }); + it("disambiguates option labels for two steps of the same converter type", () => { + const { optionLabels } = buildColumnKeysAndTypes({ + datasetTypes: {}, + preprocessing: [ + { + converter: "SimpleImputer", + outputSlots: [{ slot: null, type: "Float" }], + }, + { + converter: "SimpleImputer", + outputSlots: [{ slot: null, type: "Float" }], + }, + ], + convertersMeta: { + SimpleImputer: { display_name: "Simple Imputer" }, + }, + }); + + expect(optionLabels[groupKey(0)]).toBe("Simple Imputer: output"); + expect(optionLabels[groupKey(1)]).toBe("Simple Imputer (2): output"); + }); + it("falls back to a single unslotted key for a step predating outputSlots", () => { const { allKeys, columnTypes } = buildColumnKeysAndTypes({ datasetTypes: {}, @@ -258,6 +281,53 @@ describe("sessionColumnRefs", () => { }); }); + describe("buildStepDisplayNames", () => { + it("leaves a converter type's name unchanged when it appears only once", () => { + const steps = [ + { converter: "SimpleImputer" }, + { converter: "Binarizer" }, + ]; + const convertersMeta = { + SimpleImputer: { display_name: "Simple Imputer" }, + Binarizer: { display_name: "Binarizer" }, + }; + expect(buildStepDisplayNames(steps, convertersMeta)).toEqual([ + "Simple Imputer", + "Binarizer", + ]); + }); + + it("numbers repeated converter types by order of appearance", () => { + const steps = [ + { converter: "SimpleImputer" }, + { converter: "Binarizer" }, + { converter: "SimpleImputer" }, + { converter: "SimpleImputer" }, + ]; + const convertersMeta = { + SimpleImputer: { display_name: "Simple Imputer" }, + Binarizer: { display_name: "Binarizer" }, + }; + expect(buildStepDisplayNames(steps, convertersMeta)).toEqual([ + "Simple Imputer", + "Binarizer", + "Simple Imputer (2)", + "Simple Imputer (3)", + ]); + }); + + it("falls back to the raw registry name when metadata hasn't loaded, still disambiguating", () => { + const steps = [ + { converter: "SimpleImputer" }, + { converter: "SimpleImputer" }, + ]; + expect(buildStepDisplayNames(steps, {})).toEqual([ + "SimpleImputer", + "SimpleImputer (2)", + ]); + }); + }); + describe("rawColumnsNeededFor", () => { it("returns raw ref names as-is", () => { const refs = [ From ad36abb0097f51d54a7d3d4d819bb7f16ef42334 Mon Sep 17 00:00:00 2001 From: Creylay Date: Fri, 11 Sep 2026 12:59:49 -0300 Subject: [PATCH 17/39] feat: add convertersMeta prop to enhance converter output handling across components --- .../FormSessionConverterSection.jsx | 3 ++ .../ScopeStepSessionConverter.jsx | 10 ++-- .../SessionConvertersRightBar.jsx | 12 ++++- .../components/notebooks/ColumnSelector.jsx | 51 ++++++++++++++++++- 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx index 1ff3be744..d6520bdc0 100644 --- a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx +++ b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx @@ -24,6 +24,7 @@ export default function FormSessionConverterSection({ datasetTypes, filePath, hideButtons = false, + convertersMeta, }) { const [scope, setScope] = useState([]); @@ -67,6 +68,7 @@ export default function FormSessionConverterSection({ filePath={filePath} scope={scope} setScope={setScope} + convertersMeta={convertersMeta} nextStep={ Object.values(tool.schema.properties).length > 0 ? () => setStep((s) => s + 1) @@ -100,4 +102,5 @@ FormSessionConverterSection.propTypes = { datasetTypes: PropTypes.object.isRequired, filePath: PropTypes.string, hideButtons: PropTypes.bool, + convertersMeta: PropTypes.object, }; diff --git a/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx index 7d1976e0a..e2363e960 100644 --- a/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx +++ b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx @@ -30,6 +30,7 @@ export default function ScopeStepSessionConverter({ scope, setScope, nextStep, + convertersMeta, }) { const theme = useTheme(); const { t } = useTranslation(["common", "datasets", "models"]); @@ -40,9 +41,10 @@ export default function ScopeStepSessionConverter({ const nonAllowedDtypes = tool?.metadata?.non_allowed_dtypes || []; const inputCardinality = tool?.metadata?.input_cardinality || {}; - const { columnTypes: columnTypesForSelector } = useMemo( - () => buildColumnKeysAndTypes({ datasetTypes, preprocessing }), - [datasetTypes, preprocessing], + const { columnTypes: columnTypesForSelector, optionLabels } = useMemo( + () => + buildColumnKeysAndTypes({ datasetTypes, preprocessing, convertersMeta }), + [datasetTypes, preprocessing, convertersMeta], ); const handleSelectionChange = (selected) => { @@ -77,6 +79,7 @@ export default function ScopeStepSessionConverter({ nonAllowedDtypes={nonAllowedDtypes} inputCardinality={inputCardinality} columnTypes={columnTypesForSelector} + optionLabels={optionLabels} onSelectionChange={handleSelectionChange} onValidationChange={setIsColumnSelectionValid} /> @@ -105,4 +108,5 @@ ScopeStepSessionConverter.propTypes = { scope: PropTypes.array.isRequired, setScope: PropTypes.func.isRequired, nextStep: PropTypes.func.isRequired, + convertersMeta: PropTypes.object, }; diff --git a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx index c5128d2ff..3a71c5c6c 100644 --- a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx +++ b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx @@ -120,6 +120,15 @@ export default function SessionConvertersRightBar({ return { disabled, tooltip, validColumns }; }; + // Type-level friendly names (e.g. "Simple Imputer" for "SimpleImputer"), + // reused when scoping a new converter over an earlier one's output group + // — see FormSessionConverterSection -> ScopeStepSessionConverter, which + // disambiguates repeated types the same way AppliedConvertersView does. + const convertersMeta = useMemo( + () => Object.fromEntries(converters.map((c) => [c.name, c])), + [converters], + ); + const validatedConverters = useMemo( () => converters.map((converter) => { @@ -151,11 +160,12 @@ export default function SessionConvertersRightBar({ setNewExp={setNewExp} datasetTypes={datasetTypes} filePath={dataset?.file_path} + convertersMeta={convertersMeta} /> ); } return Wrapped; - }, [newExp, setNewExp, datasetTypes, dataset]); + }, [newExp, setNewExp, datasetTypes, dataset, convertersMeta]); return ( {}, onValidationChange = () => {}, columnTypes = null, + optionLabels = {}, }) { const [rows, setRows] = useState([]); const [rowSelectionModel, setRowSelectionModel] = useState([]); @@ -72,6 +74,50 @@ function ColumnSelector({ accessorKey: "columnName", header: t("datasets:label.columnName"), flex: 1, + // The underlying value stays the real column name/synthetic group + // key (selection tracking and the caller's onSelectionChange both + // depend on it). Only a converter's output group — the only rows + // `optionLabels` ever has an entry for, never a raw column — gets + // the RefChip-style label + type badge, so it looks identical to + // that same group's chip in AppliedConvertersView; a plain raw + // column keeps rendering as plain text, unchanged. + Cell: ({ cell, row }) => { + const key = cell.getValue(); + const rawLabel = optionLabels[key]; + if (!rawLabel) return key; + // Drop a trailing "(slot)" — e.g. "SimpleImputer: output + // (Integer)" — since the type badge right next to it already + // shows that same value; text-only consumers of this same label + // (e.g. the column Autocomplete elsewhere) keep the full string, + // this strip only affects this chip's own rendering. + const label = rawLabel.replace(/ \([^)]*\)$/, ""); + const type = row.original.valueType; + return ( + + {label} + {type && type !== t("common:unknown") && ( + + getColorByColumnType(type, cellTheme), + color: "#fff", + fontWeight: 600, + fontSize: "0.65rem", + height: "18px", + ml: 1, + }} + /> + )} + + } + /> + ); + }, }, { accessorKey: "valueType", @@ -89,7 +135,7 @@ function ColumnSelector({ flex: 0.5, }, ], - [t], + [t, optionLabels], ); useEffect(() => { @@ -496,6 +542,7 @@ ColumnSelector.propTypes = { excludedColumnIds: PropTypes.array, onSelectionChange: PropTypes.func, onValidationChange: PropTypes.func, + optionLabels: PropTypes.object, }; export default ColumnSelector; From 8ed098ebd669e43aebd945dd586bcf893ec77fec Mon Sep 17 00:00:00 2001 From: Creylay Date: Fri, 11 Sep 2026 13:56:53 -0300 Subject: [PATCH 18/39] feat: enhance InfoModal layout with flexible sizing and overflow handling --- DashAI/front/src/components/shared/InfoModal.jsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DashAI/front/src/components/shared/InfoModal.jsx b/DashAI/front/src/components/shared/InfoModal.jsx index ff69d79c9..6af897dee 100644 --- a/DashAI/front/src/components/shared/InfoModal.jsx +++ b/DashAI/front/src/components/shared/InfoModal.jsx @@ -36,6 +36,9 @@ export default function InfoModal({ left: "50%", transform: "translate(-50%, -50%)", width: { xs: "90%", sm: 500 }, + maxHeight: "85vh", + display: "flex", + flexDirection: "column", bgcolor: "background.paper", borderRadius: 2, boxShadow: 12, @@ -50,6 +53,7 @@ export default function InfoModal({ justifyContent: "space-between", alignItems: "center", borderBottom: "1px solid rgba(255, 255, 255, 0.1)", + flexShrink: 0, }} > @@ -71,7 +75,7 @@ export default function InfoModal({ - + {extraContent} {rows.length > 0 && ( From a2f506a8f5de857ab261a3cf0e8553c4b1f17226 Mon Sep 17 00:00:00 2001 From: Creylay Date: Fri, 11 Sep 2026 13:57:16 -0300 Subject: [PATCH 19/39] feat: add appliedConverters label to multiple language files and enhance SessionInfoContent with converter metadata --- .../components/models/SessionInfoContent.jsx | 72 ++++++++++++++++++- .../src/utils/i18n/locales/de/models.json | 1 + .../src/utils/i18n/locales/en/models.json | 1 + .../src/utils/i18n/locales/es/models.json | 1 + .../src/utils/i18n/locales/pt/models.json | 1 + .../src/utils/i18n/locales/zh/models.json | 1 + 6 files changed, 75 insertions(+), 2 deletions(-) diff --git a/DashAI/front/src/components/models/SessionInfoContent.jsx b/DashAI/front/src/components/models/SessionInfoContent.jsx index 3d97764fe..8b79df8ae 100644 --- a/DashAI/front/src/components/models/SessionInfoContent.jsx +++ b/DashAI/front/src/components/models/SessionInfoContent.jsx @@ -1,9 +1,15 @@ -import React from "react"; +import React, { useEffect, useState } from "react"; import PropTypes from "prop-types"; import { Box, Typography, Chip } from "@mui/material"; import { useTranslation } from "react-i18next"; import { formatDate } from "../../utils"; -import ParamInfoList from "./ParamInfoBox"; +import { getComponents } from "../../api/component"; +import ParamInfoList, { ParamInfoBox } from "./ParamInfoBox"; +import { + buildStepDisplayNames, + buildColumnKeysAndTypes, + refToKey, +} from "./modelSession/sessionColumnRefs"; const SPLIT_TYPE_LABEL_KEYS = { random: "experiments:label.random", @@ -24,9 +30,41 @@ export default function SessionInfoContent({ tasks = [], }) { const { t } = useTranslation(["common", "experiments", "models"]); + const [convertersMeta, setConvertersMeta] = useState({}); + + useEffect(() => { + let cancelled = false; + getComponents({ selectTypes: ["Converter"] }) + .then((data) => { + if (cancelled) return; + setConvertersMeta( + Object.fromEntries((data || []).map((c) => [c.name, c])), + ); + }) + .catch((error) => + console.error("Failed to fetch converter metadata:", error), + ); + return () => { + cancelled = true; + }; + }, []); if (!session) return null; + const preprocessingSteps = session.preprocessing?.steps || []; + const stepDisplayNames = buildStepDisplayNames( + preprocessingSteps, + convertersMeta, + ); + // datasetTypes isn't needed here: a raw scope ref is shown as its own + // column name regardless, and a group scope ref's label only depends on + // the earlier step it points to (see buildColumnKeysAndTypes). + const { optionLabels: scopeOptionLabels } = buildColumnKeysAndTypes({ + datasetTypes: {}, + preprocessing: preprocessingSteps, + convertersMeta, + }); + const getDatasetName = () => { if (!session.dataset_id || !datasets.length) return t("common:unknown"); const dataset = datasets.find((d) => d.id === session.dataset_id); @@ -150,6 +188,28 @@ export default function SessionInfoContent({ + {preprocessingSteps.length > 0 && ( + + + {t("models:label.appliedConverters")} + + + {preprocessingSteps.map((step, index) => ( + { + const key = refToKey(ref); + return scopeOptionLabels[key] || key; + }) + .join(", ")} + /> + ))} + + + )} + {t("common:metadata")} @@ -172,6 +232,14 @@ SessionInfoContent.propTypes = { created: PropTypes.string, last_modified: PropTypes.string, description: PropTypes.string, + preprocessing: PropTypes.shape({ + steps: PropTypes.arrayOf( + PropTypes.shape({ + converter: PropTypes.string, + scope: PropTypes.array, + }), + ), + }), }), datasets: PropTypes.array, tasks: PropTypes.array, diff --git a/DashAI/front/src/utils/i18n/locales/de/models.json b/DashAI/front/src/utils/i18n/locales/de/models.json index 501e34c7e..71507b0cf 100644 --- a/DashAI/front/src/utils/i18n/locales/de/models.json +++ b/DashAI/front/src/utils/i18n/locales/de/models.json @@ -74,6 +74,7 @@ "addModelToSession": "Modell zur Sitzung hinzufügen", "allRepetitions": "Alle Wiederholungen", "alternativeHypothesis": "Alternativhypothese", + "appliedConverters": "Angewendete Converter", "applyPreprocessing": "Vorverarbeitung anwenden", "applyPreprocessingDescription": "Die ausgewählten Converter werden nur anhand der Trainingsdaten angepasst und auf den Rest angewendet, um Datenlecks zu vermeiden. Wird dies nicht angewendet, trainiert das Modell mit den Daten wie sie sind.", "availableExplainers": "Verfügbare Erklärungsmodelle", diff --git a/DashAI/front/src/utils/i18n/locales/en/models.json b/DashAI/front/src/utils/i18n/locales/en/models.json index 8bcbd08d3..962e0b225 100644 --- a/DashAI/front/src/utils/i18n/locales/en/models.json +++ b/DashAI/front/src/utils/i18n/locales/en/models.json @@ -232,6 +232,7 @@ "predictionsCount_other": "• <1>{{count}} predictions", "divideColumnsAndSplits": "Divide columns and configure dataset splits", "prepareDataset": "Prepare Dataset", + "appliedConverters": "Applied Converters", "applyPreprocessing": "Apply preprocessing", "applyPreprocessingDescription": "Selected converters are fit only on training data and applied to the rest, to avoid data leakage. If not applied, the model trains on the data as-is.", "preprocessingOptional": "Preprocessing (optional)", diff --git a/DashAI/front/src/utils/i18n/locales/es/models.json b/DashAI/front/src/utils/i18n/locales/es/models.json index fd98076b8..4705e406d 100644 --- a/DashAI/front/src/utils/i18n/locales/es/models.json +++ b/DashAI/front/src/utils/i18n/locales/es/models.json @@ -238,6 +238,7 @@ "predictionsCount_other": "• <1>{{count}} predicciones", "divideColumnsAndSplits": "Divide columnas y configura las particiones del dataset", "prepareDataset": "Preparar Dataset", + "appliedConverters": "Converters Aplicados", "applyPreprocessing": "Aplicar preprocesamiento", "applyPreprocessingDescription": "Los converters seleccionados se ajustan solo sobre los datos de entrenamiento y se aplican al resto, para evitar fuga de datos. Si no se aplica, el modelo se entrena con los datos tal como están.", "preprocessingOptional": "Preprocesamiento (opcional)", diff --git a/DashAI/front/src/utils/i18n/locales/pt/models.json b/DashAI/front/src/utils/i18n/locales/pt/models.json index 4d9d71dbb..40d807c4d 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/models.json +++ b/DashAI/front/src/utils/i18n/locales/pt/models.json @@ -74,6 +74,7 @@ "addModelToSession": "Adicionar Modelo à Sessão", "allRepetitions": "Todas as Repetições", "alternativeHypothesis": "Hipótese Alternativa", + "appliedConverters": "Conversores Aplicados", "applyPreprocessing": "Aplicar pré-processamento", "applyPreprocessingDescription": "Os conversores selecionados são ajustados apenas com os dados de treinamento e aplicados ao restante, para evitar vazamento de dados. Se não for aplicado, o modelo treina com os dados como estão.", "availableExplainers": "Explicadores Disponíveis", diff --git a/DashAI/front/src/utils/i18n/locales/zh/models.json b/DashAI/front/src/utils/i18n/locales/zh/models.json index 6ab14cc6d..b77edcbc9 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/models.json +++ b/DashAI/front/src/utils/i18n/locales/zh/models.json @@ -74,6 +74,7 @@ "addModelToSession": "将模型添加到会话", "allRepetitions": "所有重复", "alternativeHypothesis": "备择假设", + "appliedConverters": "已应用的转换器", "applyPreprocessing": "应用预处理", "applyPreprocessingDescription": "所选的转换器仅在训练数据上拟合，然后应用于其余数据，以避免数据泄漏。如果不应用，模型将按原样使用数据进行训练。", "availableExplainers": "可用解释器", From 2014a6bd905439194d27a5fe71d3114e37d320fe Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 22 Sep 2026 10:54:16 +0200 Subject: [PATCH 20/39] feat: add LEARNS_FROM_DATA flag to converters to indicate data leakage potential --- DashAI/back/converters/base_converter.py | 11 +++++++++++ DashAI/back/converters/hugging_face/embedding.py | 1 + .../back/converters/hugging_face/image_embedding.py | 1 + DashAI/back/converters/hugging_face/tokenizer.py | 1 + .../converters/scikit_learn/additive_chi_2_sampler.py | 1 + DashAI/back/converters/scikit_learn/binarizer.py | 1 + DashAI/back/converters/scikit_learn/normalizer.py | 1 + .../converters/scikit_learn/polynomial_features.py | 1 + .../converters/scikit_learn/skewed_chi_2_sampler.py | 1 + .../converters/segmentation/sam3_segment_converter.py | 1 + .../simple_converters/character_replacer.py | 1 + .../converters/simple_converters/column_arithmetic.py | 1 + .../converters/simple_converters/column_concat.py | 1 + .../converters/simple_converters/column_remover.py | 1 + .../converters/simple_converters/date_features.py | 1 + .../back/converters/simple_converters/nan_remover.py | 1 + .../converters/simple_converters/numeric_expansion.py | 1 + .../converters/simple_converters/time_resampler.py | 1 + .../simple_converters/time_series_window.py | 1 + DashAI/back/converters/simple_converters/type_cast.py | 1 + 20 files changed, 30 insertions(+) diff --git a/DashAI/back/converters/base_converter.py b/DashAI/back/converters/base_converter.py index 79306a3dc..3eda0d764 100644 --- a/DashAI/back/converters/base_converter.py +++ b/DashAI/back/converters/base_converter.py @@ -42,6 +42,16 @@ class BaseConverter(ConfigObject, ABC): COLOR: Final[str] = "rgb(255, 255, 255)" SUPERVISED: bool = False CHANGES_ROW_COUNT: bool = False + # Whether `fit` computes anything from the actual values of the input + # data (e.g. a mean, a vocabulary, learned components) rather than just + # validating shapes/types or applying a fixed, user-specified rule. + # Defaults to True (the conservative choice): a converter added without + # setting this explicitly warns about possible leakage rather than + # silently skipping the warning for a converter that does learn from + # data. Used to flag possible data leakage when applied inside a + # notebook, where fit/transform runs on the whole dataset with no + # train/test split. + LEARNS_FROM_DATA: bool = True # True for converters that never transform values, only keep or drop # whole columns as-is (feature selection, variance thresholding): the # output type of a surviving column is always exactly its input type, no @@ -84,6 +94,7 @@ def get_metadata(cls) -> Dict[str, Any]: meta["supervised"] = cls.SUPERVISED meta["changes_row_count"] = cls.CHANGES_ROW_COUNT meta["preserves_input_type"] = cls.PRESERVES_INPUT_TYPE + meta["learns_from_data"] = cls.LEARNS_FROM_DATA meta["n_components_features_bounded"] = getattr( cls, "N_COMPONENTS_FEATURES_BOUNDED", False ) diff --git a/DashAI/back/converters/hugging_face/embedding.py b/DashAI/back/converters/hugging_face/embedding.py index cbc726d35..f0666d7fc 100644 --- a/DashAI/back/converters/hugging_face/embedding.py +++ b/DashAI/back/converters/hugging_face/embedding.py @@ -104,6 +104,7 @@ class Embedding(AdvancedPreprocessingConverter, HuggingFaceWrapper): """HuggingFace embedding converter.""" SCHEMA = EmbeddingSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en="Convert text to embeddings using HuggingFace transformer models.", es="Convierte texto a embeddings usando modelos de HuggingFace.", diff --git a/DashAI/back/converters/hugging_face/image_embedding.py b/DashAI/back/converters/hugging_face/image_embedding.py index ea48647d5..c529b8646 100644 --- a/DashAI/back/converters/hugging_face/image_embedding.py +++ b/DashAI/back/converters/hugging_face/image_embedding.py @@ -119,6 +119,7 @@ class ImageEmbeddingConverter(AdvancedPreprocessingConverter, HuggingFaceWrapper """ SCHEMA = ImageEmbeddingSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en="Embed an image column into dense float columns using a " "HuggingFace vision model. Each detected image column is encoded " diff --git a/DashAI/back/converters/hugging_face/tokenizer.py b/DashAI/back/converters/hugging_face/tokenizer.py index 79e874e37..33a213a06 100644 --- a/DashAI/back/converters/hugging_face/tokenizer.py +++ b/DashAI/back/converters/hugging_face/tokenizer.py @@ -81,6 +81,7 @@ class TokenizerConverter(AdvancedPreprocessingConverter, HuggingFaceWrapper): """Converter that tokenizes text and stores each token ID in a separate column.""" SCHEMA = TokenizerSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Tokenize text into input IDs; each token ID goes into its own column. " diff --git a/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py b/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py index 02c028836..d131b9bc3 100644 --- a/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py +++ b/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py @@ -57,6 +57,7 @@ class AdditiveChi2Sampler( """ SCHEMA = AdditiveChi2SamplerSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Uses sampling the Fourier transform of the kernel characteristic " diff --git a/DashAI/back/converters/scikit_learn/binarizer.py b/DashAI/back/converters/scikit_learn/binarizer.py index bf6d16f79..4f738640c 100644 --- a/DashAI/back/converters/scikit_learn/binarizer.py +++ b/DashAI/back/converters/scikit_learn/binarizer.py @@ -46,6 +46,7 @@ class Binarizer(EncodingConverter, SklearnWrapper, BinarizerOperation): """ SCHEMA = BinarizerSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=("Binarize data (set feature values to 0 or 1) according to a threshold."), es=( diff --git a/DashAI/back/converters/scikit_learn/normalizer.py b/DashAI/back/converters/scikit_learn/normalizer.py index 88398b5b9..845ffab2b 100644 --- a/DashAI/back/converters/scikit_learn/normalizer.py +++ b/DashAI/back/converters/scikit_learn/normalizer.py @@ -64,6 +64,7 @@ class Normalizer(ScalingAndNormalizationConverter, SklearnWrapper, NormalizerOpe """ SCHEMA = NormalizerSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Normalize each row (sample) to unit norm across the selected columns. " diff --git a/DashAI/back/converters/scikit_learn/polynomial_features.py b/DashAI/back/converters/scikit_learn/polynomial_features.py index b668eb5ba..3788b6274 100644 --- a/DashAI/back/converters/scikit_learn/polynomial_features.py +++ b/DashAI/back/converters/scikit_learn/polynomial_features.py @@ -138,6 +138,7 @@ class PolynomialFeatures( """ SCHEMA = PolynomialFeaturesSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Generate polynomial and interaction features. For example, if an " diff --git a/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py b/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py index 5af2a3514..e97e1733e 100644 --- a/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py +++ b/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py @@ -127,6 +127,7 @@ class SkewedChi2Sampler( """ SCHEMA = SkewedChi2SamplerSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Approximates the feature map of a chi-squared kernel by Monte Carlo " diff --git a/DashAI/back/converters/segmentation/sam3_segment_converter.py b/DashAI/back/converters/segmentation/sam3_segment_converter.py index 7cea36ade..12639d1cf 100644 --- a/DashAI/back/converters/segmentation/sam3_segment_converter.py +++ b/DashAI/back/converters/segmentation/sam3_segment_converter.py @@ -333,6 +333,7 @@ class SAM3SegmentConverter(HFPretrainedDownloadMixin, AdvancedPreprocessingConve SCHEMA = SAM3SegmentConverterSchema CHANGES_ROW_COUNT = False + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en="Detects every instance of a prompt in each image and appends " "2 times max_masks new columns, keeping every row unchanged: " diff --git a/DashAI/back/converters/simple_converters/character_replacer.py b/DashAI/back/converters/simple_converters/character_replacer.py index fec5d0b0e..c02bd41eb 100644 --- a/DashAI/back/converters/simple_converters/character_replacer.py +++ b/DashAI/back/converters/simple_converters/character_replacer.py @@ -67,6 +67,7 @@ class CharacterReplacer(BasicPreprocessingConverter, BaseConverter): """ SCHEMA = CharacterReplacerSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Replaces or removes specified characters/substrings in selected " diff --git a/DashAI/back/converters/simple_converters/column_arithmetic.py b/DashAI/back/converters/simple_converters/column_arithmetic.py index 1396f9dc4..e0b065b4c 100644 --- a/DashAI/back/converters/simple_converters/column_arithmetic.py +++ b/DashAI/back/converters/simple_converters/column_arithmetic.py @@ -164,6 +164,7 @@ class ColumnArithmetic(FeatureEngineeringConverter, BaseConverter): """ SCHEMA = ColumnArithmeticSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Applies an arithmetic operation (add, subtract, multiply, divide) " diff --git a/DashAI/back/converters/simple_converters/column_concat.py b/DashAI/back/converters/simple_converters/column_concat.py index 9027a685b..d70659808 100644 --- a/DashAI/back/converters/simple_converters/column_concat.py +++ b/DashAI/back/converters/simple_converters/column_concat.py @@ -147,6 +147,7 @@ class ColumnConcat(FeatureEngineeringConverter, BaseConverter): """ SCHEMA = ColumnConcatSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Concatenates the selected string columns — two columns to join " diff --git a/DashAI/back/converters/simple_converters/column_remover.py b/DashAI/back/converters/simple_converters/column_remover.py index 21ad25b68..2f2558a3b 100644 --- a/DashAI/back/converters/simple_converters/column_remover.py +++ b/DashAI/back/converters/simple_converters/column_remover.py @@ -26,6 +26,7 @@ class ColumnRemover(BasicPreprocessingConverter, BaseConverter): """ SCHEMA = ColumnRemoverSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en="Removes the columns selected in scope from the dataset.", es="Elimina las columnas seleccionadas en el alcance del conjunto de datos.", diff --git a/DashAI/back/converters/simple_converters/date_features.py b/DashAI/back/converters/simple_converters/date_features.py index be1a54a92..8dd8e6298 100644 --- a/DashAI/back/converters/simple_converters/date_features.py +++ b/DashAI/back/converters/simple_converters/date_features.py @@ -225,6 +225,7 @@ class DateFeaturesConverter(FeatureEngineeringConverter, BaseConverter): """ SCHEMA = DateFeaturesSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Extracts calendar features from each date column: year, month, " diff --git a/DashAI/back/converters/simple_converters/nan_remover.py b/DashAI/back/converters/simple_converters/nan_remover.py index 6a11c82e9..8ca5e3256 100644 --- a/DashAI/back/converters/simple_converters/nan_remover.py +++ b/DashAI/back/converters/simple_converters/nan_remover.py @@ -42,6 +42,7 @@ class NanRemover(BasicPreprocessingConverter, BaseConverter): SCHEMA = NanRemoverSchema CHANGES_ROW_COUNT = True + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Removes the rows with NaN values from the dataset. Keep in mind that " diff --git a/DashAI/back/converters/simple_converters/numeric_expansion.py b/DashAI/back/converters/simple_converters/numeric_expansion.py index 4db709263..f5dd4643b 100644 --- a/DashAI/back/converters/simple_converters/numeric_expansion.py +++ b/DashAI/back/converters/simple_converters/numeric_expansion.py @@ -68,6 +68,7 @@ class NumericExpansion(FeatureEngineeringConverter, BaseConverter): """ SCHEMA = NumericExpansionSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Applies a unary numeric expansion (log1p, square, or sqrt) to " diff --git a/DashAI/back/converters/simple_converters/time_resampler.py b/DashAI/back/converters/simple_converters/time_resampler.py index 0f7a248b6..3f5cbf5ba 100644 --- a/DashAI/back/converters/simple_converters/time_resampler.py +++ b/DashAI/back/converters/simple_converters/time_resampler.py @@ -156,6 +156,7 @@ class TimeResamplerConverter(BasicPreprocessingConverter, BaseConverter): SCHEMA = TimeResamplerSchema CHANGES_ROW_COUNT = True + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Puts a dated table on a regular calendar grid: one row per day, " diff --git a/DashAI/back/converters/simple_converters/time_series_window.py b/DashAI/back/converters/simple_converters/time_series_window.py index 166001e85..a15568102 100644 --- a/DashAI/back/converters/simple_converters/time_series_window.py +++ b/DashAI/back/converters/simple_converters/time_series_window.py @@ -93,6 +93,7 @@ class TimeSeriesWindowConverter(FeatureEngineeringConverter, BaseConverter): SCHEMA = TimeSeriesWindowConverterSchema SUPERVISED = True CHANGES_ROW_COUNT = True + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Turns a time series into supervised rows: each row holds the " diff --git a/DashAI/back/converters/simple_converters/type_cast.py b/DashAI/back/converters/simple_converters/type_cast.py index 38c5877f0..2e60c0c90 100644 --- a/DashAI/back/converters/simple_converters/type_cast.py +++ b/DashAI/back/converters/simple_converters/type_cast.py @@ -86,6 +86,7 @@ class TypeCast(BasicPreprocessingConverter, BaseConverter): """ SCHEMA = TypeCastSchema + LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( "Changes the type of the selected columns (Integer, Float, Text, " From 9133346c124debe18702ec928be0a364efa6a0ac Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 22 Sep 2026 11:09:57 +0200 Subject: [PATCH 21/39] feat: add data leakage warning to ParameterStepConverter and localize messages in multiple languages --- .../notebooks/converterCreation/ParameterStepConverter.jsx | 7 +++++++ DashAI/front/src/utils/i18n/locales/de/datasets.json | 1 + DashAI/front/src/utils/i18n/locales/en/datasets.json | 1 + DashAI/front/src/utils/i18n/locales/es/datasets.json | 1 + DashAI/front/src/utils/i18n/locales/pt/datasets.json | 1 + DashAI/front/src/utils/i18n/locales/zh/datasets.json | 1 + 6 files changed, 12 insertions(+) diff --git a/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx b/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx index 745ecb4a1..5e37f2fbb 100644 --- a/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx +++ b/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx @@ -72,6 +72,8 @@ export default function ParameterStepConverter({ nColumnsSelected > 0 && currentNComponents > nColumnsSelected; + const showLeakageWarning = tool?.metadata?.learns_from_data === true; + return ( )} + {showLeakageWarning && ( + + {t("datasets:label.leakageWarning")} + + )} Date: Tue, 22 Sep 2026 11:16:03 +0200 Subject: [PATCH 22/39] feat: add tests for learns_from_data metadata in converters --- .../test_base_converter_metadata.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/back/converters/test_base_converter_metadata.py b/tests/back/converters/test_base_converter_metadata.py index 1191b67fd..c81e924b4 100644 --- a/tests/back/converters/test_base_converter_metadata.py +++ b/tests/back/converters/test_base_converter_metadata.py @@ -222,3 +222,35 @@ def test_feature_selection_and_variance_threshold_declare_preserves_input_type() assert FeatureSelectionConverter.get_metadata()["preserves_input_type"] is True assert VarianceThreshold.get_metadata()["preserves_input_type"] is True + + +def test_get_metadata_learns_from_data_defaults_to_true(): + meta = _FloatIntConverter.get_metadata() + assert meta["learns_from_data"] is True + + +def test_get_metadata_reports_learns_from_data_false_when_declared(): + class _StatelessConverter(BaseConverter): + SCHEMA = None + metadata = {"allowed_types": [Float, Integer]} + LEARNS_FROM_DATA = False + + def get_output_type(self, column_name=None): + return None + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + meta = _StatelessConverter.get_metadata() + assert meta["learns_from_data"] is False + + +def test_stateful_and_stateless_converters_declare_learns_from_data(): + from DashAI.back.converters.scikit_learn.standard_scaler import StandardScaler + from DashAI.back.converters.simple_converters.type_cast import TypeCast + + assert StandardScaler.get_metadata()["learns_from_data"] is True + assert TypeCast.get_metadata()["learns_from_data"] is False From 93734262e1c94b06a959c106ac7899bf8049dc3a Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 22 Sep 2026 13:33:50 +0200 Subject: [PATCH 23/39] fix: update dill package version to 0.4.1 for compatibility --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 29e353f0e..3678f9b8c 100644 --- a/uv.lock +++ b/uv.lock @@ -1526,11 +1526,11 @@ wheels = [ [[package]] name = "dill" -version = "0.3.7" +version = "0.4.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c4/31/54dd222e02311c2dbc9e680d37cbd50f4494ce1ee9b04c69980e4ec26f38/dill-0.3.7.tar.gz", hash = "sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03", size = 183355, upload-time = "2023-07-22T22:18:46.449Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/3a/74a29b11cf2cdfcd6ba89c0cecd70b37cd1ba7b77978ce611eb7a146a832/dill-0.3.7-py3-none-any.whl", hash = "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e", size = 115254, upload-time = "2023-07-22T22:18:44.511Z" }, + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] [[package]] From 465f133557cf4bdcffed43ebdf9d6766a6d87ae7 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 22 Sep 2026 13:59:49 +0200 Subject: [PATCH 24/39] fix: increase padding in ModelsRightBar for improved UI spacing --- DashAI/front/src/components/models/ModelsRightBar.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DashAI/front/src/components/models/ModelsRightBar.jsx b/DashAI/front/src/components/models/ModelsRightBar.jsx index f15246288..74bd0f5dc 100644 --- a/DashAI/front/src/components/models/ModelsRightBar.jsx +++ b/DashAI/front/src/components/models/ModelsRightBar.jsx @@ -307,7 +307,7 @@ export default function ModelsRightBar({ onToggle }) { Date: Tue, 22 Sep 2026 14:08:29 +0200 Subject: [PATCH 25/39] fix: update translation keys for input and output column labels in DivideDatasetColumns --- .../components/models/modelSession/DivideDatasetColumns.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx index 0b8c35d11..1508d1dc5 100644 --- a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx +++ b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx @@ -27,7 +27,7 @@ function DivideDatasetColumns({ outputHelperText = "", disabled = false, }) { - const { t } = useTranslation(["experiments", "common"]); + const { t } = useTranslation(["experiments", "common", "models"]); const theme = useTheme(); const inputOptions = inputOptionNames || allColumnNames; @@ -145,7 +145,7 @@ function DivideDatasetColumns({ Date: Tue, 22 Sep 2026 14:20:03 +0200 Subject: [PATCH 26/39] fix: replace useEffect with useLayoutEffect to prevent flashing of input fields on mount --- .../components/models/modelSession/SelectColumnsStep.jsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx index ca94e7426..af54e0bfb 100644 --- a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx +++ b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx @@ -73,7 +73,7 @@ function SelectColumnsStep({ const [columnsAreValid, setColumnsAreValid] = useState(false); const [validationPending, setValidationPending] = useState(true); - useEffect(() => { + useLayoutEffect(() => { // Auto-select sensible defaults the first time the dataset's columns load. if (rawColumnNames.length === 0) return; if ( @@ -308,7 +308,9 @@ function SelectColumnsStep({ bgcolor: (theme) => `${theme.palette[columnsAreValid ? "success" : "error"].main}40`, border: (theme) => - `1px solid ${theme.palette[columnsAreValid ? "success" : "error"].main}`, + `1px solid ${ + theme.palette[columnsAreValid ? "success" : "error"].main + }`, }} data-tour="models-validation-alert" > From 813af1fc9d7f20caf41fd0b23709d1a6c0211466 Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 09:43:58 +0200 Subject: [PATCH 27/39] fix: update applyPreprocessing label to include "(advanced)" in multiple languages --- DashAI/front/src/utils/i18n/locales/de/models.json | 2 +- DashAI/front/src/utils/i18n/locales/en/models.json | 2 +- DashAI/front/src/utils/i18n/locales/es/models.json | 2 +- DashAI/front/src/utils/i18n/locales/pt/models.json | 2 +- DashAI/front/src/utils/i18n/locales/zh/models.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DashAI/front/src/utils/i18n/locales/de/models.json b/DashAI/front/src/utils/i18n/locales/de/models.json index 71507b0cf..386f98882 100644 --- a/DashAI/front/src/utils/i18n/locales/de/models.json +++ b/DashAI/front/src/utils/i18n/locales/de/models.json @@ -75,7 +75,7 @@ "allRepetitions": "Alle Wiederholungen", "alternativeHypothesis": "Alternativhypothese", "appliedConverters": "Angewendete Converter", - "applyPreprocessing": "Vorverarbeitung anwenden", + "applyPreprocessing": "Vorverarbeitung anwenden (erweitert)", "applyPreprocessingDescription": "Die ausgewählten Converter werden nur anhand der Trainingsdaten angepasst und auf den Rest angewendet, um Datenlecks zu vermeiden. Wird dies nicht angewendet, trainiert das Modell mit den Daten wie sie sind.", "availableExplainers": "Verfügbare Erklärungsmodelle", "availableModels": "Verfügbare Modelle", diff --git a/DashAI/front/src/utils/i18n/locales/en/models.json b/DashAI/front/src/utils/i18n/locales/en/models.json index 962e0b225..a3264b4c6 100644 --- a/DashAI/front/src/utils/i18n/locales/en/models.json +++ b/DashAI/front/src/utils/i18n/locales/en/models.json @@ -233,7 +233,7 @@ "divideColumnsAndSplits": "Divide columns and configure dataset splits", "prepareDataset": "Prepare Dataset", "appliedConverters": "Applied Converters", - "applyPreprocessing": "Apply preprocessing", + "applyPreprocessing": "Apply preprocessing (advanced)", "applyPreprocessingDescription": "Selected converters are fit only on training data and applied to the rest, to avoid data leakage. If not applied, the model trains on the data as-is.", "preprocessingOptional": "Preprocessing (optional)", "preprocessingOptionalDescription": "Selected converters are fit only on training data and applied to the rest, to avoid data leakage.", diff --git a/DashAI/front/src/utils/i18n/locales/es/models.json b/DashAI/front/src/utils/i18n/locales/es/models.json index 4705e406d..3136aee1e 100644 --- a/DashAI/front/src/utils/i18n/locales/es/models.json +++ b/DashAI/front/src/utils/i18n/locales/es/models.json @@ -239,7 +239,7 @@ "divideColumnsAndSplits": "Divide columnas y configura las particiones del dataset", "prepareDataset": "Preparar Dataset", "appliedConverters": "Converters Aplicados", - "applyPreprocessing": "Aplicar preprocesamiento", + "applyPreprocessing": "Aplicar preprocesamiento (avanzado)", "applyPreprocessingDescription": "Los converters seleccionados se ajustan solo sobre los datos de entrenamiento y se aplican al resto, para evitar fuga de datos. Si no se aplica, el modelo se entrena con los datos tal como están.", "preprocessingOptional": "Preprocesamiento (opcional)", "preprocessingOptionalDescription": "Los converters seleccionados se ajustan solo sobre los datos de entrenamiento y se aplican al resto, para evitar fuga de datos.", diff --git a/DashAI/front/src/utils/i18n/locales/pt/models.json b/DashAI/front/src/utils/i18n/locales/pt/models.json index 40d807c4d..39c218dbb 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/models.json +++ b/DashAI/front/src/utils/i18n/locales/pt/models.json @@ -75,7 +75,7 @@ "allRepetitions": "Todas as Repetições", "alternativeHypothesis": "Hipótese Alternativa", "appliedConverters": "Conversores Aplicados", - "applyPreprocessing": "Aplicar pré-processamento", + "applyPreprocessing": "Aplicar pré-processamento (avançado)", "applyPreprocessingDescription": "Os conversores selecionados são ajustados apenas com os dados de treinamento e aplicados ao restante, para evitar vazamento de dados. Se não for aplicado, o modelo treina com os dados como estão.", "availableExplainers": "Explicadores Disponíveis", "availableModels": "Modelos Disponíveis", diff --git a/DashAI/front/src/utils/i18n/locales/zh/models.json b/DashAI/front/src/utils/i18n/locales/zh/models.json index b77edcbc9..249cb1375 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/models.json +++ b/DashAI/front/src/utils/i18n/locales/zh/models.json @@ -75,7 +75,7 @@ "allRepetitions": "所有重复", "alternativeHypothesis": "备择假设", "appliedConverters": "已应用的转换器", - "applyPreprocessing": "应用预处理", + "applyPreprocessing": "应用预处理（高级）", "applyPreprocessingDescription": "所选的转换器仅在训练数据上拟合，然后应用于其余数据，以避免数据泄漏。如果不应用，模型将按原样使用数据进行训练。", "availableExplainers": "可用解释器", "availableModels": "可用模型", From 991f8017cd37400c43391880bb58eb38f3bdfb4c Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 11:00:33 +0200 Subject: [PATCH 28/39] feat: implement session preprocessing structure types and infer_output_columns method --- DashAI/back/converters/base_converter.py | 118 +++++++++++++-- DashAI/back/preprocessing/structure_types.py | 124 ++++++++++++++++ .../test_infer_output_columns_defaults.py | 137 ++++++++++++++++++ 3 files changed, 365 insertions(+), 14 deletions(-) create mode 100644 DashAI/back/preprocessing/structure_types.py create mode 100644 tests/back/converters/test_infer_output_columns_defaults.py diff --git a/DashAI/back/converters/base_converter.py b/DashAI/back/converters/base_converter.py index 3eda0d764..02a43d809 100644 --- a/DashAI/back/converters/base_converter.py +++ b/DashAI/back/converters/base_converter.py @@ -1,10 +1,18 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Dict, Final, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Type, Union from DashAI.back.config_object import ConfigObject from DashAI.back.core.schema_fields.base_schema import BaseSchema +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + ColumnItem, + RowsNotSupportedError, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.static.icons import Icon from DashAI.back.types.dashai_data_type import DashAIDataType @@ -61,6 +69,17 @@ class BaseConverter(ConfigObject, ABC): # which — called on a bare unfitted instance — has no idea what column # it will actually run on. PRESERVES_INPUT_TYPE: bool = False + # What the converter does to the columns in its scope, used to estimate + # the dataset structure of a session's converter chain before any fit + # (see infer_output_columns): + # - "replace": same columns, values (and maybe types) changed + # - "add": keeps its scope columns and adds new ones + # - "expand": consumes its scope columns and outputs new ones + # - "select": keeps a subset of its scope columns, unchanged + # - "rows": changes the row count (not supported in sessions yet) + # None (e.g. an older plugin) is treated as "expand" with an unknown + # column count, which never promises a column that may not exist. + COLUMN_OPERATION: Optional[str] = None SCHEMA: BaseConverterSchema @classmethod @@ -95,6 +114,7 @@ def get_metadata(cls) -> Dict[str, Any]: meta["changes_row_count"] = cls.CHANGES_ROW_COUNT meta["preserves_input_type"] = cls.PRESERVES_INPUT_TYPE meta["learns_from_data"] = cls.LEARNS_FROM_DATA + meta["column_operation"] = cls.COLUMN_OPERATION meta["n_components_features_bounded"] = getattr( cls, "N_COMPONENTS_FEATURES_BOUNDED", False ) @@ -135,19 +155,11 @@ def get_metadata(cls) -> Dict[str, Any]: # require constructor params with no default), so this is # best-effort: None means "unknown until configured". try: - output_type = cls().get_output_type() - meta["output_type"] = ( - output_type.display_name() - if output_type is not None and hasattr(output_type, "display_name") - else None - ) - # The concrete storage dtype (e.g. "int64"), so a group column can - # show one instead of "unknown" before any real fit exists — same - # best-effort default-constructed instance as output_type above. - meta["output_dtype"] = ( - output_type.to_string().get("dtype") - if output_type is not None and hasattr(output_type, "to_string") - else None + # output_dtype is the concrete storage dtype (e.g. "int64"), so a + # group column can show one instead of "unknown" before any real + # fit exists. + meta["output_type"], meta["output_dtype"] = type_fields( + cls().get_output_type() ) except Exception: meta["output_type"] = None @@ -155,6 +167,84 @@ def get_metadata(cls) -> Dict[str, Any]: return meta + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate what this converter does to its scope, without any data. + + Called on an instance built with the user's params but never fit, so + a session wizard can show which columns exist after each step of a + converter chain. The default follows COLUMN_OPERATION; converters + that know more (e.g. an exact column count from `n_components`, or + output names built from their inputs) override it. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The scope items that survive (maybe retyped) and the new items. + + Raises + ------ + RowsNotSupportedError + If the converter changes the row count. + """ + operation = type(self).COLUMN_OPERATION or "expand" + if operation == "rows": + raise RowsNotSupportedError(type(self).__name__) + if operation == "replace": + return StructureDelta(kept=[self._retype(item) for item in inputs]) + if operation == "add": + return StructureDelta(kept=list(inputs), added=self._default_blocks()) + if operation == "select": + return StructureDelta(added=self._selection_blocks(inputs)) + return StructureDelta(added=self._default_blocks()) + + def _retype(self, item: StateItem) -> StateItem: + """Copy an item replaced in place with this converter's output type.""" + if type(self).PRESERVES_INPUT_TYPE: + return item + column_name = item.name if isinstance(item, ColumnItem) else None + type_name, dtype = type_fields(self.get_output_type(column_name)) + return item.model_copy(update={"type": type_name, "dtype": dtype}) + + def _default_blocks(self) -> List[StateItem]: + """One block of unknown size typed with this converter's output type.""" + type_name, dtype = type_fields(self.get_output_type()) + return [BlockItem(type=type_name, dtype=dtype)] + + def _selected_count(self, inputs: List[StateItem]) -> Optional[int]: + """How many scope columns a "select" converter keeps, if params say. + + None (the default) means it depends on the data, e.g. a statistical + test threshold. Overridden by selectors with a fixed count (`k`). + """ + return None + + def _selection_blocks(self, inputs: List[StateItem]) -> List[StateItem]: + """One block per distinct input type, since selection keeps types. + + A selector says at most how many columns survive, never which ones, + so its output is always symbolic, even with a known count. + """ + by_type: Dict[Optional[str], List[StateItem]] = {} + for item in inputs: + by_type.setdefault(item.type, []).append(item) + count = self._selected_count(inputs) if len(by_type) == 1 else None + blocks = [] + for type_name, items in by_type.items(): + dtypes = {item.dtype for item in items} + blocks.append( + BlockItem( + type=type_name, + dtype=dtypes.pop() if len(dtypes) == 1 else None, + count=count, + ) + ) + return blocks + @abstractmethod def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a given column. diff --git a/DashAI/back/preprocessing/structure_types.py b/DashAI/back/preprocessing/structure_types.py new file mode 100644 index 000000000..5bdd3b591 --- /dev/null +++ b/DashAI/back/preprocessing/structure_types.py @@ -0,0 +1,124 @@ +"""Types describing the estimated structure of a dataset during preprocessing. + +The Models-module wizard needs to know which columns (and of which type) +exist at every point of a session's converter chain before anything is +fit. Each converter describes what it does to its scope through +``BaseConverter.infer_output_columns``, which returns a StructureDelta over +these state items; ``infer_structure`` (structure.py) applies the deltas step +by step. + +A state item is either a concrete column, whose name is known ahead of any +fit, or a symbolic block: columns whose names (and sometimes count) only +exist once the converter is fit, e.g. one-hot columns, one per category. +""" + +from typing import Any, Dict, List, Literal, Optional, Tuple, Union + +from pydantic import BaseModel, Field +from typing_extensions import Annotated + + +class RowsNotSupportedError(Exception): + """Raised by a converter that changes the dataset's row count. + + Session preprocessing does not support these converters yet: resampling + must run on the training partition only and carry the target along, + which the session preprocessor does not do. + """ + + +class ColumnItem(BaseModel): + kind: Literal["column"] = "column" + name: str + # A DashAI type's display_name(), e.g. "Integer". None means unknown. + type: Optional[str] = None + dtype: Optional[str] = None + # None for an original dataset column (even after being replaced in + # place); otherwise the index of the step that created it. + origin: Optional[int] = None + + +class BlockItem(BaseModel): + kind: Literal["block"] = "block" + # Index of the step that produced the block, filled by infer_structure. + step: int = -1 + # The slot a GroupColumnRef uses to reference this block. It is fixed + # when the block is created, while `type` follows later in-place + # conversions of its columns, mirroring the runtime, where a step's + # slots are classified once, at that step. + slot: Optional[str] = None + label: str = "output" + type: Optional[str] = None + dtype: Optional[str] = None + # Number of columns in the block, or None when it is only known after + # fit (e.g. one-hot encoding: one column per category). + count: Optional[int] = None + + +StateItem = Annotated[Union[ColumnItem, BlockItem], Field(discriminator="kind")] + + +class StructureMessage(BaseModel): + # Suffix of the frontend i18n key the message is translated with. + code: str + params: Dict[str, Any] = Field(default_factory=dict) + + +class StructureDelta(BaseModel): + """What a converter does to the items in its scope. + + Scope items missing from `kept` are removed from the dataset state. A + kept item keeps its identity (a column's name, a block's step and slot) + but may carry a new type. `added` are the items the converter creates: + either all concrete columns or all blocks, never a mix. + """ + + kept: List[StateItem] = Field(default_factory=list) + added: List[StateItem] = Field(default_factory=list) + warnings: List[StructureMessage] = Field(default_factory=list) + + +class StepStructure(BaseModel): + status: Literal["ok", "error", "blocked"] + # Dataset state after this step. + state: List[StateItem] = Field(default_factory=list) + # Items this step produced. + added: List[StateItem] = Field(default_factory=list) + error: Optional[StructureMessage] = None + warnings: List[StructureMessage] = Field(default_factory=list) + + +class StructureResult(BaseModel): + initial: List[StateItem] + steps: List[StepStructure] + final: List[StateItem] + valid: bool + + +def type_fields(dashai_type: Any) -> Tuple[Optional[str], Optional[str]]: + """Return the (display name, dtype) pair the frontend shows for a type. + + Parameters + ---------- + dashai_type : DashAIDataType or None + The type to describe. + + Returns + ------- + tuple of (str or None, str or None) + The type's display_name() and its storage dtype (e.g. "int64"), + each None when unknown. + """ + if dashai_type is None: + return None, None + name = ( + dashai_type.display_name() + if hasattr(dashai_type, "display_name") + else type(dashai_type).__name__ + ) + dtype = None + if hasattr(dashai_type, "to_string"): + description = dashai_type.to_string() + if isinstance(description, dict): + dtype = description.get("dtype") + return name, dtype diff --git a/tests/back/converters/test_infer_output_columns_defaults.py b/tests/back/converters/test_infer_output_columns_defaults.py new file mode 100644 index 000000000..28812f7fe --- /dev/null +++ b/tests/back/converters/test_infer_output_columns_defaults.py @@ -0,0 +1,137 @@ +import pyarrow as pa +import pytest + +from DashAI.back.converters.base_converter import BaseConverter +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + ColumnItem, + RowsNotSupportedError, + type_fields, +) +from DashAI.back.types.value_types import Float, Integer + + +class _FakeConverter(BaseConverter): + SCHEMA = None + metadata = {} + + def get_output_type(self, column_name=None): + return Float(arrow_type=pa.float64()) + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + +class _Replace(_FakeConverter): + COLUMN_OPERATION = "replace" + + +class _ReplacePreserving(_FakeConverter): + COLUMN_OPERATION = "replace" + PRESERVES_INPUT_TYPE = True + + +class _Add(_FakeConverter): + COLUMN_OPERATION = "add" + + +class _Expand(_FakeConverter): + COLUMN_OPERATION = "expand" + + +class _Select(_FakeConverter): + COLUMN_OPERATION = "select" + + +class _Rows(_FakeConverter): + COLUMN_OPERATION = "rows" + + +class _Undeclared(_FakeConverter): + pass + + +def _inputs(): + return [ + ColumnItem(name="age", type="Integer", dtype="int64"), + ColumnItem(name="height", type="Float", dtype="float64"), + ] + + +def test_replace_keeps_inputs_retyped_with_output_type(): + delta = _Replace().infer_output_columns(_inputs()) + + assert [item.name for item in delta.kept] == ["age", "height"] + assert {item.type for item in delta.kept} == {"Float"} + assert delta.added == [] + + +def test_replace_preserving_input_type_keeps_inputs_unchanged(): + inputs = _inputs() + delta = _ReplacePreserving().infer_output_columns(inputs) + + assert delta.kept == inputs + assert delta.added == [] + + +def test_replace_retypes_a_block_without_changing_its_identity(): + block = BlockItem(step=0, slot="Categorical", label="ohe_*", type="Categorical") + delta = _Replace().infer_output_columns([block]) + + (kept,) = delta.kept + assert (kept.step, kept.slot) == (0, "Categorical") + assert kept.type == "Float" + + +def test_add_keeps_inputs_and_adds_one_unknown_block(): + inputs = _inputs() + delta = _Add().infer_output_columns(inputs) + + assert delta.kept == inputs + (block,) = delta.added + assert isinstance(block, BlockItem) + assert block.type == "Float" + assert block.count is None + + +def test_expand_removes_inputs_and_adds_one_unknown_block(): + delta = _Expand().infer_output_columns(_inputs()) + + assert delta.kept == [] + (block,) = delta.added + assert block.type == "Float" + assert block.count is None + + +def test_select_adds_one_block_per_input_type_preserving_types(): + delta = _Select().infer_output_columns(_inputs()) + + assert delta.kept == [] + assert sorted(block.type for block in delta.added) == ["Float", "Integer"] + assert all(block.count is None for block in delta.added) + + +def test_rows_is_not_supported(): + with pytest.raises(RowsNotSupportedError): + _Rows().infer_output_columns(_inputs()) + + +def test_undeclared_operation_behaves_like_expand(): + delta = _Undeclared().infer_output_columns(_inputs()) + + assert delta.kept == [] + (block,) = delta.added + assert block.count is None + + +def test_metadata_exposes_column_operation(): + assert _Replace.get_metadata()["column_operation"] == "replace" + assert _Undeclared.get_metadata()["column_operation"] is None + + +def test_type_fields_reads_display_name_and_dtype(): + assert type_fields(Integer(arrow_type=pa.int32())) == ("Integer", "int32") + assert type_fields(None) == (None, None) From 512d81a4c80e6a40bca22562882f5cd76f1c15e1 Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 11:17:25 +0200 Subject: [PATCH 29/39] feat: add COLUMN_OPERATION attribute to various converters for enhanced processing capabilities --- .../category/advanced_preprocessing.py | 1 + .../category/basic_preprocessing.py | 1 + .../category/dimensionality_reduction.py | 1 + DashAI/back/converters/category/encoding.py | 1 + .../category/feature_engineering.py | 1 + .../converters/category/feature_selection.py | 1 + .../converters/category/polynomial_kernel.py | 1 + DashAI/back/converters/category/sampling.py | 1 + .../category/scaling_and_normalization.py | 1 + .../back/converters/hugging_face/embedding.py | 1 + .../back/converters/hugging_face/tokenizer.py | 1 + .../scikit_learn/missing_indicator.py | 1 + .../scikit_learn/variance_threshold.py | 1 + .../simple_converters/column_remover.py | 1 + .../simple_converters/nan_remover.py | 1 + .../simple_converters/time_resampler.py | 1 + .../simple_converters/time_series_window.py | 1 + .../test_structure_matches_runtime.py | 266 ++++++++++++++++++ 18 files changed, 283 insertions(+) create mode 100644 tests/back/preprocessing/test_structure_matches_runtime.py diff --git a/DashAI/back/converters/category/advanced_preprocessing.py b/DashAI/back/converters/category/advanced_preprocessing.py index cba5f6ba7..b541df481 100644 --- a/DashAI/back/converters/category/advanced_preprocessing.py +++ b/DashAI/back/converters/category/advanced_preprocessing.py @@ -25,3 +25,4 @@ class AdvancedPreprocessingConverter(BaseConverter): ) ICON: Final[str] = Icon.Psychology.value COLOR: Final[str] = "rgb(70, 130, 180)" + COLUMN_OPERATION = "add" diff --git a/DashAI/back/converters/category/basic_preprocessing.py b/DashAI/back/converters/category/basic_preprocessing.py index 3a0dc8d56..dcb6ac98c 100644 --- a/DashAI/back/converters/category/basic_preprocessing.py +++ b/DashAI/back/converters/category/basic_preprocessing.py @@ -25,3 +25,4 @@ class BasicPreprocessingConverter(BaseConverter): ) ICON: Final[str] = Icon.Build.value COLOR: Final[str] = "rgb(60, 179, 113)" + COLUMN_OPERATION = "replace" diff --git a/DashAI/back/converters/category/dimensionality_reduction.py b/DashAI/back/converters/category/dimensionality_reduction.py index 378296343..6dd14bc51 100644 --- a/DashAI/back/converters/category/dimensionality_reduction.py +++ b/DashAI/back/converters/category/dimensionality_reduction.py @@ -26,4 +26,5 @@ class DimensionalityReductionConverter(BaseConverter): ) ICON: Final[str] = Icon.Layers.value COLOR: Final[str] = "rgb(255, 99, 132)" + COLUMN_OPERATION = "expand" N_COMPONENTS_FEATURES_BOUNDED: bool = True diff --git a/DashAI/back/converters/category/encoding.py b/DashAI/back/converters/category/encoding.py index dc812159e..986c9933d 100644 --- a/DashAI/back/converters/category/encoding.py +++ b/DashAI/back/converters/category/encoding.py @@ -28,6 +28,7 @@ class EncodingConverter(BaseConverter): ) ICON: Final[str] = Icon.Dns.value COLOR: Final[str] = "rgb(138, 43, 226)" + COLUMN_OPERATION = "add" PREFIX: str = "encoded_" def transform( diff --git a/DashAI/back/converters/category/feature_engineering.py b/DashAI/back/converters/category/feature_engineering.py index 5dca3da00..8f03cd9ae 100644 --- a/DashAI/back/converters/category/feature_engineering.py +++ b/DashAI/back/converters/category/feature_engineering.py @@ -28,3 +28,4 @@ class FeatureEngineeringConverter(BaseConverter): ) ICON: Final[str] = Icon.Functions.value COLOR: Final[str] = "rgb(0, 188, 212)" + COLUMN_OPERATION = "add" diff --git a/DashAI/back/converters/category/feature_selection.py b/DashAI/back/converters/category/feature_selection.py index a8c9919e2..e3cd5a52e 100644 --- a/DashAI/back/converters/category/feature_selection.py +++ b/DashAI/back/converters/category/feature_selection.py @@ -34,6 +34,7 @@ class FeatureSelectionConverter(BaseConverter): ) ICON: Final[str] = Icon.FilterList.value COLOR: Final[str] = "rgb(255, 206, 86)" + COLUMN_OPERATION = "select" PRESERVES_INPUT_TYPE = True def fit( diff --git a/DashAI/back/converters/category/polynomial_kernel.py b/DashAI/back/converters/category/polynomial_kernel.py index 96dd1cf7d..827c0f65a 100644 --- a/DashAI/back/converters/category/polynomial_kernel.py +++ b/DashAI/back/converters/category/polynomial_kernel.py @@ -26,3 +26,4 @@ class PolynomialKernelConverter(BaseConverter): ) ICON: Final[str] = Icon.Functions.value COLOR: Final[str] = "rgb(153, 102, 255)" + COLUMN_OPERATION = "expand" diff --git a/DashAI/back/converters/category/sampling.py b/DashAI/back/converters/category/sampling.py index 77cd19062..93f8de27b 100644 --- a/DashAI/back/converters/category/sampling.py +++ b/DashAI/back/converters/category/sampling.py @@ -25,3 +25,4 @@ class SamplingConverter(BaseConverter): ) ICON: Final[str] = Icon.Casino.value COLOR: Final[str] = "rgb(255, 159, 64)" + COLUMN_OPERATION = "rows" diff --git a/DashAI/back/converters/category/scaling_and_normalization.py b/DashAI/back/converters/category/scaling_and_normalization.py index a2878740c..776eb7819 100644 --- a/DashAI/back/converters/category/scaling_and_normalization.py +++ b/DashAI/back/converters/category/scaling_and_normalization.py @@ -25,3 +25,4 @@ class ScalingAndNormalizationConverter(BaseConverter): ) ICON: Final[str] = Icon.TrendingUp.value COLOR: Final[str] = "rgb(255, 165, 0)" + COLUMN_OPERATION = "replace" diff --git a/DashAI/back/converters/hugging_face/embedding.py b/DashAI/back/converters/hugging_face/embedding.py index f0666d7fc..58eaa9197 100644 --- a/DashAI/back/converters/hugging_face/embedding.py +++ b/DashAI/back/converters/hugging_face/embedding.py @@ -105,6 +105,7 @@ class Embedding(AdvancedPreprocessingConverter, HuggingFaceWrapper): SCHEMA = EmbeddingSchema LEARNS_FROM_DATA = False + COLUMN_OPERATION = "expand" DESCRIPTION = MultilingualString( en="Convert text to embeddings using HuggingFace transformer models.", es="Convierte texto a embeddings usando modelos de HuggingFace.", diff --git a/DashAI/back/converters/hugging_face/tokenizer.py b/DashAI/back/converters/hugging_face/tokenizer.py index 33a213a06..189f0ea4b 100644 --- a/DashAI/back/converters/hugging_face/tokenizer.py +++ b/DashAI/back/converters/hugging_face/tokenizer.py @@ -82,6 +82,7 @@ class TokenizerConverter(AdvancedPreprocessingConverter, HuggingFaceWrapper): SCHEMA = TokenizerSchema LEARNS_FROM_DATA = False + COLUMN_OPERATION = "expand" DESCRIPTION = MultilingualString( en=( "Tokenize text into input IDs; each token ID goes into its own column. " diff --git a/DashAI/back/converters/scikit_learn/missing_indicator.py b/DashAI/back/converters/scikit_learn/missing_indicator.py index 8b1297201..d80b975e0 100644 --- a/DashAI/back/converters/scikit_learn/missing_indicator.py +++ b/DashAI/back/converters/scikit_learn/missing_indicator.py @@ -52,6 +52,7 @@ class MissingIndicator( """ SCHEMA = MissingIndicatorSchema + COLUMN_OPERATION = "add" DESCRIPTION = MultilingualString( en="Binary indicators for missing values.", es="Indicadores binarios para valores faltantes.", diff --git a/DashAI/back/converters/scikit_learn/variance_threshold.py b/DashAI/back/converters/scikit_learn/variance_threshold.py index 335893904..916f140b0 100644 --- a/DashAI/back/converters/scikit_learn/variance_threshold.py +++ b/DashAI/back/converters/scikit_learn/variance_threshold.py @@ -93,6 +93,7 @@ class VarianceThreshold( zh="方差阈值", ) PRESERVES_INPUT_TYPE = True + COLUMN_OPERATION = "select" def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/converters/simple_converters/column_remover.py b/DashAI/back/converters/simple_converters/column_remover.py index 2f2558a3b..33bac88a8 100644 --- a/DashAI/back/converters/simple_converters/column_remover.py +++ b/DashAI/back/converters/simple_converters/column_remover.py @@ -27,6 +27,7 @@ class ColumnRemover(BasicPreprocessingConverter, BaseConverter): SCHEMA = ColumnRemoverSchema LEARNS_FROM_DATA = False + COLUMN_OPERATION = "select" DESCRIPTION = MultilingualString( en="Removes the columns selected in scope from the dataset.", es="Elimina las columnas seleccionadas en el alcance del conjunto de datos.", diff --git a/DashAI/back/converters/simple_converters/nan_remover.py b/DashAI/back/converters/simple_converters/nan_remover.py index 8ca5e3256..a9cb5506a 100644 --- a/DashAI/back/converters/simple_converters/nan_remover.py +++ b/DashAI/back/converters/simple_converters/nan_remover.py @@ -42,6 +42,7 @@ class NanRemover(BasicPreprocessingConverter, BaseConverter): SCHEMA = NanRemoverSchema CHANGES_ROW_COUNT = True + COLUMN_OPERATION = "rows" LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( diff --git a/DashAI/back/converters/simple_converters/time_resampler.py b/DashAI/back/converters/simple_converters/time_resampler.py index 3f5cbf5ba..4a5c3d325 100644 --- a/DashAI/back/converters/simple_converters/time_resampler.py +++ b/DashAI/back/converters/simple_converters/time_resampler.py @@ -156,6 +156,7 @@ class TimeResamplerConverter(BasicPreprocessingConverter, BaseConverter): SCHEMA = TimeResamplerSchema CHANGES_ROW_COUNT = True + COLUMN_OPERATION = "rows" LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( diff --git a/DashAI/back/converters/simple_converters/time_series_window.py b/DashAI/back/converters/simple_converters/time_series_window.py index a15568102..e03b79b8e 100644 --- a/DashAI/back/converters/simple_converters/time_series_window.py +++ b/DashAI/back/converters/simple_converters/time_series_window.py @@ -93,6 +93,7 @@ class TimeSeriesWindowConverter(FeatureEngineeringConverter, BaseConverter): SCHEMA = TimeSeriesWindowConverterSchema SUPERVISED = True CHANGES_ROW_COUNT = True + COLUMN_OPERATION = "rows" LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( diff --git a/tests/back/preprocessing/test_structure_matches_runtime.py b/tests/back/preprocessing/test_structure_matches_runtime.py new file mode 100644 index 000000000..32c72aeb8 --- /dev/null +++ b/tests/back/preprocessing/test_structure_matches_runtime.py @@ -0,0 +1,266 @@ +"""The structure a converter estimates must match what it really produces. + +For every registered converter, `infer_output_columns` (called on an unfitted +instance, with no data) is compared against a real fit + transform on a small +dataset, spliced back the way SessionPreprocessor does it. This is what +guarantees that a converter chain the session wizard accepts never fails at +training time because it references a column that does not exist. +""" + +import pandas as pd +import pytest + +from DashAI.back.converters.base_converter import BaseConverter +from DashAI.back.converters.dataset_columns import ( + rebuild_dataset_with_transformed_columns, +) +from DashAI.back.dataloaders.classes.dashai_dataset import ( + to_dashai_dataset, + transform_dataset_with_schema, +) +from DashAI.back.initial_components import get_initial_components +from DashAI.back.preprocessing.session_preprocessor import SessionPreprocessor +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + ColumnItem, + RowsNotSupportedError, + type_fields, +) +from DashAI.back.splitters.splits_payload import schema_placeholder_defaults + +N_ROWS = 30 + +SCHEMA = { + "i_full": {"type": "Integer", "dtype": "int64"}, + "i2": {"type": "Integer", "dtype": "int64"}, + "i_null": {"type": "Integer", "dtype": "int64"}, + "f1": {"type": "Float", "dtype": "float64"}, + "f2": {"type": "Float", "dtype": "float64"}, + "f_null": {"type": "Float", "dtype": "float64"}, + "cat": {"type": "Categorical", "dtype": "string"}, + "text": {"type": "Text", "dtype": "string"}, + "code": {"type": "Text", "dtype": "string"}, + "date": {"type": "Date", "dtype": "%Y-%m-%d"}, + "target": {"type": "Categorical", "dtype": "string"}, +} + +# Converters that cannot run here: they download pretrained models. Their +# overrides get direct unit tests instead. +NEEDS_DOWNLOAD = { + "Embedding", + "TokenizerConverter", + "ImageEmbeddingConverter", + "SAM3SegmentConverter", +} + +# (converter name, params on top of the schema placeholders, scope columns) +CASES = [ + ("StandardScaler", {}, ["i_full", "f1"]), + ("MinMaxScaler", {}, ["i_full", "f1"]), + ("MaxAbsScaler", {}, ["i_full", "f1"]), + ("Normalizer", {}, ["i_full", "f1"]), + ("SimpleImputer", {"strategy": "mean"}, ["i_null", "f_null"]), + ("SimpleImputer", {"strategy": "most_frequent"}, ["cat", "i_null"]), + ( + "SimpleImputer", + {"strategy": "median", "add_indicator": True}, + ["i_null", "f_null"], + ), + ("KNNImputer", {}, ["i_null", "f_null"]), + ("MissingIndicator", {}, ["i_null", "f_null"]), + ("Binarizer", {}, ["i_full", "f1"]), + ("OrdinalEncoder", {}, ["cat"]), + ("OneHotEncoder", {}, ["cat"]), + ("LabelEncoder", {}, ["cat"]), + ("PCA", {"n_components": 2}, ["i_full", "f1", "f2"]), + ("IncrementalPCA", {"n_components": 2}, ["i_full", "f1", "f2"]), + ("TruncatedSVD", {"n_components": 2}, ["i_full", "f1", "f2"]), + ("FastICA", {"n_components": 2}, ["i_full", "f1", "f2"]), + ("Nystroem", {"n_components": 5}, ["i_full", "f1"]), + ("RBFSampler", {"n_components": 5}, ["i_full", "f1"]), + ("SkewedChi2Sampler", {"n_components": 5}, ["i_full", "i2"]), + ("AdditiveChi2Sampler", {}, ["i_full", "i2"]), + ("PolynomialFeatures", {"degree": 2}, ["i_full", "f1"]), + ("VarianceThreshold", {}, ["i_full", "f1"]), + ("SelectKBest", {"k": 1}, ["i_full", "f1"]), + ("SelectPercentile", {}, ["i_full", "f1"]), + ("SelectFdr", {}, ["i_full", "f1"]), + ("SelectFpr", {}, ["i_full", "f1"]), + ("SelectFwe", {}, ["i_full", "f1"]), + ("GenericUnivariateSelect", {}, ["i_full", "f1"]), + ("BagOfWordsConverter", {}, ["text"]), + ("TFIDFConverter", {}, ["text"]), + ( + "CharacterReplacer", + {"char_to_replace": "-", "replacement_char": ""}, + ["code"], + ), + ("ColumnArithmetic", {"operation": "add"}, ["i_full", "i2"]), + ("ColumnArithmetic", {"operation": "multiply"}, ["i_null", "i2"]), + ("ColumnArithmetic", {"operation": "divide"}, ["i_full", "f1"]), + ("ColumnConcat", {}, ["cat", "text"]), + ("NumericExpansion", {"operation": "square"}, ["i_null", "i_full"]), + ("NumericExpansion", {"operation": "log1p"}, ["f1"]), + ("DateFeaturesConverter", {}, ["date"]), + ("TypeCast", {"new_type": "Float"}, ["i_full"]), + ("ColumnRemover", {}, ["f1"]), +] + + +def _converter_classes(): + return { + cls.__name__: cls + for cls in get_initial_components() + if isinstance(cls, type) + and issubclass(cls, BaseConverter) + and cls.__module__.startswith("DashAI.back.converters") + } + + +def _is_rows(cls): + return cls.COLUMN_OPERATION == "rows" + + +def _dataset(): + words = ["red apple", "green pear", "blue plum", "red plum", "green apple"] + frame = pd.DataFrame( + { + "i_full": [(i * 7) % 11 + 1 for i in range(N_ROWS)], + "i2": [(i * 3) % 5 + 1 for i in range(N_ROWS)], + "i_null": pd.array( + [None if i % 6 == 0 else i % 9 for i in range(N_ROWS)], + dtype="Int64", + ), + "f1": [0.5 + (i * 13) % 17 / 3 for i in range(N_ROWS)], + "f2": [1.25 + (i * 5) % 7 / 2 for i in range(N_ROWS)], + "f_null": [ + None if i % 5 == 0 else 0.1 + (i % 4) * 1.3 for i in range(N_ROWS) + ], + "cat": [["a", "b", "c"][i % 3] for i in range(N_ROWS)], + "text": [words[i % len(words)] for i in range(N_ROWS)], + "code": [f"{i % 4}-{i % 7}" for i in range(N_ROWS)], + "date": [f"2024-{i % 12 + 1:02d}-{i % 28 + 1:02d}" for i in range(N_ROWS)], + "target": [["yes", "no"][(i * 7) % 11 % 2] for i in range(N_ROWS)], + } + ) + return transform_dataset_with_schema(to_dashai_dataset(frame), SCHEMA) + + +def _build(cls, params): + return cls(**{**schema_placeholder_defaults(cls), **params}) + + +def _scope_items(dataset, scope): + items = [] + for name in scope: + type_name, dtype = type_fields(dataset.types[name]) + items.append(ColumnItem(name=name, type=type_name, dtype=dtype)) + return items + + +def _case_id(case): + name, params, _ = case + extra = ",".join(f"{k}={v}" for k, v in params.items()) + return f"{name}[{extra}]" if extra else name + + +@pytest.mark.parametrize("case", CASES, ids=[_case_id(c) for c in CASES]) +def test_estimated_structure_matches_runtime(case): + name, params, scope = case + cls = _converter_classes()[name] + dataset = _dataset() + + delta = _build(cls, params).infer_output_columns(_scope_items(dataset, scope)) + + converter = _build(cls, params) + x = dataset.select_columns(scope) + if cls.SUPERVISED: + converter.fit(x, dataset.select_columns(["target"])) + else: + converter.fit(x) + transformed = converter.transform(x) + rebuilt = rebuild_dataset_with_transformed_columns( + dataset, + transformed, + scope, + [dataset.column_names.index(column) for column in scope], + ) + + def real_type(column): + return type_fields(rebuilt.types[column])[0] + + real_out = list(transformed.column_names) + real_new = [c for c in real_out if c not in scope] + concrete = [item for item in delta.added if isinstance(item, ColumnItem)] + blocks = [item for item in delta.added if isinstance(item, BlockItem)] + assert not (concrete and blocks), "a delta never mixes columns and blocks" + + # Every scope column the estimate says survives really survives, with + # the estimated type. This is the property that keeps a later step from + # referencing a column that no longer exists. + kept_names = [item.name for item in delta.kept if isinstance(item, ColumnItem)] + for item in delta.kept: + assert item.name in real_out, f"{item.name} was estimated to survive" + assert real_type(item.name) == item.type, f"type of kept {item.name}" + + # A scope column the estimate drops may still survive only when a block + # stands for it (a selector keeps some scope columns, unknown which). + real_survivors = [c for c in scope if c in real_out and c not in kept_names] + if real_survivors: + assert blocks, f"{real_survivors} survive but nothing accounts for them" + + # New concrete columns: exact names and types. + if concrete: + assert [item.name for item in concrete] == real_new, "new column names" + for item in concrete: + assert real_type(item.name) == item.type, f"type of new {item.name}" + return + if not blocks: + assert real_new == [], f"unexpected new columns {real_new}" + return + + # Symbolic blocks stand for the new columns plus any unnamed survivors. + # With an unknown count a block may turn out empty (a selector keeping + # nothing of that type), so real types only need to be a subset; with a + # known count, the block has exactly that many columns. + block_columns = real_new + real_survivors + real_by_type = {} + for column in block_columns: + real_by_type.setdefault(real_type(column), []).append(column) + estimated_types = {block.type for block in blocks} + assert set(real_by_type) <= estimated_types, "block types" + for block in blocks: + if block.count is not None: + assert block.count == len(real_by_type.get(block.type, [])), "count" + + # The runtime classifies a step's group (its new columns, or all its + # output when it adds none) into slots by type; they must line up with + # the blocks. + group = real_new or real_out + runtime_slots = SessionPreprocessor._classify_by_type(converter, group) + assert set(runtime_slots) <= estimated_types, "runtime slots" + + +def test_every_converter_is_covered(): + covered = {name for name, _, _ in CASES} | NEEDS_DOWNLOAD + uncovered = { + name + for name, cls in _converter_classes().items() + if not _is_rows(cls) and name not in covered + } + assert uncovered == set(), "add a CASES entry for every new converter" + + +def test_rows_operation_matches_changes_row_count(): + for name, cls in _converter_classes().items(): + assert _is_rows(cls) == bool(cls.CHANGES_ROW_COUNT), name + + +def test_rows_converters_are_not_supported_in_sessions(): + for cls in _converter_classes().values(): + if not _is_rows(cls): + continue + with pytest.raises(RowsNotSupportedError): + BaseConverter.infer_output_columns( + cls.__new__(cls), _scope_items(_dataset(), ["i_full"]) + ) From 71daad1a48c7f83a5c88e2ea3aee77138529cc73 Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 11:30:20 +0200 Subject: [PATCH 30/39] feat: enhance output column inference across various converters with new structure mixins and utility functions --- DashAI/back/converters/base_converter.py | 13 ++- DashAI/back/converters/category/encoding.py | 47 ++++++++++- .../scikit_learn/additive_chi_2_sampler.py | 27 +++++++ .../back/converters/scikit_learn/binarizer.py | 1 + .../back/converters/scikit_learn/fast_ica.py | 8 +- .../scikit_learn/incremental_pca.py | 6 +- .../converters/scikit_learn/knn_imputer.py | 28 +++++++ .../converters/scikit_learn/label_encoder.py | 10 +++ .../scikit_learn/missing_indicator.py | 35 +++++++- .../back/converters/scikit_learn/nystroem.py | 8 +- .../scikit_learn/ordinal_encoder.py | 1 + DashAI/back/converters/scikit_learn/pca.py | 8 +- .../scikit_learn/polynomial_features.py | 42 ++++++++++ .../converters/scikit_learn/rbf_sampler.py | 8 +- .../converters/scikit_learn/select_k_best.py | 11 +++ .../converters/scikit_learn/simple_imputer.py | 80 ++++++++++++------- .../scikit_learn/skewed_chi_2_sampler.py | 6 +- .../converters/scikit_learn/truncated_svd.py | 6 +- DashAI/back/converters/structure_mixins.py | 28 +++++++ DashAI/back/preprocessing/structure_types.py | 17 ++++ .../test_structure_matches_runtime.py | 33 ++++++++ 21 files changed, 384 insertions(+), 39 deletions(-) create mode 100644 DashAI/back/converters/structure_mixins.py diff --git a/DashAI/back/converters/base_converter.py b/DashAI/back/converters/base_converter.py index 02a43d809..ba794cd01 100644 --- a/DashAI/back/converters/base_converter.py +++ b/DashAI/back/converters/base_converter.py @@ -210,10 +210,17 @@ def _retype(self, item: StateItem) -> StateItem: type_name, dtype = type_fields(self.get_output_type(column_name)) return item.model_copy(update={"type": type_name, "dtype": dtype}) - def _default_blocks(self) -> List[StateItem]: - """One block of unknown size typed with this converter's output type.""" + def _default_blocks(self, count: Optional[int] = None) -> List[StateItem]: + """One block typed with this converter's output type. + + Parameters + ---------- + count : int, optional + The block's column count, when params determine it. Defaults to + None (known only after fit). + """ type_name, dtype = type_fields(self.get_output_type()) - return [BlockItem(type=type_name, dtype=dtype)] + return [BlockItem(type=type_name, dtype=dtype, count=count)] def _selected_count(self, inputs: List[StateItem]) -> Optional[int]: """How many scope columns a "select" converter keeps, if params say. diff --git a/DashAI/back/converters/category/encoding.py b/DashAI/back/converters/category/encoding.py index 986c9933d..c61bb83b3 100644 --- a/DashAI/back/converters/category/encoding.py +++ b/DashAI/back/converters/category/encoding.py @@ -1,7 +1,13 @@ -from typing import TYPE_CHECKING, Final, Union +from typing import TYPE_CHECKING, Final, List, Union from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.static.icons import Icon if TYPE_CHECKING: @@ -30,6 +36,45 @@ class EncodingConverter(BaseConverter): COLOR: Final[str] = "rgb(138, 43, 226)" COLUMN_OPERATION = "add" PREFIX: str = "encoded_" + # Whether the converter appends exactly one PREFIX column per encoded + # scope column (ordinal, label, binarizer). One-hot appends one column + # per category instead, a count only known after fit. + ONE_COLUMN_PER_INPUT: bool = False + + def _encodes(self, item: ColumnItem) -> bool: + """Whether a scope column gets an encoded copy. Defaults to all.""" + return True + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus one encoded copy each. + + Only exact for ONE_COLUMN_PER_INPUT converters over concrete + columns; otherwise the new columns are one block of unknown size. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the encoded columns. + """ + concrete = all(isinstance(item, ColumnItem) for item in inputs) + if not (type(self).ONE_COLUMN_PER_INPUT and concrete): + return super().infer_output_columns(inputs) + added = [] + for item in inputs: + if not self._encodes(item): + continue + type_name, dtype = type_fields(self.get_output_type(item.name)) + added.append( + ColumnItem( + name=f"{self.PREFIX}{item.name}", type=type_name, dtype=dtype + ) + ) + return StructureDelta(kept=list(inputs), added=added) def transform( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py b/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py index d131b9bc3..c87a6a0b9 100644 --- a/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py +++ b/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py @@ -1,3 +1,5 @@ +from typing import List + from sklearn.kernel_approximation import ( AdditiveChi2Sampler as AdditiveChi2SamplerOperation, ) @@ -12,6 +14,11 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + StateItem, + StructureDelta, + known_width, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -88,6 +95,26 @@ class AdditiveChi2Sampler( metadata = {"allowed_types": [Float, Integer], "allowed_dtypes": []} + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: 2 * sample_steps - 1 Float columns per input. + + Each input column is expanded into one square-root term plus a cosine + and a sine term per extra sample step. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The inputs are consumed; one block replaces them. + """ + width = known_width(inputs) + count = None if width is None else width * (2 * self.sample_steps - 1) + return StructureDelta(added=self._default_blocks(count=count)) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a column. diff --git a/DashAI/back/converters/scikit_learn/binarizer.py b/DashAI/back/converters/scikit_learn/binarizer.py index 4f738640c..f4e70f426 100644 --- a/DashAI/back/converters/scikit_learn/binarizer.py +++ b/DashAI/back/converters/scikit_learn/binarizer.py @@ -72,6 +72,7 @@ class Binarizer(EncodingConverter, SklearnWrapper, BinarizerOperation): IMAGE_PREVIEW = "binarizer.png" PREFIX = "bin_" + ONE_COLUMN_PER_INPUT = True metadata = { "allowed_types": [Float, Integer], diff --git a/DashAI/back/converters/scikit_learn/fast_ica.py b/DashAI/back/converters/scikit_learn/fast_ica.py index 0e0ab35dd..71ba72dcc 100644 --- a/DashAI/back/converters/scikit_learn/fast_ica.py +++ b/DashAI/back/converters/scikit_learn/fast_ica.py @@ -9,6 +9,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( bool_field, enum_field, @@ -173,7 +174,12 @@ class FastICASchema(BaseSchema): ) # type: ignore -class FastICA(DimensionalityReductionConverter, SklearnWrapper, FastICAOperation): +class FastICA( + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + FastICAOperation, +): """Decompose features into statistically independent components using FastICA. Independent Component Analysis (ICA) models the observed data X as a linear diff --git a/DashAI/back/converters/scikit_learn/incremental_pca.py b/DashAI/back/converters/scikit_learn/incremental_pca.py index 9761051f8..650256fe1 100644 --- a/DashAI/back/converters/scikit_learn/incremental_pca.py +++ b/DashAI/back/converters/scikit_learn/incremental_pca.py @@ -4,6 +4,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( bool_field, int_field, @@ -71,7 +72,10 @@ class IncrementalPCASchema(BaseSchema): class IncrementalPCA( - DimensionalityReductionConverter, SklearnWrapper, IncrementalPCAOperation + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + IncrementalPCAOperation, ): """Reduce dimensionality using PCA computed incrementally over mini-batches. diff --git a/DashAI/back/converters/scikit_learn/knn_imputer.py b/DashAI/back/converters/scikit_learn/knn_imputer.py index 16ca91abb..4697f5666 100644 --- a/DashAI/back/converters/scikit_learn/knn_imputer.py +++ b/DashAI/back/converters/scikit_learn/knn_imputer.py @@ -1,3 +1,5 @@ +from typing import List + from sklearn.impute import KNNImputer as KNNImputerOperation from DashAI.back.converters.category.basic_preprocessing import ( @@ -12,6 +14,10 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + StateItem, + StructureDelta, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -150,6 +156,28 @@ def __init__(self, **kwargs): """ super().__init__(**kwargs) + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope imputed in place as Float. + + With ``add_indicator``, one Float indicator column is added per + column that had missing values during fit, a count only known after + fit. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The imputed inputs, plus an indicator block if requested. + """ + delta = super().infer_output_columns(inputs) + if self.add_indicator: + delta.added = self._default_blocks() + return delta + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a column. diff --git a/DashAI/back/converters/scikit_learn/label_encoder.py b/DashAI/back/converters/scikit_learn/label_encoder.py index 8d743cf31..5912ce8a9 100644 --- a/DashAI/back/converters/scikit_learn/label_encoder.py +++ b/DashAI/back/converters/scikit_learn/label_encoder.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + from DashAI.back.preprocessing.structure_types import ColumnItem class LabelEncoderSchema(BaseSchema): @@ -66,6 +67,7 @@ class LabelEncoder(EncodingConverter, SklearnWrapper): IMAGE_PREVIEW = "label_encoder.png" PREFIX = "le_" + ONE_COLUMN_PER_INPUT = True metadata = { "changes_data_types": True, @@ -90,6 +92,14 @@ def __init__(self, **kwargs): self.encoders = {} self.fitted_columns = [] + def _encodes(self, item: "ColumnItem") -> bool: + """Whether ``fit`` will encode a scope column. + + ``fit`` only encodes columns stored as strings, so a Categorical + column of numeric labels (e.g. dtype int64) gets no encoded copy. + """ + return item.type == "Text" or item.dtype == "string" + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a column. diff --git a/DashAI/back/converters/scikit_learn/missing_indicator.py b/DashAI/back/converters/scikit_learn/missing_indicator.py index d80b975e0..133dc71c0 100644 --- a/DashAI/back/converters/scikit_learn/missing_indicator.py +++ b/DashAI/back/converters/scikit_learn/missing_indicator.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, List, Union from sklearn.impute import MissingIndicator as MissingIndicatorOperation @@ -8,6 +8,12 @@ from DashAI.back.converters.sklearn_wrapper import SklearnWrapper from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Integer @@ -74,6 +80,33 @@ class MissingIndicator( "allowed_dtypes": [], } + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus one indicator per column. + + ``features="all"`` is forced in ``__init__``, so every scope column + gets its ``missingindicator_`` column, missing values or not. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the indicator columns. + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + type_name, dtype = type_fields(self.get_output_type()) + added = [ + ColumnItem( + name=f"missingindicator_{item.name}", type=type_name, dtype=dtype + ) + for item in inputs + ] + return StructureDelta(kept=list(inputs), added=added) + def __init__(self, **kwargs): """Initialize the MissingIndicator converter. diff --git a/DashAI/back/converters/scikit_learn/nystroem.py b/DashAI/back/converters/scikit_learn/nystroem.py index 7aef4b009..c3600af6b 100644 --- a/DashAI/back/converters/scikit_learn/nystroem.py +++ b/DashAI/back/converters/scikit_learn/nystroem.py @@ -5,6 +5,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( enum_field, float_field, @@ -132,7 +133,12 @@ class NystroemSchema(BaseSchema): ) # type: ignore -class Nystroem(DimensionalityReductionConverter, SklearnWrapper, NystroemOperation): +class Nystroem( + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + NystroemOperation, +): """Approximate a kernel feature map using the Nystroem method. The Nystroem method constructs an explicit low-dimensional feature map diff --git a/DashAI/back/converters/scikit_learn/ordinal_encoder.py b/DashAI/back/converters/scikit_learn/ordinal_encoder.py index 04f936f89..410a9da73 100644 --- a/DashAI/back/converters/scikit_learn/ordinal_encoder.py +++ b/DashAI/back/converters/scikit_learn/ordinal_encoder.py @@ -161,6 +161,7 @@ class OrdinalEncoder(EncodingConverter, SklearnWrapper, OrdinalEncoderOperation) IMAGE_PREVIEW = "ordinal_encoder.png" PREFIX = "oe_" + ONE_COLUMN_PER_INPUT = True metadata = { "allowed_types": [Categorical], diff --git a/DashAI/back/converters/scikit_learn/pca.py b/DashAI/back/converters/scikit_learn/pca.py index a55042624..193933825 100644 --- a/DashAI/back/converters/scikit_learn/pca.py +++ b/DashAI/back/converters/scikit_learn/pca.py @@ -5,6 +5,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( bool_field, enum_field, @@ -200,7 +201,12 @@ class PCASchema(BaseSchema): ) # type: ignore -class PCA(DimensionalityReductionConverter, SklearnWrapper, PCAOPERATION): +class PCA( + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + PCAOPERATION, +): """Reduce dimensionality using Principal Component Analysis (PCA). PCA finds a set of orthogonal axes (principal components) that successively diff --git a/DashAI/back/converters/scikit_learn/polynomial_features.py b/DashAI/back/converters/scikit_learn/polynomial_features.py index 3788b6274..4be63d11c 100644 --- a/DashAI/back/converters/scikit_learn/polynomial_features.py +++ b/DashAI/back/converters/scikit_learn/polynomial_features.py @@ -1,3 +1,6 @@ +import math +from typing import List + from sklearn.preprocessing import PolynomialFeatures as PolynomialFeaturesOperation from DashAI.back.converters.category.polynomial_kernel import PolynomialKernelConverter @@ -10,6 +13,11 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + StateItem, + StructureDelta, + known_width, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -176,6 +184,40 @@ class PolynomialFeatures( metadata = {"allowed_types": [Float, Integer], "allowed_dtypes": []} + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: inputs kept as Float, plus the other terms. + + The degree-1 terms keep their input column's name, so the inputs + survive (as Float); the bias column and higher-degree terms are new. + With n inputs there are C(n + degree, degree) monomials of degree at + most `degree`, or sum(C(n, k) for k <= degree) products of distinct + features when `interaction_only`, one less without the bias. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The retyped inputs, plus one block for the new terms. + """ + width = known_width(inputs) + count = None + if width is not None and isinstance(self.degree, int): + if self.interaction_only: + total = sum(math.comb(width, k) for k in range(self.degree + 1)) + else: + total = math.comb(width + self.degree, self.degree) + if not self.include_bias: + total -= 1 + count = total - width + return StructureDelta( + kept=[self._retype(item) for item in inputs], + added=self._default_blocks(count=count), + ) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return ``Float64`` as the output type for all polynomial feature columns. diff --git a/DashAI/back/converters/scikit_learn/rbf_sampler.py b/DashAI/back/converters/scikit_learn/rbf_sampler.py index 509306e06..d0f328a9e 100644 --- a/DashAI/back/converters/scikit_learn/rbf_sampler.py +++ b/DashAI/back/converters/scikit_learn/rbf_sampler.py @@ -3,6 +3,7 @@ from DashAI.back.api.utils import create_random_state from DashAI.back.converters.category.polynomial_kernel import PolynomialKernelConverter from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( enum_field, float_field, @@ -80,7 +81,12 @@ class RBFSamplerSchema(BaseSchema): ) # type: ignore -class RBFSampler(PolynomialKernelConverter, SklearnWrapper, RBFSamplerOperation): +class RBFSampler( + ComponentsOutputMixin, + PolynomialKernelConverter, + SklearnWrapper, + RBFSamplerOperation, +): """Approximate the RBF (Gaussian) kernel feature map via random Fourier features. The Radial Basis Function (RBF) kernel is one of the most widely used diff --git a/DashAI/back/converters/scikit_learn/select_k_best.py b/DashAI/back/converters/scikit_learn/select_k_best.py index 1bffc79f7..14601934c 100644 --- a/DashAI/back/converters/scikit_learn/select_k_best.py +++ b/DashAI/back/converters/scikit_learn/select_k_best.py @@ -1,3 +1,5 @@ +from typing import List, Optional + from sklearn.feature_selection import SelectKBest as SelectKBestOperation from DashAI.back.converters.category.feature_selection import FeatureSelectionConverter @@ -10,6 +12,7 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import StateItem, known_width from DashAI.back.types.value_types import Float, Integer @@ -92,3 +95,11 @@ def __init__(self, **kwargs): schema fields. Forwarded to the underlying scikit-learn class. """ super().__init__(**kwargs) + + def _selected_count(self, inputs: List[StateItem]) -> Optional[int]: + """Keep exactly ``k`` columns, or every column for ``k="all"``.""" + if self.k == "all": + return known_width(inputs) + if isinstance(self.k, int) and not isinstance(self.k, bool): + return self.k + return None diff --git a/DashAI/back/converters/scikit_learn/simple_imputer.py b/DashAI/back/converters/scikit_learn/simple_imputer.py index 8f427089d..c7d46de34 100644 --- a/DashAI/back/converters/scikit_learn/simple_imputer.py +++ b/DashAI/back/converters/scikit_learn/simple_imputer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, List, Union from sklearn.impute import SimpleImputer as SimpleImputerOperation @@ -24,6 +24,12 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.categorical import Categorical from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -181,12 +187,11 @@ class SimpleImputer( ``MissingIndicator`` binary matrix is stacked onto the output. Output typing preserves the original column type whenever the strategy - does not force a fractional result: ``"most_frequent"`` and - ``"constant"`` never perform arithmetic, so the source type (Integer, - Float, or Categorical) is kept. For ``"mean"``/``"median"`` the computed - per-column statistic is inspected, and an originally-Integer column - stays ``Integer`` if that statistic happens to be a whole number (e.g. a - median over an odd count of integers); otherwise it becomes ``Float64``. + does not perform arithmetic: ``"most_frequent"`` and ``"constant"`` only + copy existing values, so the source type (Integer, Float, or + Categorical) is kept. ``"mean"``/``"median"`` always produce ``Float64``, + even when the statistic happens to be a whole number, so the output type + follows from the params alone and never from the data. Wraps ``sklearn.impute.SimpleImputer``. @@ -254,10 +259,10 @@ def __init__(self, **kwargs): def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None ) -> "SimpleImputer": - """Fit the imputer, remembering input types and column order. + """Fit the imputer, remembering the input column types. - These are needed by ``get_output_type`` to preserve the original - column type instead of always coercing to ``Float64``. + ``get_output_type`` needs them to preserve the original column type + for the "most_frequent" and "constant" strategies. Parameters ---------- @@ -273,9 +278,42 @@ def fit( """ if hasattr(x, "types") and x.types is not None: self._input_types = dict(x.types) - self._input_columns = {name: idx for idx, name in enumerate(x.column_names)} return super().fit(x, y) + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope imputed in place, plus indicators. + + "most_frequent" and "constant" only copy existing values, so every + column keeps its type; "mean" and "median" produce Float. With + ``add_indicator``, one indicator column is added per column that had + missing values during fit, a count only known after fit. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The imputed inputs, plus an indicator block if requested. + """ + import pyarrow as pa + + if self.strategy in ("most_frequent", "constant"): + kept = list(inputs) + else: + type_name, dtype = type_fields(Float(arrow_type=pa.float64())) + kept = [ + item.model_copy(update={"type": type_name, "dtype": dtype}) + for item in inputs + ] + added = [] + if self.add_indicator: + type_name, dtype = type_fields(Integer(arrow_type=pa.int64())) + added = [BlockItem(label="missingindicator_*", type=type_name, dtype=dtype)] + return StructureDelta(kept=kept, added=added) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a column. @@ -290,10 +328,10 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType: ``Integer`` for the binary ``MissingIndicator`` columns appended when ``add_indicator=True``. Otherwise, the original column type for ``"most_frequent"``/``"constant"`` (no arithmetic is performed - on the values), or for ``"mean"``/``"median"``, ``Integer`` if the - source column was an Integer and the computed statistic is a - whole number — otherwise a Float type backed by - ``pyarrow.float64()``. + on the values), and a Float type backed by ``pyarrow.float64()`` + for ``"mean"``/``"median"``. The type never depends on the fitted + statistics, so it can be known before fitting (see + ``infer_output_columns``). """ import pyarrow as pa @@ -308,16 +346,4 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType: return input_type return Float(arrow_type=pa.float64()) - if isinstance(input_type, Integer): - columns = getattr(self, "_input_columns", None) - statistics = getattr(self, "statistics_", None) - if ( - columns is not None - and statistics is not None - and column_name in columns - ): - value = statistics[columns[column_name]] - if float(value).is_integer(): - return Integer(arrow_type=pa.int64()) - return Float(arrow_type=pa.float64()) diff --git a/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py b/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py index e97e1733e..cf519365f 100644 --- a/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py +++ b/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py @@ -3,6 +3,7 @@ from DashAI.back.api.utils import create_random_state from DashAI.back.converters.category.polynomial_kernel import PolynomialKernelConverter from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( enum_field, float_field, @@ -94,7 +95,10 @@ class SkewedChi2SamplerSchema(BaseSchema): class SkewedChi2Sampler( - PolynomialKernelConverter, SklearnWrapper, SkewedChi2SamplerOperation + ComponentsOutputMixin, + PolynomialKernelConverter, + SklearnWrapper, + SkewedChi2SamplerOperation, ): """Approximate the skewed chi-squared kernel feature map via random Fourier features. diff --git a/DashAI/back/converters/scikit_learn/truncated_svd.py b/DashAI/back/converters/scikit_learn/truncated_svd.py index b25195168..59d53ad0e 100644 --- a/DashAI/back/converters/scikit_learn/truncated_svd.py +++ b/DashAI/back/converters/scikit_learn/truncated_svd.py @@ -5,6 +5,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( enum_field, float_field, @@ -127,7 +128,10 @@ class TruncatedSVDSchema(BaseSchema): class TruncatedSVD( - DimensionalityReductionConverter, SklearnWrapper, TruncatedSVDOperation + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + TruncatedSVDOperation, ): """Reduce dimensionality using Truncated Singular Value Decomposition (LSA). diff --git a/DashAI/back/converters/structure_mixins.py b/DashAI/back/converters/structure_mixins.py new file mode 100644 index 000000000..59ec5bebb --- /dev/null +++ b/DashAI/back/converters/structure_mixins.py @@ -0,0 +1,28 @@ +"""Reusable `infer_output_columns` overrides shared by several converters.""" + +from typing import List, Optional + +from DashAI.back.preprocessing.structure_types import StateItem, StructureDelta + + +class ComponentsOutputMixin: + """Structure of a converter that outputs `n_components` new columns. + + Covers projections and kernel approximations (PCA, TruncatedSVD, + Nystroem, RBFSampler...), which consume their scope and output one + column per component. The count is known whenever `n_components` is an + integer. None, a float (explained-variance ratio) or "mle" only resolve + on fit: None means min(n_samples, n_features) for the PCA family. List + this mixin before the converter's category base so its override wins. + """ + + def _components_count(self, inputs: List[StateItem]) -> Optional[int]: + n_components = getattr(self, "n_components", None) + if isinstance(n_components, int) and not isinstance(n_components, bool): + return n_components + return None + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + return StructureDelta( + added=self._default_blocks(count=self._components_count(inputs)) + ) diff --git a/DashAI/back/preprocessing/structure_types.py b/DashAI/back/preprocessing/structure_types.py index 5bdd3b591..449b894a1 100644 --- a/DashAI/back/preprocessing/structure_types.py +++ b/DashAI/back/preprocessing/structure_types.py @@ -95,6 +95,23 @@ class StructureResult(BaseModel): valid: bool +def known_width(items: List[StateItem]) -> Optional[int]: + """Return how many real columns the items stand for, if that is known. + + A concrete column counts as one; a block counts as its `count`. None + when any block's size is only known after fit. + """ + width = 0 + for item in items: + if isinstance(item, ColumnItem): + width += 1 + elif item.count is None: + return None + else: + width += item.count + return width + + def type_fields(dashai_type: Any) -> Tuple[Optional[str], Optional[str]]: """Return the (display name, dtype) pair the frontend shows for a type. diff --git a/tests/back/preprocessing/test_structure_matches_runtime.py b/tests/back/preprocessing/test_structure_matches_runtime.py index 32c72aeb8..65a55dec0 100644 --- a/tests/back/preprocessing/test_structure_matches_runtime.py +++ b/tests/back/preprocessing/test_structure_matches_runtime.py @@ -107,6 +107,32 @@ ] +# Cases whose output size follows from params and input columns alone: the +# estimate must be exact (no block of unknown size), not just conservative. +EXACT = { + "SimpleImputer[strategy=mean]", + "SimpleImputer[strategy=most_frequent]", + "KNNImputer", + "MissingIndicator", + "Binarizer", + "OrdinalEncoder", + "LabelEncoder", + "PCA[n_components=2]", + "IncrementalPCA[n_components=2]", + "TruncatedSVD[n_components=2]", + "FastICA[n_components=2]", + "Nystroem[n_components=5]", + "RBFSampler[n_components=5]", + "SkewedChi2Sampler[n_components=5]", + "AdditiveChi2Sampler", + "PolynomialFeatures[degree=2]", + "StandardScaler", + "MinMaxScaler", + "MaxAbsScaler", + "Normalizer", +} + + def _converter_classes(): return { cls.__name__: cls @@ -171,6 +197,13 @@ def test_estimated_structure_matches_runtime(case): dataset = _dataset() delta = _build(cls, params).infer_output_columns(_scope_items(dataset, scope)) + if _case_id(case) in EXACT: + unknown = [ + item + for item in delta.added + if isinstance(item, BlockItem) and item.count is None + ] + assert unknown == [], "the estimate should know the output size" converter = _build(cls, params) x = dataset.select_columns(scope) From 6ae36aed14379f5697e0256f359944278a70175f Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 11:38:58 +0200 Subject: [PATCH 31/39] Enhance converters with infer_output_columns method and structure estimation - Added infer_output_columns method to ImageEmbeddingConverter to estimate output structure based on input StateItems. - Implemented infer_output_columns in SAM3SegmentConverter to provide output structure based on segmentation masks. - Updated CharacterReplacer to preserve input types and added infer_output_columns method for output estimation. - Enhanced ColumnArithmetic with infer_output_columns to estimate output based on selected columns and operations. - Introduced infer_output_columns in ColumnConcat to estimate output structure based on concatenated columns. - Added infer_output_columns to ColumnRemover to indicate no columns are kept after removal. - Implemented infer_output_columns in DateFeaturesConverter to estimate output based on date features. - Enhanced NumericExpansion with infer_output_columns to estimate output structure based on numeric operations. - Added infer_output_columns in TypeCast to provide warnings for potential casting issues. - Introduced tests for structure estimation in converters that download pretrained models. - Updated test_structure_matches_runtime to include new converters and ensure output structure matches runtime behavior. --- .../hugging_face/image_embedding.py | 25 ++- .../segmentation/sam3_segment_converter.py | 38 +++- .../simple_converters/character_replacer.py | 81 ++------ .../simple_converters/column_arithmetic.py | 175 +++++++++++++----- .../simple_converters/column_concat.py | 98 ++++++++-- .../simple_converters/column_remover.py | 18 +- .../simple_converters/date_features.py | 74 ++++++-- .../simple_converters/numeric_expansion.py | 98 +++++++--- .../converters/simple_converters/type_cast.py | 33 +++- .../test_download_converters_structure.py | 80 ++++++++ .../test_structure_matches_runtime.py | 24 +++ 11 files changed, 569 insertions(+), 175 deletions(-) create mode 100644 tests/back/converters/test_download_converters_structure.py diff --git a/DashAI/back/converters/hugging_face/image_embedding.py b/DashAI/back/converters/hugging_face/image_embedding.py index c529b8646..1f6711c48 100644 --- a/DashAI/back/converters/hugging_face/image_embedding.py +++ b/DashAI/back/converters/hugging_face/image_embedding.py @@ -1,6 +1,6 @@ """HuggingFace image embedding converter""" -from typing import TYPE_CHECKING, Callable, Dict, Optional +from typing import TYPE_CHECKING, Callable, Dict, List, Optional from DashAI.back.converters.category.advanced_preprocessing import ( AdvancedPreprocessingConverter, @@ -19,6 +19,7 @@ DEVICE_PLACEHOLDER, DEVICE_TO_IDX, ) +from DashAI.back.preprocessing.structure_types import StateItem, StructureDelta from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.dashai_image import DashAIImage from DashAI.back.types.value_types import Float @@ -264,6 +265,28 @@ def fit( return self + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: one block of embedding columns. + + The number of embedding columns is the encoder's hidden size, only + known once the model is loaded. The source image columns survive + only when ``keep_source_column`` is true. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The source columns (if kept), plus the embedding block. + """ + return StructureDelta( + kept=list(inputs) if self.keep_source_column else [], + added=self._default_blocks(), + ) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI type produced for a given output column. diff --git a/DashAI/back/converters/segmentation/sam3_segment_converter.py b/DashAI/back/converters/segmentation/sam3_segment_converter.py index 12639d1cf..e14ec2358 100644 --- a/DashAI/back/converters/segmentation/sam3_segment_converter.py +++ b/DashAI/back/converters/segmentation/sam3_segment_converter.py @@ -1,7 +1,7 @@ """Promptable instance segmentation converter that appends ranked columns.""" import io -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Dict, List, Optional from kink import di @@ -27,6 +27,12 @@ DEVICE_PLACEHOLDER, DEVICE_TO_IDX, ) +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.segmenters.rendering import render_binary_mask, render_segment from DashAI.back.segmenters.sam3_segmenter import SAM3Segmenter from DashAI.back.types.dashai_data_type import DashAIDataType @@ -816,6 +822,36 @@ def transform( return DashAIDataset(result_table, splits=x.splits, types=output_types) + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus one column set per rank. + + ``max_masks`` fixes how many ``segment_i`` and ``seg_score_i`` + columns (and ``mask_i`` with ``keep_binary_mask``) are appended, + whatever the model detects, so every name is known ahead of fit. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the segment, score and mask columns. + """ + names = [ + name + for rank in range(1, self.max_masks + 1) + for name in (_segment_column(rank), _score_column(rank)) + ] + if self.keep_binary_mask: + names += [_mask_column(rank) for rank in range(1, self.max_masks + 1)] + added = [] + for name in names: + type_name, dtype = type_fields(self.get_output_type(name)) + added.append(ColumnItem(name=name, type=type_name, dtype=dtype)) + return StructureDelta(kept=list(inputs), added=added) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI type produced for a given output column. diff --git a/DashAI/back/converters/simple_converters/character_replacer.py b/DashAI/back/converters/simple_converters/character_replacer.py index c02bd41eb..739dc9603 100644 --- a/DashAI/back/converters/simple_converters/character_replacer.py +++ b/DashAI/back/converters/simple_converters/character_replacer.py @@ -9,7 +9,7 @@ from DashAI.back.core.utils import MultilingualString from DashAI.back.types.categorical import Categorical from DashAI.back.types.dashai_data_type import DashAIDataType -from DashAI.back.types.value_types import Integer, Text +from DashAI.back.types.value_types import Text if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset @@ -61,13 +61,16 @@ class CharacterReplacer(BasicPreprocessingConverter, BaseConverter): """Replace or remove a character or substring in all selected text columns. Scans each value in the configured string columns and substitutes every - occurrence of ``char_to_replace`` with ``replacement_char``. If the - replacement produces a column of pure integers, the column type is promoted - to ``Integer``. + occurrence of ``char_to_replace`` with ``replacement_char``. Every column + keeps its type, even when the replacement leaves only digits: follow up + with TypeCast to turn such a column into ``Integer``. Deciding that from + the values would make the output type depend on the data (and on each + batch), so it could not be known before fitting. """ SCHEMA = CharacterReplacerSchema LEARNS_FROM_DATA = False + PRESERVES_INPUT_TYPE = True DESCRIPTION = MultilingualString( en=( "Replaces or removes specified characters/substrings in selected " @@ -182,37 +185,15 @@ def transform( Returns ------- DashAIDataset - A new dataset with ``char_to_replace`` substituted in all text columns. - If replacement yields only integer-like values, the column type is - promoted to ``Integer``. + A new dataset with ``char_to_replace`` substituted in all text + columns, each keeping its type. """ - import pyarrow as pa - from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset if not self._target_columns: # if no target columns were set, return the dataset unchanged return x - def try_convert_to_int(value): - """Try to convert a value to integer, return the original if not possible. - - Parameters - ---------- - value : object - The value to convert. - - Returns - ------- - int or object - ``int(value)`` on success, or the original ``value`` unchanged - if a ``ValueError`` or ``TypeError`` is raised. - """ - try: - return int(value) - except (ValueError, TypeError): - return value - new_types = x.types.copy() categorical_new_values: dict = { col: set() @@ -225,40 +206,16 @@ def replace_function(batch): processed_batch = {} for column_name, values in batch.items(): if column_name in self._target_columns: - if isinstance(x.types[column_name], Text): - replaced_values = [ - ( - val.replace(self.char_to_replace, self.replacement_char) - if isinstance(val, str) - else val - ) - for val in values - ] - - all_numeric = all( - isinstance(val, str) and val.strip().isdigit() - for val in replaced_values + replaced_values = [ + ( + val.replace(self.char_to_replace, self.replacement_char) if isinstance(val, str) + else val ) - - if all_numeric: - processed_batch[column_name] = [ - try_convert_to_int(val) for val in replaced_values - ] - new_types[column_name] = Integer(arrow_type=pa.int64()) - else: - processed_batch[column_name] = replaced_values - new_types[column_name] = Text(arrow_type=pa.string()) - else: - replaced_values = [ - ( - val.replace(self.char_to_replace, self.replacement_char) - if isinstance(val, str) - else val - ) - for val in values - ] - processed_batch[column_name] = replaced_values + for val in values + ] + processed_batch[column_name] = replaced_values + if column_name in categorical_new_values: categorical_new_values[column_name].update( v for v in replaced_values if v is not None ) @@ -285,8 +242,8 @@ def replace_function(batch): def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the default output type for a transformed column. - The actual type may be ``Integer`` if all values become numeric during - ``transform``, but the static default is ``Text``. + A transformed column keeps its own input type (``Text`` or + ``Categorical``); ``Text`` is only the default with no column given. Parameters ---------- diff --git a/DashAI/back/converters/simple_converters/column_arithmetic.py b/DashAI/back/converters/simple_converters/column_arithmetic.py index e0b065b4c..f156b15c0 100644 --- a/DashAI/back/converters/simple_converters/column_arithmetic.py +++ b/DashAI/back/converters/simple_converters/column_arithmetic.py @@ -1,5 +1,5 @@ import math -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.converters.category.feature_engineering import ( @@ -15,6 +15,12 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -154,13 +160,13 @@ class ColumnArithmetic(FeatureEngineeringConverter, BaseConverter): ``__`` or ``__``. The output column is ``Integer`` when both operands are ``Integer`` (a - whole-number ``constant`` counts as ``Integer``), the operation is + whole-number ``constant`` counts as ``Integer``) and the operation is ``add``, ``subtract``, or ``multiply`` (all of which stay exact on - integers), and neither operand column has missing values (since a - missing value has no exact integer representation). ``divide`` always + integers); a missing operand gives a missing result. ``divide`` always produces a ``Float`` column, since integer division is not exact in - general, and any operation involving a ``Float`` operand, or an - operand column with missing values, also produces a ``Float`` column. + general, and any operation involving a ``Float`` operand also produces a + ``Float`` column. The type never depends on the values, so it is known + before fitting (see ``infer_output_columns``). """ SCHEMA = ColumnArithmeticSchema @@ -307,7 +313,46 @@ def fit( selected column is not numeric (Float or Integer), or if a single column is selected without a valid ``constant``. """ - columns = list(x.column_names) + types = {name: type(x.types.get(name)).__name__ for name in x.column_names} + ( + self.operand_b_mode, + self.column_a, + self.column_b, + self._result_column_name, + self._output_is_integer, + ) = self._plan(list(x.column_names), types) + if self.operand_b_mode == "constant": + self.constant = float(self.constant) + return self + + def _plan( + self, columns: List[str], types: Dict[str, str] + ) -> Tuple[str, str, Optional[str], str, bool]: + """Work out the operands, result name and result type from the scope. + + Shared by ``fit`` and ``infer_output_columns``, so the estimated + column is exactly the one ``transform`` appends. + + Parameters + ---------- + columns : List[str] + The scope column names, in dataset order. + types : Dict[str, str] + Each scope column's DashAI type name (e.g. "Integer"). + + Returns + ------- + tuple + (operand_b_mode, column_a, column_b, result_column_name, + output_is_integer); column_b is None in "constant" mode. + + Raises + ------ + ValueError + If the number of selected columns is not one or two, if a + selected column is not numeric (Float or Integer), or if a + single column is selected without a valid ``constant``. + """ if len(columns) not in (1, 2): raise ValueError( "ColumnArithmetic requires selecting one or two columns in " @@ -315,25 +360,19 @@ def fit( ) for col in columns: - if not isinstance(x.types.get(col), (Float, Integer)): + if types.get(col) not in ("Float", "Integer"): raise ValueError( f"Column '{col}' must be numeric (Float or Integer) to be " "used in ColumnArithmetic." ) if len(columns) == 2: - self.operand_b_mode = "column" - first, second = columns + operand_b_mode = "column" + column_a, column_b = columns if self.swap_operands: - first, second = second, first - self.column_a = first - self.column_b = second - operand_b_label = self.column_b - operand_b_is_integer = isinstance(x.types.get(self.column_b), Integer) - has_missing_values = ( - x.arrow_table[self.column_a].null_count > 0 - or x.arrow_table[self.column_b].null_count > 0 - ) + column_a, column_b = column_b, column_a + operand_b_label = column_b + operand_b_is_integer = types[column_b] == "Integer" else: if not isinstance(self.constant, (int, float)) or isinstance( self.constant, bool @@ -347,24 +386,55 @@ def fit( "'constant' must be a finite number (not NaN or " "infinite) when a single column is selected in scope." ) - self.operand_b_mode = "constant" - self.constant = float(self.constant) - self.column_a = columns[0] - self.column_b = None - operand_b_label = self._format_constant(self.constant) + operand_b_mode = "constant" + column_a, column_b = columns[0], None + operand_b_label = self._format_constant(float(self.constant)) operand_b_is_integer = self.constant == int(self.constant) - has_missing_values = x.arrow_table[self.column_a].null_count > 0 - self._result_column_name = self.output_column_name or ( - f"{self.column_a}_{self.operation}_{operand_b_label}" + result_column_name = self.output_column_name or ( + f"{column_a}_{self.operation}_{operand_b_label}" ) - self._output_is_integer = ( + output_is_integer = ( self.operation != "divide" - and isinstance(x.types.get(self.column_a), Integer) + and types[column_a] == "Integer" and operand_b_is_integer - and not has_missing_values ) - return self + return ( + operand_b_mode, + column_a, + column_b, + result_column_name, + output_is_integer, + ) + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus the result column. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the single result column. + + Raises + ------ + ValueError + If the scope or ``constant`` is invalid (see ``_plan``). + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + _, _, _, name, is_integer = self._plan( + [item.name for item in inputs], {item.name: item.type for item in inputs} + ) + type_name, dtype = type_fields(self._result_type(is_integer)) + return StructureDelta( + kept=list(inputs), + added=[ColumnItem(name=name, type=type_name, dtype=dtype)], + ) def _get_operand_b(self, x_pandas, dtype: str): """Return the second operand as an array aligned with ``x_pandas``. @@ -400,23 +470,26 @@ def transform( """ import numpy as np import pyarrow as pa + import pyarrow.compute as pc from DashAI.back.dataloaders.classes.dashai_dataset import modify_table x_pandas = x.to_pandas() if self._output_is_integer: - a = x_pandas[self.column_a].to_numpy(dtype="int64") - b = self._get_operand_b(x_pandas, "int64") - - if self.operation == "add": - result = a + b - elif self.operation == "subtract": - result = a - b - else: # multiply - result = a * b - - arrow_type = pa.int64() + # Arrow arithmetic keeps a missing operand as a missing result, + # which a numpy int64 array cannot represent. + a = x.arrow_table[self.column_a].cast(pa.int64()) + if self.operand_b_mode == "column": + b = x.arrow_table[self.column_b].cast(pa.int64()) + else: + b = pa.scalar(int(self.constant), type=pa.int64()) + operation = { + "add": pc.add, + "subtract": pc.subtract, + "multiply": pc.multiply, + }[self.operation] + result = operation(a, b) else: a = x_pandas[self.column_a].to_numpy(dtype="float64") b = self._get_operand_b(x_pandas, "float64") @@ -431,21 +504,18 @@ def transform( else: # divide result = np.where(b != 0, a / b, np.nan) - arrow_type = pa.float64() + result = pa.array(result, type=pa.float64()) new_types = dict(x.types) new_types[self._result_column_name] = self.get_output_type() - return modify_table( - x, - {self._result_column_name: pa.array(result, type=arrow_type)}, - types=new_types, - ) + return modify_table(x, {self._result_column_name: result}, types=new_types) def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the output type for the arithmetic result. - Determined during ``fit``: ``Integer`` when both operands are + Determined during ``fit`` (or ``infer_output_columns``): ``Integer`` + when both operands are ``Integer`` (a whole-number ``constant`` counts as ``Integer``) and the operation isn't ``divide``, ``Float`` otherwise. @@ -461,8 +531,13 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType: An ``Integer`` type backed by ``pyarrow.int64()``, or a ``Float`` type backed by ``pyarrow.float64()``. """ + return self._result_type(self._output_is_integer) + + @staticmethod + def _result_type(is_integer: bool) -> DashAIDataType: + """Return ``Integer`` or ``Float`` for the result column.""" import pyarrow as pa - if self._output_is_integer: + if is_integer: return Integer(arrow_type=pa.int64()) return Float(arrow_type=pa.float64()) diff --git a/DashAI/back/converters/simple_converters/column_concat.py b/DashAI/back/converters/simple_converters/column_concat.py index d70659808..c71731931 100644 --- a/DashAI/back/converters/simple_converters/column_concat.py +++ b/DashAI/back/converters/simple_converters/column_concat.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.converters.category.feature_engineering import ( @@ -12,6 +12,12 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.categorical import Categorical from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Text @@ -269,7 +275,43 @@ def fit( selected column is not string-like (Text or Categorical), or if a single column is selected without a valid ``constant``. """ - columns = list(x.column_names) + types = {name: type(x.types.get(name)).__name__ for name in x.column_names} + ( + self.operand_b_mode, + self.column_a, + self.column_b, + self._result_column_name, + ) = self._plan(list(x.column_names), types) + return self + + def _plan( + self, columns: List[str], types: Dict[str, str] + ) -> Tuple[str, str, Optional[str], str]: + """Work out the operands and the result name from the scope. + + Shared by ``fit`` and ``infer_output_columns``, so the estimated + column is exactly the one ``transform`` appends. + + Parameters + ---------- + columns : List[str] + The scope column names, in dataset order. + types : Dict[str, str] + Each scope column's DashAI type name (e.g. "Text"). + + Returns + ------- + tuple + (operand_b_mode, column_a, column_b, result_column_name); + column_b is None in "constant" mode. + + Raises + ------ + ValueError + If the number of selected columns is not one or two, if a + selected column is not string-like (Text or Categorical), or if + a single column is selected without a valid ``constant``. + """ if len(columns) not in (1, 2): raise ValueError( "ColumnConcat requires selecting one or two columns in " @@ -277,35 +319,61 @@ def fit( ) for col in columns: - if not isinstance(x.types.get(col), (Text, Categorical)): + if types.get(col) not in ("Text", "Categorical"): raise ValueError( f"Column '{col}' must be a string (Text or Categorical) " "to be used in ColumnConcat." ) if len(columns) == 2: - self.operand_b_mode = "column" - first, second = columns + operand_b_mode = "column" + column_a, column_b = columns if self.swap_operands: - first, second = second, first - self.column_a = first - self.column_b = second - operand_b_label = self.column_b + column_a, column_b = column_b, column_a + operand_b_label = column_b else: if not isinstance(self.constant, str): raise ValueError( "'constant' must be a string when a single column is " "selected in scope." ) - self.operand_b_mode = "constant" - self.column_a = columns[0] - self.column_b = None + operand_b_mode = "constant" + column_a, column_b = columns[0], None operand_b_label = self.constant - self._result_column_name = self.output_column_name or ( - f"{self.column_a}_concat_{operand_b_label}" + result_column_name = self.output_column_name or ( + f"{column_a}_concat_{operand_b_label}" + ) + return operand_b_mode, column_a, column_b, result_column_name + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus the Text result column. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the single result column. + + Raises + ------ + ValueError + If the scope or ``constant`` is invalid (see ``_plan``). + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + _, _, _, name = self._plan( + [item.name for item in inputs], {item.name: item.type for item in inputs} + ) + type_name, dtype = type_fields(self.get_output_type()) + return StructureDelta( + kept=list(inputs), + added=[ColumnItem(name=name, type=type_name, dtype=dtype)], ) - return self def transform( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/converters/simple_converters/column_remover.py b/DashAI/back/converters/simple_converters/column_remover.py index 33bac88a8..776cd9dd6 100644 --- a/DashAI/back/converters/simple_converters/column_remover.py +++ b/DashAI/back/converters/simple_converters/column_remover.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.converters.category.basic_preprocessing import ( @@ -6,6 +6,7 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import StateItem, StructureDelta from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Text @@ -64,6 +65,21 @@ def __init__(self): super().__init__() self.columns = [] + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: every scope column is removed, nothing added. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + An empty delta: no input survives and no column is created. + """ + return StructureDelta() + def fit(self, x: "DashAIDataset", y: "DashAIDataset" = None) -> "ColumnRemover": """Record the column names that will be removed during ``transform``. diff --git a/DashAI/back/converters/simple_converters/date_features.py b/DashAI/back/converters/simple_converters/date_features.py index 8dd8e6298..7d7b0acb7 100644 --- a/DashAI/back/converters/simple_converters/date_features.py +++ b/DashAI/back/converters/simple_converters/date_features.py @@ -7,6 +7,12 @@ from DashAI.back.core.schema_fields import bool_field, schema_field from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Date, Float, Integer @@ -344,6 +350,57 @@ def _selected_features(self) -> List[str]: """ return [name for name in FEATURE_SUFFIXES if self.features[name]] + def _feature_columns(self, date_columns: List[str]) -> Dict[str, DashAIDataType]: + """Name and type every column added for the given date columns. + + Parameters + ---------- + date_columns : List[str] + The date columns to extract features from. + + Returns + ------- + Dict[str, DashAIDataType] + New column name to its type, in the order they are written out: + one Integer column per selected feature, plus a Float sine and + cosine column per periodic feature when cyclical encoding is on. + """ + import pyarrow as pa + + columns: Dict[str, DashAIDataType] = {} + for column in date_columns: + for feature in self._selected_features(): + suffix = FEATURE_SUFFIXES[feature] + columns[f"{column}_{suffix}"] = Integer(arrow_type=pa.int64()) + if self.cyclical and feature in CYCLICAL_PERIODS: + columns[f"{column}_{suffix}_sin"] = Float(arrow_type=pa.float64()) + columns[f"{column}_{suffix}_cos"] = Float(arrow_type=pa.float64()) + return columns + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus the calendar features. + + Only Date columns get features, exactly as ``fit`` picks them. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus one column per feature and date column. + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + date_columns = [item.name for item in inputs if item.type == "Date"] + added = [] + for name, dashai_type in self._feature_columns(date_columns).items(): + type_name, dtype = type_fields(dashai_type) + added.append(ColumnItem(name=name, type=type_name, dtype=dtype)) + return StructureDelta(kept=list(inputs), added=added) + def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None ) -> "DateFeaturesConverter": @@ -361,25 +418,10 @@ def fit( DateFeaturesConverter The fitted converter instance (self). """ - import pyarrow as pa - self._date_columns = [ name for name in x.column_names if isinstance(x.types.get(name), Date) ] - self._output_types = {} - for column in self._date_columns: - for feature in self._selected_features(): - suffix = FEATURE_SUFFIXES[feature] - self._output_types[f"{column}_{suffix}"] = Integer( - arrow_type=pa.int64() - ) - if self.cyclical and feature in CYCLICAL_PERIODS: - self._output_types[f"{column}_{suffix}_sin"] = Float( - arrow_type=pa.float64() - ) - self._output_types[f"{column}_{suffix}_cos"] = Float( - arrow_type=pa.float64() - ) + self._output_types = self._feature_columns(self._date_columns) if not self._date_columns: print( diff --git a/DashAI/back/converters/simple_converters/numeric_expansion.py b/DashAI/back/converters/simple_converters/numeric_expansion.py index f5dd4643b..742c9e4df 100644 --- a/DashAI/back/converters/simple_converters/numeric_expansion.py +++ b/DashAI/back/converters/simple_converters/numeric_expansion.py @@ -7,6 +7,12 @@ from DashAI.back.core.schema_fields import enum_field, schema_field from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -58,13 +64,11 @@ class NumericExpansion(FeatureEngineeringConverter, BaseConverter): ``sqrt``) become ``NaN`` in the corresponding output. The original columns are left untouched. ``square`` preserves the input - column's type (``Integer`` stays ``Integer``, ``Float`` stays ``Float``) - when the source column has no missing values, since squaring is exact - for both. ``log1p`` and ``sqrt`` always produce a ``Float`` column, since - they can yield non-integer or ``NaN`` results even from integer input, - and ``square`` also falls back to ``Float`` when the source ``Integer`` - column has missing values (since a missing value has no exact integer - representation). + column's type (``Integer`` stays ``Integer``, ``Float`` stays ``Float``), + since squaring is exact for both; a missing value stays missing. ``log1p`` + and ``sqrt`` always produce a ``Float`` column, since they can yield + non-integer or ``NaN`` results even from integer input. The output type + only depends on the operation and the input type, never on the values. """ SCHEMA = NumericExpansionSchema @@ -158,24 +162,15 @@ def fit( NumericExpansion The fitted converter instance (self). """ - import pyarrow as pa - self._target_columns = [] self._output_types = {} for col_name in x.column_names: col_type = x.types.get(col_name) if isinstance(col_type, (Float, Integer)): self._target_columns.append(col_name) - new_col_name = f"{self.operation}_{col_name}" - has_missing_values = x.arrow_table[col_name].null_count > 0 - if ( - self.operation == "square" - and isinstance(col_type, Integer) - and not has_missing_values - ): - self._output_types[new_col_name] = Integer(arrow_type=pa.int64()) - else: - self._output_types[new_col_name] = Float(arrow_type=pa.float64()) + self._output_types[f"{self.operation}_{col_name}"] = ( + self._expansion_type(isinstance(col_type, Integer)) + ) else: print( f"Warning: Column '{col_name}' in scope is not numeric " @@ -188,6 +183,54 @@ def fit( ) return self + def _expansion_type(self, input_is_integer: bool) -> DashAIDataType: + """Return the type of an expanded column. + + Parameters + ---------- + input_is_integer : bool + Whether the source column is ``Integer`` (otherwise ``Float``). + + Returns + ------- + DashAIDataType + ``Integer`` for the square of an ``Integer`` column, ``Float`` + otherwise. + """ + import pyarrow as pa + + if self.operation == "square" and input_is_integer: + return Integer(arrow_type=pa.int64()) + return Float(arrow_type=pa.float64()) + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus one column per numeric one. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus an ``_`` column for + every Float or Integer input, as ``fit`` picks them. + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + added = [] + for item in inputs: + if item.type not in ("Float", "Integer"): + continue + type_name, dtype = type_fields(self._expansion_type(item.type == "Integer")) + added.append( + ColumnItem( + name=f"{self.operation}_{item.name}", type=type_name, dtype=dtype + ) + ) + return StructureDelta(kept=list(inputs), added=added) + def transform( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None ) -> "DashAIDataset": @@ -210,6 +253,7 @@ def transform( """ import numpy as np import pyarrow as pa + import pyarrow.compute as pc from DashAI.back.dataloaders.classes.dashai_dataset import modify_table @@ -226,9 +270,10 @@ def transform( output_type = self._output_types[new_col_name] if isinstance(output_type, Integer): - values = x_pandas[col].to_numpy(dtype="int64") - result = values**2 - arrow_type = pa.int64() + # Arrow arithmetic keeps nulls as nulls, which a numpy + # int64 array cannot represent. + values = x.arrow_table[col].cast(pa.int64()) + new_columns[new_col_name] = pc.multiply(values, values) else: values = x_pandas[col].to_numpy(dtype="float64") if self.operation == "log1p": @@ -237,9 +282,7 @@ def transform( result = values**2 else: # sqrt result = np.where(values >= 0, np.sqrt(values), np.nan) - arrow_type = pa.float64() - - new_columns[new_col_name] = pa.array(result, type=arrow_type) + new_columns[new_col_name] = pa.array(result, type=pa.float64()) new_types[new_col_name] = output_type return modify_table(x, new_columns, types=new_types) @@ -247,9 +290,8 @@ def transform( def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the output type for a given expanded column. - Determined during ``fit``: ``Integer`` when the operation is - ``square`` and the source column is ``Integer``, ``Float`` - otherwise. + ``Integer`` when the operation is ``square`` and the source column is + ``Integer``, ``Float`` otherwise. Parameters ---------- diff --git a/DashAI/back/converters/simple_converters/type_cast.py b/DashAI/back/converters/simple_converters/type_cast.py index 2e60c0c90..64866b2a6 100644 --- a/DashAI/back/converters/simple_converters/type_cast.py +++ b/DashAI/back/converters/simple_converters/type_cast.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, List, Union from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.converters.category.basic_preprocessing import ( @@ -7,6 +7,11 @@ from DashAI.back.core.schema_fields import enum_field, schema_field from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + StateItem, + StructureDelta, + StructureMessage, +) from DashAI.back.types.categorical import Categorical from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer, Text @@ -179,6 +184,32 @@ def __init__(self, new_type: str, on_error: str = "raise"): self._fit_x = None self._converted_cache: dict = {} + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope cast in place to ``new_type``. + + With ``on_error="skip"`` a column whose values cannot be cast keeps + its original type, which is only known once the data is seen, so the + estimate carries a ``cast_may_skip`` warning. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The inputs retyped to ``new_type``. + """ + delta = super().infer_output_columns(inputs) + if self.on_error == "skip": + delta.warnings.append( + StructureMessage( + code="cast_may_skip", params={"new_type": self.new_type} + ) + ) + return delta + def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None ) -> "TypeCast": diff --git a/tests/back/converters/test_download_converters_structure.py b/tests/back/converters/test_download_converters_structure.py new file mode 100644 index 000000000..2d98e6d6f --- /dev/null +++ b/tests/back/converters/test_download_converters_structure.py @@ -0,0 +1,80 @@ +"""Structure estimates of converters that download pretrained models. + +They cannot join the estimate-vs-runtime test (fitting loads the model), but +building them is cheap because the model is only loaded on fit, so their +`infer_output_columns` overrides are checked directly. +""" + +import pytest + +from DashAI.back.converters.hugging_face.embedding import Embedding +from DashAI.back.converters.hugging_face.image_embedding import ( + ImageEmbeddingConverter, +) +from DashAI.back.converters.hugging_face.tokenizer import TokenizerConverter +from DashAI.back.converters.segmentation.sam3_segment_converter import ( + SAM3SegmentConverter, +) +from DashAI.back.preprocessing.structure_types import BlockItem, ColumnItem +from DashAI.back.splitters.splits_payload import schema_placeholder_defaults + + +def _build(cls, **params): + return cls(**{**schema_placeholder_defaults(cls), **params}) + + +@pytest.mark.parametrize( + ("cls", "type_name"), + [(Embedding, "Float"), (TokenizerConverter, "Integer")], +) +def test_text_encoders_replace_the_text_with_one_block(cls, type_name): + inputs = [ColumnItem(name="review", type="Text", dtype="string")] + + delta = _build(cls).infer_output_columns(inputs) + + assert delta.kept == [] + (block,) = delta.added + assert isinstance(block, BlockItem) + assert (block.type, block.count) == (type_name, None) + + +@pytest.mark.parametrize("keep_source", [True, False]) +def test_image_embedding_keeps_the_source_only_when_asked(keep_source): + inputs = [ColumnItem(name="photo", type="Image")] + converter = _build(ImageEmbeddingConverter, keep_source_column=keep_source) + + delta = converter.infer_output_columns(inputs) + + assert delta.kept == (inputs if keep_source else []) + (block,) = delta.added + assert (block.type, block.count) == ("Float", None) + + +@pytest.mark.parametrize( + ("keep_mask", "expected"), + [ + (False, ["segment_1", "seg_score_1", "segment_2", "seg_score_2"]), + ( + True, + [ + "segment_1", + "seg_score_1", + "segment_2", + "seg_score_2", + "mask_1", + "mask_2", + ], + ), + ], +) +def test_sam3_names_one_column_set_per_rank(keep_mask, expected): + inputs = [ColumnItem(name="photo", type="Image")] + converter = _build(SAM3SegmentConverter, max_masks=2, keep_binary_mask=keep_mask) + + delta = converter.infer_output_columns(inputs) + + assert delta.kept == inputs + assert [item.name for item in delta.added] == expected + types = {item.name: item.type for item in delta.added} + assert types["segment_1"] == "Image" + assert types["seg_score_1"] == "Float" diff --git a/tests/back/preprocessing/test_structure_matches_runtime.py b/tests/back/preprocessing/test_structure_matches_runtime.py index 65a55dec0..dca9f28b8 100644 --- a/tests/back/preprocessing/test_structure_matches_runtime.py +++ b/tests/back/preprocessing/test_structure_matches_runtime.py @@ -130,6 +130,28 @@ "MinMaxScaler", "MaxAbsScaler", "Normalizer", + "CharacterReplacer[char_to_replace=-,replacement_char=]", + "ColumnArithmetic[operation=add]", + "ColumnArithmetic[operation=multiply]", + "ColumnArithmetic[operation=divide]", + "ColumnConcat", + "NumericExpansion[operation=square]", + "NumericExpansion[operation=log1p]", + "DateFeaturesConverter", + "TypeCast[new_type=Float]", + "ColumnRemover", +} + +# Cases whose output is only concrete columns: names must be known. +CONCRETE_ONLY = { + "ColumnArithmetic[operation=add]", + "ColumnArithmetic[operation=multiply]", + "ColumnArithmetic[operation=divide]", + "ColumnConcat", + "NumericExpansion[operation=square]", + "NumericExpansion[operation=log1p]", + "DateFeaturesConverter", + "ColumnRemover", } @@ -204,6 +226,8 @@ def test_estimated_structure_matches_runtime(case): if isinstance(item, BlockItem) and item.count is None ] assert unknown == [], "the estimate should know the output size" + if _case_id(case) in CONCRETE_ONLY: + assert not any(isinstance(i, BlockItem) for i in delta.added), "no blocks" converter = _build(cls, params) x = dataset.select_columns(scope) From 6f67a16900a63968eb772f4826d5776c6d7494e5 Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 11:42:16 +0200 Subject: [PATCH 32/39] feat: implement column name planning and enhance session preprocessing with target columns --- DashAI/back/converters/dataset_columns.py | 49 +++++++--- DashAI/back/job/preprocessing_job.py | 8 +- DashAI/back/preprocessing/column_ref.py | 33 +++++-- .../preprocessing/session_preprocessor.py | 98 ++++++++++++++----- tests/back/preprocessing/test_column_ref.py | 22 +++++ .../test_session_preprocessor.py | 72 ++++++++++++++ 6 files changed, 239 insertions(+), 43 deletions(-) diff --git a/DashAI/back/converters/dataset_columns.py b/DashAI/back/converters/dataset_columns.py index cad8406c5..51f5026d7 100644 --- a/DashAI/back/converters/dataset_columns.py +++ b/DashAI/back/converters/dataset_columns.py @@ -3,12 +3,47 @@ preprocessing (SessionPreprocessor). """ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, Dict, Iterable, List if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset +def plan_new_column_names( + taken_names: Iterable[str], new_columns: List[str] +) -> Dict[str, str]: + """Name every new column so it never collides with an existing one. + + A new column whose name is already taken gets the first free + ``_1``, ``_2``... suffix. Shared by the dataset rebuild below + and by the structure estimate of a session's converter chain, so both + always agree on the final names. + + Parameters + ---------- + taken_names : Iterable[str] + Names already present once the converter's removed columns are gone. + new_columns : List[str] + The converter's new column names, in output order. + + Returns + ------- + Dict[str, str] + Each new column name mapped to its final, unique name. + """ + seen = set(taken_names) + mapping = {} + for col in new_columns: + unique_col = col + counter = 1 + while unique_col in seen: + unique_col = f"{col}_{counter}" + counter += 1 + seen.add(unique_col) + mapping[col] = unique_col + return mapping + + def rebuild_dataset_with_transformed_columns( base: "DashAIDataset", transformed: "DashAIDataset", @@ -70,16 +105,8 @@ def rebuild_dataset_with_transformed_columns( new_columns_order.append(col) seen_cols.add(col) - col_name_mapping = {} - for col in new_cols: - unique_col = col - counter = 1 - while unique_col in seen_cols: - unique_col = f"{col}_{counter}" - counter += 1 - new_columns_order.append(unique_col) - seen_cols.add(unique_col) - col_name_mapping[col] = unique_col + col_name_mapping = plan_new_column_names(seen_cols, new_cols) + new_columns_order.extend(col_name_mapping.values()) updated_arrays = {} for col in replacement_cols: diff --git a/DashAI/back/job/preprocessing_job.py b/DashAI/back/job/preprocessing_job.py index 993d997d9..8d2b90968 100644 --- a/DashAI/back/job/preprocessing_job.py +++ b/DashAI/back/job/preprocessing_job.py @@ -138,7 +138,9 @@ def run(self) -> None: f"Fitting preprocessing for fold {i + 1}/{total_folds}", ) fold_preprocessor = SessionPreprocessor( - sequence, component_registry + sequence, + component_registry, + target_columns=model_session.output_columns, ) fold_preprocessor.fit_transform(x_folds[i]) with open(os.path.join(tmp_dir, f"fold_{i}.pkl"), "wb") as f: @@ -146,7 +148,9 @@ def run(self) -> None: self.report_progress(0.85, "Fitting final preprocessing") final_preprocessor = SessionPreprocessor( - sequence, component_registry + sequence, + component_registry, + target_columns=model_session.output_columns, ) final_transformed, final_resolved = ( final_preprocessor.fit_transform(x_folds[-1]) diff --git a/DashAI/back/preprocessing/column_ref.py b/DashAI/back/preprocessing/column_ref.py index 652860a64..4e666baa1 100644 --- a/DashAI/back/preprocessing/column_ref.py +++ b/DashAI/back/preprocessing/column_ref.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import BaseModel, Field, TypeAdapter +from pydantic import BaseModel, Field, TypeAdapter, model_validator from typing_extensions import Annotated @@ -31,6 +31,19 @@ class GroupColumnRef(BaseModel): # them (see SessionPreprocessor._classify_by_type). The slot name is a # DashAI type's display_name(), e.g. "Categorical" or "Integer". slot: Optional[str] = None + # One specific column the step produced, for converters whose output + # names are known before fit (e.g. DateFeatures' "date_month"). Keeping + # the producing step in the ref, instead of a plain RawColumnRef, lets a + # caller walk it back to the original columns it depends on. + name: Optional[str] = None + + @model_validator(mode="after") + def _slot_or_name(self) -> "GroupColumnRef": + if self.slot is not None and self.name is not None: + raise ValueError( + "A group reference takes either a slot or a column name, not both." + ) + return self ColumnRef = Annotated[Union[RawColumnRef, GroupColumnRef], Field(discriminator="kind")] @@ -85,21 +98,29 @@ def resolve_refs( Returns ------- list of str - Concrete column names, in order. A GroupColumnRef with `slot=None` - expands to every column its step produced; with a `slot` set, only - to that step's columns of that declared type. + Concrete column names, in order. A GroupColumnRef with a `name` + resolves to that one column; with a `slot`, to that step's columns + of that declared type; with neither, to every column its step + produced. Raises ------ KeyError If a GroupColumnRef names a step with no entry in resolved_columns - (or, for a slot ref, no matching entry in resolved_slots) — the step - has not been fit yet, or produced no column of that type. + (or, for a slot ref, no matching entry in resolved_slots), or a + column its step did not produce: the step has not been fit yet, or + produced no such column. """ names: List[str] = [] for ref in refs: if ref.kind == "raw": names.append(ref.name) + elif ref.name is not None: + if ref.name not in resolved_columns[ref.step]: + raise KeyError( + f"Step {ref.step} did not produce a column named '{ref.name}'." + ) + names.append(ref.name) elif ref.slot is None: names.extend(resolved_columns[ref.step]) else: diff --git a/DashAI/back/preprocessing/session_preprocessor.py b/DashAI/back/preprocessing/session_preprocessor.py index 7ca6ff801..2e9c73540 100644 --- a/DashAI/back/preprocessing/session_preprocessor.py +++ b/DashAI/back/preprocessing/session_preprocessor.py @@ -8,9 +8,10 @@ import os import pickle -from typing import TYPE_CHECKING, Any, Dict, List, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from DashAI.back.converters.dataset_columns import ( + plan_new_column_names, rebuild_dataset_with_transformed_columns, ) from DashAI.back.preprocessing.column_ref import ConverterSequence, resolve_refs @@ -26,11 +27,21 @@ class SessionPreprocessor: of an earlier step. Fitting always uses only the "train" entry of whatever split dict is passed in, so no step ever sees validation or test rows during fit. + + Supervised converters (e.g. feature selectors) are fit with the + `target_columns` of the train split as `y`. The target is never part of + a step's scope, so no converter ever transforms it. """ - def __init__(self, sequence: ConverterSequence, component_registry: Any): + def __init__( + self, + sequence: ConverterSequence, + component_registry: Any, + target_columns: Optional[List[str]] = None, + ): self.sequence = sequence self.component_registry = component_registry + self.target_columns = list(target_columns or []) self.fitted_converters: List[Any] = [] self.resolved_columns: Dict[int, List[str]] = {} self.resolved_slots: Dict[int, Dict[str, List[str]]] = {} @@ -66,6 +77,60 @@ def _classify_by_type( slots.setdefault(type_name, []).append(name) return slots + def _fit(self, converter: Any, train: "DashAIDataset", train_scope) -> Any: + """Fit a step's converter, passing the target to supervised ones.""" + # A preprocessor pickled before target_columns existed has no such + # attribute; it is only unpickled to transform, never to fit again. + target_columns = getattr(self, "target_columns", None) + if type(converter).SUPERVISED and target_columns: + return converter.fit(train_scope, train.select_columns(target_columns)) + return converter.fit(train_scope) + + def _record_group( + self, + index: int, + converter: Any, + train: "DashAIDataset", + scope_names: List[str], + output_names: List[str], + ) -> None: + """Record which real columns a step produced, and their slots. + + A converter that only rewrites its scope columns in place (e.g. a + scaler: "age" in, scaled "age" out) has no other way to expose its + result, so the scope names ARE the group. A converter that adds new + columns (e.g. Bag-of-Words, which also keeps its scope column + verbatim) only exposes the new ones: a passthrough scope column is + not this step's output, and would still carry its pre-conversion + type (e.g. Text), which is never valid as a resolved input column. + + New columns are recorded under the final name they get in the + dataset: one whose name is already taken is renamed by + rebuild_dataset_with_transformed_columns (e.g. "derived_1"). Slots + are classified with the names the converter itself produced, since + that is what its get_output_type knows. + """ + new_columns = [name for name in output_names if name not in scope_names] + if not new_columns: + self.resolved_columns[index] = list(output_names) + self.resolved_slots[index] = self._classify_by_type( + converter, self.resolved_columns[index] + ) + return + + removed = {name for name in scope_names if name not in output_names} + final_names = plan_new_column_names( + [name for name in train.column_names if name not in removed], + new_columns, + ) + self.resolved_columns[index] = [final_names[name] for name in new_columns] + self.resolved_slots[index] = { + type_name: [final_names[name] for name in names] + for type_name, names in self._classify_by_type( + converter, new_columns + ).items() + } + @staticmethod def _transform_split(converter, dataset, scope_names, train_transformed): """Transform one split's scoped columns, without crashing on 0 rows. @@ -129,7 +194,7 @@ def fit_transform( converter = self._instantiate(step) train_scope = current["train"].select_columns(scope_names) - converter = converter.fit(train_scope) + converter = self._fit(converter, current["train"], train_scope) train_transformed = converter.transform(train_scope) transformed_by_split = {"train": train_transformed} @@ -140,27 +205,12 @@ def fit_transform( converter, dataset, scope_names, train_transformed ) - # A converter that only rewrites its scope columns in place (e.g. a - # scaler: "age" in, scaled "age" out) has no other way to expose - # its result, so the scope names ARE the group. But a converter - # like Bag-of-Words additionally keeps its scope column verbatim - # alongside brand-new derived columns (see BagOfWordsConverter. - # transform's docstring: "the source text column is preserved - # unchanged") — for those, the untouched scope column is a - # passthrough, not this step's own output, so it must not leak - # into the group a later step or the wizard's input selection can - # reference (it would still carry the pre-conversion dtype, e.g. - # Text, which is never valid as a resolved input column). - new_columns = [ - name - for name in train_transformed.column_names - if name not in scope_names - ] - self.resolved_columns[index] = ( - new_columns if new_columns else list(train_transformed.column_names) - ) - self.resolved_slots[index] = self._classify_by_type( - converter, self.resolved_columns[index] + self._record_group( + index, + converter, + current["train"], + scope_names, + list(train_transformed.column_names), ) new_current = {} diff --git a/tests/back/preprocessing/test_column_ref.py b/tests/back/preprocessing/test_column_ref.py index 18510c20a..60087bc76 100644 --- a/tests/back/preprocessing/test_column_ref.py +++ b/tests/back/preprocessing/test_column_ref.py @@ -125,3 +125,25 @@ def test_converter_sequence_model_dump_round_trips(): dumped = sequence.model_dump(mode="json") restored = ConverterSequence.model_validate(dumped) assert restored == sequence + + +def test_named_group_ref_resolves_to_that_one_column(): + refs = [GroupColumnRef(step=0, name="date_month")] + resolved = resolve_refs(refs, {0: ["date_year", "date_month"]}) + assert resolved == ["date_month"] + + +def test_named_group_ref_to_a_column_the_step_never_produced_raises_key_error(): + with pytest.raises(KeyError): + resolve_refs([GroupColumnRef(step=0, name="age")], {0: ["date_year"]}) + + +def test_group_ref_cannot_have_both_a_slot_and_a_name(): + with pytest.raises(ValueError, match="slot"): + GroupColumnRef(step=0, slot="Integer", name="date_year") + + +def test_parse_column_refs_round_trips_a_named_group_ref(): + raw = [{"kind": "group", "step": 1, "name": "date_month"}] + (ref,) = parse_column_refs(raw) + assert (ref.step, ref.name, ref.slot) == (1, "date_month", None) diff --git a/tests/back/preprocessing/test_session_preprocessor.py b/tests/back/preprocessing/test_session_preprocessor.py index e2bd22c6c..1cb3a4ee5 100644 --- a/tests/back/preprocessing/test_session_preprocessor.py +++ b/tests/back/preprocessing/test_session_preprocessor.py @@ -500,3 +500,75 @@ def __init__(self, classes): assert restored.transform_dataset(single).to_pandas()["age"].tolist() == [10] assert restored.component_registry is None + + +class _TargetRecordingConverter(BaseConverter): + """A supervised converter that fails without a target, like a selector.""" + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + SUPERVISED = True + seen_targets = [] + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + if y is None: + raise ValueError("This transformer requires y for fitting") + type(self).seen_targets.append(list(y.column_names)) + return self + + def transform(self, x, y=None): + return x + + +def test_supervised_step_is_fit_with_the_target_columns(): + registry = _FakeRegistry({"Selector": _TargetRecordingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Selector", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry, target_columns=["label"]) + schema = { + "age": {"type": "Integer", "dtype": "int64"}, + "label": {"type": "Integer", "dtype": "int64"}, + } + split = {"train": _dataset({"age": [1, 2, 3], "label": [0, 1, 0]}, schema)} + + preprocessor.fit_transform(split) + + assert _TargetRecordingConverter.seen_targets == [["label"]] + + +def test_a_renamed_new_column_is_recorded_under_its_final_name(): + # The additive converter appends "derived", which already exists in the + # dataset outside the step's scope, so the new column becomes + # "derived_1"; the step's group must point at that one, not at the + # untouched original. + registry = _FakeRegistry({"Additive": _FakeAdditiveConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Additive", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + schema = { + "age": {"type": "Integer", "dtype": "int64"}, + "derived": {"type": "Integer", "dtype": "int64"}, + } + split = {"train": _dataset({"age": [1, 2], "derived": [7, 7]}, schema)} + + transformed, resolved = preprocessor.fit_transform(split) + + assert resolved == {0: ["derived_1"]} + assert transformed["train"].to_pandas()["derived_1"].tolist() == [10, 20] + assert preprocessor.resolved_slots == {0: {"Integer": ["derived_1"]}} From e17b5b3a3eea9d1ea085e3ffcd379d098c87573e Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 11:46:02 +0200 Subject: [PATCH 33/39] feat: add infer_structure function and corresponding tests for session preprocessing --- DashAI/back/preprocessing/structure.py | 368 +++++++++++++++++++++ tests/back/preprocessing/test_structure.py | 228 +++++++++++++ 2 files changed, 596 insertions(+) create mode 100644 DashAI/back/preprocessing/structure.py create mode 100644 tests/back/preprocessing/test_structure.py diff --git a/DashAI/back/preprocessing/structure.py b/DashAI/back/preprocessing/structure.py new file mode 100644 index 000000000..863bf7e5b --- /dev/null +++ b/DashAI/back/preprocessing/structure.py @@ -0,0 +1,368 @@ +"""Estimate the dataset structure along a session's converter chain. + +`infer_structure` walks a ConverterSequence without fitting anything: each +step's scope is resolved against the estimated dataset state at that point, +checked against the converter's metadata, and the converter's own +`infer_output_columns` delta is applied the same way SessionPreprocessor +splices real output back into the dataset. The Models-module wizard uses it +to show which columns exist after every step, and session creation uses it +to reject a chain that would reference a column that no longer exists. +""" + +from typing import Any, Dict, List, Set, Tuple + +from DashAI.back.converters.dataset_columns import plan_new_column_names +from DashAI.back.preprocessing.column_ref import ConverterStep +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + ColumnItem, + RowsNotSupportedError, + StateItem, + StepStructure, + StructureDelta, + StructureMessage, + StructureResult, + known_width, + type_fields, +) +from DashAI.back.splitters.splits_payload import schema_placeholder_defaults + + +class _StepError(Exception): + """A step the estimate rejects, carrying the message for the frontend.""" + + def __init__(self, code: str, **params: Any): + super().__init__(code) + self.message = StructureMessage(code=code, params=params) + + +def _identity(item: StateItem) -> Tuple: + """What a later delta or ref uses to recognise an item.""" + if isinstance(item, ColumnItem): + return ("column", item.name) + return ("block", item.step, item.slot) + + +def infer_structure( + dataset_types: Dict[str, Any], + candidates: List[str], + target: List[str], + steps: List[ConverterStep], + component_registry: Any, +) -> StructureResult: + """Estimate the dataset state after every step of a converter chain. + + Parameters + ---------- + dataset_types : dict + Every dataset column name, in dataset order, mapped to its DashAI + type. + candidates : list of str + The original columns the user may feed into the chain. The initial + state is these columns, minus the target. + target : list of str + The output columns. They never enter the state and no scope may + reference them. + steps : list of ConverterStep + The chain, in order. + component_registry : ComponentRegistry or mapping + Resolves a converter name to ``{"class": converter_class}``. + + Returns + ------- + StructureResult + The initial state, one entry per step (its status, the state after + it and what it produced) and the final state. After the first + invalid step the rest are "blocked" and the final state is the one + before it. + """ + target_set = set(target) + state: List[StateItem] = [] + for name in dataset_types: + if name in candidates and name not in target_set: + type_name, dtype = type_fields(dataset_types[name]) + state.append(ColumnItem(name=name, type=type_name, dtype=dtype)) + initial = list(state) + # The runtime renames a new column against every column of the dataset, + # not just the candidates, so name clashes are checked against all. + taken: Set[str] = set(dataset_types) + + results: List[StepStructure] = [] + blocked = False + for index, step in enumerate(steps): + if blocked: + results.append(StepStructure(status="blocked", state=state)) + continue + try: + state, added, warnings = _apply_step( + index, step, state, taken, target_set, component_registry + ) + except _StepError as e: + results.append(StepStructure(status="error", state=state, error=e.message)) + blocked = True + continue + results.append( + StepStructure(status="ok", state=state, added=added, warnings=warnings) + ) + + return StructureResult( + initial=initial, + steps=results, + final=state, + valid=not blocked, + ) + + +def _apply_step( + index: int, + step: ConverterStep, + state: List[StateItem], + taken: Set[str], + target: Set[str], + component_registry: Any, +) -> Tuple[List[StateItem], List[StateItem], List[StructureMessage]]: + """Estimate one step. Updates `taken` in place; raises _StepError.""" + try: + converter_class = component_registry[step.converter]["class"] + except KeyError as e: + raise _StepError("unknown_converter", converter=step.converter) from e + converter = _instantiate(converter_class, step.params) + + scope = _resolve_scope(step, state, target) + _check_scope(converter_class, scope) + + warnings: List[StructureMessage] = [] + try: + delta = converter.infer_output_columns(scope) + except RowsNotSupportedError as e: + raise _StepError("rows_not_supported", converter=step.converter) from e + except ValueError as e: + # A converter validating its own configuration against the scope + # (e.g. ColumnArithmetic without a constant for a single column). + raise _StepError("converter_rejected", detail=str(e)) from e + except Exception: + # A converter whose estimate is broken (e.g. an older plugin) falls + # back to the conservative default: its scope is consumed and its + # output is one block of unknown size and type. + delta = StructureDelta(added=[BlockItem()]) + warnings.append( + StructureMessage(code="inference_fallback", params={"step": index}) + ) + warnings.extend(delta.warnings) + warnings.extend(_check_bounds(converter_class, converter, scope)) + + new_state, added = _apply_delta(index, state, scope, delta, taken) + return new_state, added, warnings + + +def _instantiate(converter_class: Any, params: Dict[str, Any]) -> Any: + """Validate the user's params and build the converter with them. + + The params are validated the way the frontend form submits them, with + every field it seeds from the schema placeholders (as _validate_splits + does for splitters), but the converter is built from the params exactly + as given, as SessionPreprocessor builds it, so the estimate describes + the converter that will really run. + """ + try: + schema = getattr(converter_class, "SCHEMA", None) + if schema is not None: + schema.model_validate( + {**schema_placeholder_defaults(converter_class), **params} + ) + return converter_class(**params) + except Exception as e: + raise _StepError("invalid_params", detail=str(e)) from e + + +def _resolve_scope( + step: ConverterStep, state: List[StateItem], target: Set[str] +) -> List[StateItem]: + """Map a step's ColumnRefs to the state items they point at.""" + scope: List[StateItem] = [] + seen: Set[Tuple] = set() + for ref in step.scope: + if ref.kind == "raw": + if ref.name in target: + raise _StepError("target_in_scope", column=ref.name) + matches = [ + item + for item in state + if isinstance(item, ColumnItem) + and item.origin is None + and item.name == ref.name + ] + label = ref.name + elif ref.name is not None: + matches = [ + item + for item in state + if isinstance(item, ColumnItem) + and item.origin == ref.step + and item.name == ref.name + ] + label = ref.name + elif ref.slot is not None: + matches = [ + item + for item in state + if isinstance(item, BlockItem) + and item.step == ref.step + and item.slot == ref.slot + ] + label = f"{ref.step}:{ref.slot}" + else: + # Whole group: everything the step produced that still exists. + matches = [ + item + for item in state + if (isinstance(item, ColumnItem) and item.origin == ref.step) + or (isinstance(item, BlockItem) and item.step == ref.step) + ] + label = str(ref.step) + if not matches: + raise _StepError("missing_ref", ref=label) + for item in matches: + if _identity(item) not in seen: + seen.add(_identity(item)) + scope.append(item) + return scope + + +def _check_scope(converter_class: Any, scope: List[StateItem]) -> None: + """Apply the converter's declared column restrictions to its scope. + + Mirrors the frontend's evaluateColumnEligibility: empty allow-lists mean + no restriction. An item of unknown type is let through, since only the + fit can tell. + """ + metadata = converter_class.get_metadata() + allowed_types = metadata.get("allowed_types") or [] + allowed_dtypes = metadata.get("allowed_dtypes") or [] + non_allowed_dtypes = metadata.get("non_allowed_dtypes") or [] + for item in scope: + if item.type is None: + continue + if ( + (allowed_types and item.type not in allowed_types) + or (allowed_dtypes and item.dtype not in allowed_dtypes) + or (item.dtype in non_allowed_dtypes) + ): + raise _StepError( + "type_not_allowed", + column=item.name if isinstance(item, ColumnItem) else item.label, + type=item.type, + allowed=allowed_types, + ) + + cardinality = metadata.get("input_cardinality") or {} + # A block of unknown size holds at least one column. + at_least = sum( + 1 if isinstance(item, ColumnItem) else (item.count or 1) for item in scope + ) + exactly = known_width(scope) + required = cardinality.get("exact") + minimum = required if required is not None else cardinality.get("min") + maximum = required if required is not None else cardinality.get("max") + too_few = minimum is not None and (exactly or at_least) < minimum + too_many = maximum is not None and exactly is not None and exactly > maximum + if too_few or too_many: + raise _StepError( + "cardinality", + min=minimum, + max=maximum, + selected=exactly if exactly is not None else at_least, + ) + + +def _check_bounds( + converter_class: Any, converter: Any, scope: List[StateItem] +) -> List[StructureMessage]: + """Check `n_components` against the scope width, for projections. + + Only converters flagged N_COMPONENTS_FEATURES_BOUNDED (the PCA family) + cannot output more components than input columns; kernel approximations + can. + """ + n_components = getattr(converter, "n_components", None) + if not getattr(converter_class, "N_COMPONENTS_FEATURES_BOUNDED", False): + return [] + if not isinstance(n_components, int) or isinstance(n_components, bool): + return [] + width = known_width(scope) + if width is None: + return [ + StructureMessage( + code="count_unknown_bound", params={"n_components": n_components} + ) + ] + if n_components > width: + raise _StepError( + "n_components_exceeds", n_components=n_components, columns=width + ) + return [] + + +def _apply_delta( + index: int, + state: List[StateItem], + scope: List[StateItem], + delta: StructureDelta, + taken: Set[str], +) -> Tuple[List[StateItem], List[StateItem]]: + """Apply a converter's delta to the state, as the runtime splices output. + + Surviving scope items stay in place (maybe retyped), consumed ones are + dropped, and new items are appended at the end with runtime names: a + new column whose name is taken is renamed like + rebuild_dataset_with_transformed_columns does. + """ + concrete = [item for item in delta.added if isinstance(item, ColumnItem)] + blocks = [item for item in delta.added if isinstance(item, BlockItem)] + if concrete and blocks: + raise ValueError( + f"Step {index} mixes concrete columns and blocks in one delta." + ) + + scope_ids = {_identity(item) for item in scope} + kept = {_identity(item): item for item in delta.kept} + new_state: List[StateItem] = [] + removed_names: List[str] = [] + for item in state: + item_id = _identity(item) + if item_id not in scope_ids: + new_state.append(item) + elif item_id in kept: + new_state.append(kept[item_id]) + elif isinstance(item, ColumnItem): + removed_names.append(item.name) + + # A step that outputs blocks may keep some consumed columns under their + # own names (a selector keeps a subset), so their names stay taken. + if not blocks: + taken.difference_update(removed_names) + + added: List[StateItem] = [] + if concrete: + final_names = plan_new_column_names(taken, [item.name for item in concrete]) + for item in concrete: + added.append( + item.model_copy( + update={"name": final_names[item.name], "origin": index} + ) + ) + taken.update(final_names.values()) + else: + # A single block is the step's whole group (a ref with no slot); + # several are told apart by type, as the runtime classifies them. + for block in blocks: + added.append( + block.model_copy( + update={ + "step": index, + "slot": block.type if len(blocks) > 1 else None, + } + ) + ) + + return new_state + added, added diff --git a/tests/back/preprocessing/test_structure.py b/tests/back/preprocessing/test_structure.py new file mode 100644 index 000000000..63e80c566 --- /dev/null +++ b/tests/back/preprocessing/test_structure.py @@ -0,0 +1,228 @@ +import pandas as pd + +from DashAI.back.converters.imbalanced_learn.smote_converter import SMOTEConverter +from DashAI.back.converters.scikit_learn.pca import PCA +from DashAI.back.converters.scikit_learn.select_k_best import SelectKBest +from DashAI.back.converters.scikit_learn.standard_scaler import StandardScaler +from DashAI.back.converters.simple_converters.date_features import ( + DateFeaturesConverter, +) +from DashAI.back.dataloaders.classes.dashai_dataset import ( + to_dashai_dataset, + transform_dataset_with_schema, +) +from DashAI.back.preprocessing.column_ref import ( + ConverterSequence, + ConverterStep, + GroupColumnRef, + RawColumnRef, +) +from DashAI.back.preprocessing.session_preprocessor import SessionPreprocessor +from DashAI.back.preprocessing.structure import infer_structure +from DashAI.back.preprocessing.structure_types import BlockItem, ColumnItem + +REGISTRY = { + cls.__name__: {"class": cls} + for cls in ( + PCA, + SelectKBest, + StandardScaler, + DateFeaturesConverter, + SMOTEConverter, + ) +} + +SCHEMA = { + "age": {"type": "Integer", "dtype": "int64"}, + "height": {"type": "Float", "dtype": "float64"}, + "city": {"type": "Categorical", "dtype": "string"}, + "date": {"type": "Date", "dtype": "%Y-%m-%d"}, + "date_year": {"type": "Integer", "dtype": "int64"}, + "label": {"type": "Categorical", "dtype": "string"}, +} + + +def _dataset(): + frame = pd.DataFrame( + { + "age": [30, 41, 25, 60], + "height": [1.7, 1.8, 1.6, 1.75], + "city": ["a", "b", "a", "c"], + "date": ["2024-01-05", "2024-02-10", "2024-03-15", "2024-04-20"], + "date_year": [1, 2, 3, 4], + "label": ["yes", "no", "yes", "no"], + } + ) + return transform_dataset_with_schema(to_dashai_dataset(frame), SCHEMA) + + +def _infer(steps, candidates=("age", "height", "city", "date"), target=("label",)): + return infer_structure( + dataset_types=_dataset().types, + candidates=list(candidates), + target=list(target), + steps=steps, + component_registry=REGISTRY, + ) + + +def _step(converter, scope, **params): + return ConverterStep(converter=converter, params=params, scope=scope) + + +def _names(state): + return [item.name for item in state if isinstance(item, ColumnItem)] + + +def test_initial_state_is_the_candidates_without_the_target(): + result = _infer([], candidates=["age", "label", "city"]) + + assert _names(result.initial) == ["age", "city"] + assert result.final == result.initial + assert result.valid + + +def test_a_consumed_column_cannot_be_used_by_a_later_step(): + result = _infer( + [ + _step( + "PCA", + [RawColumnRef(name="age"), RawColumnRef(name="height")], + n_components=2, + ), + _step("StandardScaler", [RawColumnRef(name="age")]), + ] + ) + + first, second = result.steps + assert first.status == "ok" + assert "age" not in _names(first.state) + (block,) = first.added + assert isinstance(block, BlockItem) + assert (block.step, block.slot, block.count) == (0, None, 2) + assert second.status == "error" + assert second.error.code == "missing_ref" + assert not result.valid + + +def test_the_target_cannot_be_in_a_scope(): + result = _infer([_step("StandardScaler", [RawColumnRef(name="label")])]) + + assert result.steps[0].error.code == "target_in_scope" + + +def test_steps_after_a_broken_one_are_blocked(): + result = _infer( + [ + _step("StandardScaler", [RawColumnRef(name="missing")]), + _step("StandardScaler", [RawColumnRef(name="age")]), + ] + ) + + assert [step.status for step in result.steps] == ["error", "blocked"] + assert _names(result.final) == ["age", "height", "city", "date"] + + +def test_an_unknown_converter_is_an_error(): + result = _infer([_step("Nope", [RawColumnRef(name="age")])]) + + assert result.steps[0].error.code == "unknown_converter" + + +def test_a_scope_type_the_converter_does_not_accept_is_an_error(): + result = _infer([_step("StandardScaler", [RawColumnRef(name="city")])]) + + assert result.steps[0].error.code == "type_not_allowed" + + +def test_row_changing_converters_are_not_supported(): + result = _infer([_step("SMOTEConverter", [RawColumnRef(name="age")])]) + + assert result.steps[0].error.code == "rows_not_supported" + + +def test_more_components_than_columns_is_an_error(): + result = _infer( + [ + _step( + "PCA", + [RawColumnRef(name="age"), RawColumnRef(name="height")], + n_components=5, + ) + ] + ) + + error = result.steps[0].error + assert error.code == "n_components_exceeds" + assert error.params == {"n_components": 5, "columns": 2} + + +def test_a_named_generated_column_can_be_used_by_a_later_step(): + result = _infer( + [ + _step("DateFeaturesConverter", [RawColumnRef(name="date")]), + _step("StandardScaler", [GroupColumnRef(step=0, name="date_month")]), + ] + ) + + assert result.valid + month = next(item for item in result.final if item.name == "date_month") + assert (month.type, month.origin) == ("Float", 0) + + +def test_a_selector_over_mixed_types_adds_one_slotted_block_per_type(): + result = _infer( + [ + _step( + "SelectKBest", + [RawColumnRef(name="age"), RawColumnRef(name="height")], + k=1, + ), + ] + ) + + blocks = result.steps[0].added + assert {(block.slot, block.type) for block in blocks} == { + ("Integer", "Integer"), + ("Float", "Float"), + } + + +def test_a_clashing_new_column_is_renamed_like_the_runtime_does(): + # "date_year" exists in the dataset but is not a candidate, so the one + # DateFeatures creates must become "date_year_1", as the runtime names it. + step = _step("DateFeaturesConverter", [RawColumnRef(name="date")]) + + estimated = _infer([step]) + + preprocessor = SessionPreprocessor(ConverterSequence(steps=[step]), REGISTRY) + _, resolved = preprocessor.fit_transform({"train": _dataset()}) + added = [item.name for item in estimated.steps[0].added] + assert added[0] == "date_year_1" + assert added == resolved[0] + + +class _BrokenEstimateConverter(StandardScaler): + """Stands in for a plugin whose structure estimate crashes.""" + + def get_output_type(self, column_name=None): + raise RuntimeError("no idea") + + +def test_a_crashing_estimate_falls_back_to_one_unknown_block(): + registry = {**REGISTRY, "Broken": {"class": _BrokenEstimateConverter}} + + result = infer_structure( + dataset_types=_dataset().types, + candidates=["age", "height"], + target=["label"], + steps=[_step("Broken", [RawColumnRef(name="age")])], + component_registry=registry, + ) + + step = result.steps[0] + assert step.status == "ok" + assert [w.code for w in step.warnings] == ["inference_fallback"] + (block,) = step.added + assert (block.type, block.count) == (None, None) + assert _names(step.state) == ["height"] From a04dc5cf2756a284f9da041b6ea74847e7b63bf0 Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 11:54:28 +0200 Subject: [PATCH 34/39] feat: implement preprocessing structure estimation and validation endpoints --- .../api/api_v1/endpoints/model_sessions.py | 218 +++++++++++++++--- .../api_v1/schemas/model_sessions_params.py | 13 ++ DashAI/back/preprocessing/structure.py | 57 +++-- tests/back/api/test_model_session_api.py | 155 +++++++++++++ .../back/api/test_run_preprocessing_guard.py | 13 +- 5 files changed, 405 insertions(+), 51 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index aad2cc72b..d20c00cb9 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -11,11 +11,17 @@ ColumnsValidationParams, ModelSessionBulkDeleteParams, ModelSessionParams, + PreprocessingStructureParams, ) from DashAI.back.api.utils import remove_path from DashAI.back.dependencies.database.models import Dataset, ModelSession, Run from DashAI.back.job.preprocessing_job import PreprocessingJob from DashAI.back.preprocessing.column_ref import ConverterSequence, RawColumnRef +from DashAI.back.preprocessing.structure import ( + StructureError, + infer_structure, + resolve_state_refs, +) from DashAI.back.splitters.splits_payload import ( META_KEYS, normalize_splits_payload, @@ -121,14 +127,26 @@ async def get_model_session( return model_session -@router.post("/validation") -@inject -async def validate_columns( - params: ColumnsValidationParams, - component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """Validate if dataset columns are compatible with a task.""" +def _load_sample_dataset(db, dataset_id: int): + """Load the first rows of a dataset, enough for its columns and types. + + Parameters + ---------- + db : Session + An open database session. + dataset_id : int + The dataset to load. + + Returns + ------- + DashAIDataset + At most 5 rows of the dataset, with every column and its type. + + Raises + ------ + HTTPException + 404 if the dataset is not registered. + """ import os import pyarrow as pa @@ -136,25 +154,92 @@ async def validate_columns( from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + dataset = db.get(Dataset, dataset_id) + if not dataset: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Dataset not found", + ) + data_filepath = os.path.join(f"{dataset.file_path}/dataset", "data.arrow") + with pa.OSFile(data_filepath, "rb") as source: + reader = ipc.open_file(source) + batch = reader.get_batch(0) + sample_batch = batch.slice(0, min(5, batch.num_rows)) + return DashAIDataset(pa.Table.from_batches([sample_batch])) + + +def _type_allowed(type_name, allowed_types) -> bool: + """Whether a column type is one of a task's allowed input types. + + A task declaring the DashAIValue wildcard accepts every concrete value + type; an unknown type (None) is let through, since only a fit can tell. + """ + if not allowed_types or type_name is None: + return True + return type_name in allowed_types or ( + "DashAIValue" in allowed_types and type_name in _DASHAI_VALUE_TYPE_NAMES + ) + + +@router.post("/preprocessing/structure") +@inject +async def get_preprocessing_structure( + params: PreprocessingStructureParams, + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Estimate the dataset structure after every step of a converter chain. + + Read only: nothing is fit and nothing is stored. The session wizard calls + it on every change to the chain, to offer only columns that exist at each + step and to flag steps that cannot work. + + Parameters + ---------- + params : PreprocessingStructureParams + The dataset, the candidate and target columns, and the chain. + + Returns + ------- + StructureResult + The state after every step, their errors and warnings, and the final + state. + """ with session_factory() as db: try: - dataset = db.get(Dataset, params.dataset_id) - if not dataset: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Dataset not found", - ) + sample = _load_sample_dataset(db, params.dataset_id) + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + return infer_structure( + dataset_types=sample.types, + candidates=params.candidates, + target=params.output_columns, + steps=params.steps, + component_registry=component_registry, + ) - dataset_path = f"{dataset.file_path}/dataset" - data_filepath = os.path.join(dataset_path, "data.arrow") - with pa.OSFile(data_filepath, "rb") as source: - reader = ipc.open_file(source) - batch = reader.get_batch(0) - sample_size = min(5, batch.num_rows) - sample_batch = batch.slice(0, sample_size) - table = pa.Table.from_batches([sample_batch]) - minimal_dataset = DashAIDataset(table) +@router.post("/validation") +@inject +async def validate_columns( + params: ColumnsValidationParams, + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Validate if dataset columns are compatible with a task. + + With `preprocessing`, every input ref is resolved against the chain's + estimated final structure and its type checked against the task's input + types; the original columns among them also go through the task's own + validation, as without preprocessing. + """ + with session_factory() as db: + try: + minimal_dataset = _load_sample_dataset(db, params.dataset_id) column_names = minimal_dataset.column_names @@ -192,7 +277,36 @@ async def validate_columns( task: "BaseTask" = component_registry[params.task_name]["class"]() - if group_refs: + if params.preprocessing: + allowed_input_types = set(task.get_metadata().get("inputs_types", [])) + structure = infer_structure( + dataset_types=minimal_dataset.types, + candidates=list(column_names), + target=outputs_names, + steps=params.preprocessing, + component_registry=component_registry, + ) + if not structure.valid: + return { + "dataset_status": "invalid", + "error": "The preprocessing chain has an invalid step.", + } + try: + items = resolve_state_refs( + params.input_refs or [], structure.final, set(outputs_names) + ) + except StructureError as e: + return {"dataset_status": "invalid", "error": e.message.code} + for item in items: + if not _type_allowed(item.type, allowed_input_types): + return { + "dataset_status": "invalid", + "error": ( + f"Input of type '{item.type}' is not one of the task's " + f"allowed input types {sorted(allowed_input_types)}." + ), + } + elif group_refs: declared_types = params.converter_output_types or {} task_metadata = task.get_metadata() allowed_input_types = set(task_metadata.get("inputs_types", [])) @@ -204,9 +318,8 @@ async def validate_columns( # "{step}" key, unchanged from before slots existed. key = str(ref.step) if ref.slot is None else f"{ref.step}:{ref.slot}" declared_type = declared_types.get(key) - type_ok = declared_type in allowed_input_types or ( - "DashAIValue" in allowed_input_types - and declared_type in _DASHAI_VALUE_TYPE_NAMES + type_ok = declared_type is not None and _type_allowed( + declared_type, allowed_input_types ) if allowed_input_types and not type_ok: return { @@ -286,6 +399,52 @@ def _validate_splits(splits: str, component_registry: "ComponentRegistry") -> No ) from e +def _check_preprocessing_structure( + db, params, sequence, input_column_refs, component_registry +) -> None: + """Reject a session whose chain or inputs cannot work, before creating it. + + Without this, the session would be created and its PreprocessingJob + would only fail afterwards, with an error like a bare missing column + name. + + Raises + ------ + HTTPException + 422 with the estimated structure when a step is invalid, or with the + error when an input ref points at something the chain does not + produce (or consumed). + """ + sample = _load_sample_dataset(db, params.dataset_id) + structure = infer_structure( + dataset_types=sample.types, + candidates=list(sample.column_names), + target=params.output_columns, + steps=sequence.steps, + component_registry=component_registry, + ) + if not structure.valid: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={ + "message": "The preprocessing chain has an invalid step.", + "structure": structure.model_dump(mode="json"), + }, + ) + try: + resolve_state_refs( + input_column_refs, structure.final, set(params.output_columns) + ) + except StructureError as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={ + "message": "An input column does not exist after preprocessing.", + "error": e.message.model_dump(mode="json"), + }, + ) from e + + @router.post("/", status_code=status.HTTP_201_CREATED) @inject async def create_model_session( @@ -362,6 +521,11 @@ async def create_model_session( schema = reader.schema column_names = schema.names + if has_preprocessing: + _check_preprocessing_structure( + db, params, sequence, input_column_refs, component_registry + ) + # When preprocessing is configured, input_columns is only a # placeholder until PreprocessingJob resolves the real ones, so # it cannot be checked against the raw dataset's column count. diff --git a/DashAI/back/api/api_v1/schemas/model_sessions_params.py b/DashAI/back/api/api_v1/schemas/model_sessions_params.py index f5e35e415..ed70e1d66 100644 --- a/DashAI/back/api/api_v1/schemas/model_sessions_params.py +++ b/DashAI/back/api/api_v1/schemas/model_sessions_params.py @@ -26,8 +26,21 @@ class ColumnsValidationParams(BaseModel): inputs_columns: List[str] outputs_columns: List[str] input_refs: Optional[List[ColumnRef]] = None + # With preprocessing, the type of every input ref is taken from the + # chain's estimated structure (see infer_structure). + preprocessing: Optional[List[ConverterStep]] = None + # Legacy: group ref types computed by the frontend, used only when no + # preprocessing is sent. converter_output_types: Optional[Dict[str, str]] = None +class PreprocessingStructureParams(BaseModel): + dataset_id: int + # Original columns the user may feed into the chain. + candidates: List[str] + output_columns: List[str] + steps: List[ConverterStep] = Field(default_factory=list) + + class ModelSessionBulkDeleteParams(BaseModel): ids: List[int] diff --git a/DashAI/back/preprocessing/structure.py b/DashAI/back/preprocessing/structure.py index 863bf7e5b..1ae485697 100644 --- a/DashAI/back/preprocessing/structure.py +++ b/DashAI/back/preprocessing/structure.py @@ -28,8 +28,8 @@ from DashAI.back.splitters.splits_payload import schema_placeholder_defaults -class _StepError(Exception): - """A step the estimate rejects, carrying the message for the frontend.""" +class StructureError(Exception): + """A chain or ref the estimate rejects, with the message for the frontend.""" def __init__(self, code: str, **params: Any): super().__init__(code) @@ -97,7 +97,7 @@ def infer_structure( state, added, warnings = _apply_step( index, step, state, taken, target_set, component_registry ) - except _StepError as e: + except StructureError as e: results.append(StepStructure(status="error", state=state, error=e.message)) blocked = True continue @@ -121,25 +121,25 @@ def _apply_step( target: Set[str], component_registry: Any, ) -> Tuple[List[StateItem], List[StateItem], List[StructureMessage]]: - """Estimate one step. Updates `taken` in place; raises _StepError.""" + """Estimate one step. Updates `taken` in place; raises StructureError.""" try: converter_class = component_registry[step.converter]["class"] except KeyError as e: - raise _StepError("unknown_converter", converter=step.converter) from e + raise StructureError("unknown_converter", converter=step.converter) from e converter = _instantiate(converter_class, step.params) - scope = _resolve_scope(step, state, target) + scope = resolve_state_refs(step.scope, state, target) _check_scope(converter_class, scope) warnings: List[StructureMessage] = [] try: delta = converter.infer_output_columns(scope) except RowsNotSupportedError as e: - raise _StepError("rows_not_supported", converter=step.converter) from e + raise StructureError("rows_not_supported", converter=step.converter) from e except ValueError as e: # A converter validating its own configuration against the scope # (e.g. ColumnArithmetic without a constant for a single column). - raise _StepError("converter_rejected", detail=str(e)) from e + raise StructureError("converter_rejected", detail=str(e)) from e except Exception: # A converter whose estimate is broken (e.g. an older plugin) falls # back to the conservative default: its scope is consumed and its @@ -172,19 +172,40 @@ def _instantiate(converter_class: Any, params: Dict[str, Any]) -> Any: ) return converter_class(**params) except Exception as e: - raise _StepError("invalid_params", detail=str(e)) from e + raise StructureError("invalid_params", detail=str(e)) from e -def _resolve_scope( - step: ConverterStep, state: List[StateItem], target: Set[str] +def resolve_state_refs( + refs: List[Any], state: List[StateItem], target: Set[str] ) -> List[StateItem]: - """Map a step's ColumnRefs to the state items they point at.""" + """Map ColumnRefs to the dataset state items they point at. + + Parameters + ---------- + refs : list of RawColumnRef | GroupColumnRef + The refs to resolve, e.g. a step's scope or the model inputs. + state : list of ColumnItem | BlockItem + The estimated dataset state to resolve them against. + target : set of str + The output columns, which no ref may point at. + + Returns + ------- + list of ColumnItem | BlockItem + The items, in ref order, without duplicates. + + Raises + ------ + StructureError + With code "target_in_scope" for a ref to the target, or + "missing_ref" for a ref to something that does not exist (anymore). + """ scope: List[StateItem] = [] seen: Set[Tuple] = set() - for ref in step.scope: + for ref in refs: if ref.kind == "raw": if ref.name in target: - raise _StepError("target_in_scope", column=ref.name) + raise StructureError("target_in_scope", column=ref.name) matches = [ item for item in state @@ -221,7 +242,7 @@ def _resolve_scope( ] label = str(ref.step) if not matches: - raise _StepError("missing_ref", ref=label) + raise StructureError("missing_ref", ref=label) for item in matches: if _identity(item) not in seen: seen.add(_identity(item)) @@ -248,7 +269,7 @@ def _check_scope(converter_class: Any, scope: List[StateItem]) -> None: or (allowed_dtypes and item.dtype not in allowed_dtypes) or (item.dtype in non_allowed_dtypes) ): - raise _StepError( + raise StructureError( "type_not_allowed", column=item.name if isinstance(item, ColumnItem) else item.label, type=item.type, @@ -267,7 +288,7 @@ def _check_scope(converter_class: Any, scope: List[StateItem]) -> None: too_few = minimum is not None and (exactly or at_least) < minimum too_many = maximum is not None and exactly is not None and exactly > maximum if too_few or too_many: - raise _StepError( + raise StructureError( "cardinality", min=minimum, max=maximum, @@ -297,7 +318,7 @@ def _check_bounds( ) ] if n_components > width: - raise _StepError( + raise StructureError( "n_components_exceeds", n_components=n_components, columns=width ) return [] diff --git a/tests/back/api/test_model_session_api.py b/tests/back/api/test_model_session_api.py index d7f018847..5c81e53f4 100644 --- a/tests/back/api/test_model_session_api.py +++ b/tests/back/api/test_model_session_api.py @@ -223,3 +223,158 @@ def test_bulk_delete_model_sessions(client: TestClient, dataset_1: Dataset) -> N for session_id in created_ids: response = client.get(f"/api/v1/model-session/{session_id}") assert response.status_code == 404, response.text + + +BINARIZE_SEPAL_LENGTH = { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], +} + + +def test_preprocessing_structure_returns_the_state_after_each_step( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/preprocessing/structure", + json={ + "dataset_id": dataset_1.id, + "candidates": ["SepalLengthCm", "SepalWidthCm"], + "output_columns": ["Species"], + "steps": [BINARIZE_SEPAL_LENGTH], + }, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["valid"] is True + assert [item["name"] for item in body["initial"]] == [ + "SepalLengthCm", + "SepalWidthCm", + ] + (added,) = body["steps"][0]["added"] + assert (added["name"], added["type"], added["origin"]) == ( + "bin_SepalLengthCm", + "Integer", + 0, + ) + + +def test_preprocessing_structure_reports_a_ref_to_a_missing_column( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/preprocessing/structure", + json={ + "dataset_id": dataset_1.id, + "candidates": ["SepalLengthCm"], + "output_columns": ["Species"], + "steps": [ + { + "converter": "StandardScaler", + "params": {}, + "scope": [{"kind": "raw", "name": "PetalWidthCm"}], + } + ], + }, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["valid"] is False + assert body["steps"][0]["error"]["code"] == "missing_ref" + + +def test_create_model_session_rejects_an_invalid_chain( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "invalid-chain-session", + "input_columns": [], + "preprocessing": [ + { + "converter": "StandardScaler", + "params": {}, + "scope": [{"kind": "raw", "name": "Species"}], + } + ], + "input_column_refs": [{"kind": "raw", "name": "SepalWidthCm"}], + }, + ) + assert response.status_code == 422, response.text + structure = response.json()["detail"]["structure"] + assert structure["steps"][0]["error"]["code"] == "target_in_scope" + + +def test_create_model_session_rejects_an_input_the_chain_consumed( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "consumed-input-session", + "input_columns": [], + "preprocessing": [ + { + "converter": "PCA", + "params": {"n_components": 1}, + "scope": [ + {"kind": "raw", "name": "SepalLengthCm"}, + {"kind": "raw", "name": "SepalWidthCm"}, + ], + } + ], + "input_column_refs": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ) + assert response.status_code == 422, response.text + assert response.json()["detail"]["error"]["code"] == "missing_ref" + + +def test_validate_columns_with_preprocessing_accepts_a_generated_column( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": ["SepalWidthCm"], + "outputs_columns": ["Species"], + "input_refs": [ + {"kind": "raw", "name": "SepalWidthCm"}, + {"kind": "group", "step": 0, "name": "bin_SepalLengthCm"}, + ], + "preprocessing": [BINARIZE_SEPAL_LENGTH], + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "valid" + + +def test_validate_columns_with_preprocessing_rejects_an_input_cast_to_text( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": ["SepalLengthCm"], + "outputs_columns": ["Species"], + "input_refs": [{"kind": "raw", "name": "SepalLengthCm"}], + "preprocessing": [ + { + "converter": "TypeCast", + "params": {"new_type": "Text", "on_error": "raise"}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + } + ], + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "invalid" diff --git a/tests/back/api/test_run_preprocessing_guard.py b/tests/back/api/test_run_preprocessing_guard.py index 4ea339e22..d3d1e6154 100644 --- a/tests/back/api/test_run_preprocessing_guard.py +++ b/tests/back/api/test_run_preprocessing_guard.py @@ -103,14 +103,15 @@ def test_run_creation_surfaces_the_preprocessing_error_when_failed( ), "preprocessing": [ { - # Binarizer only accepts numeric columns; scoping it on the - # categorical target makes PreprocessingJob's fit fail. - "converter": "Binarizer", - "params": {"threshold": 3.0}, - "scope": [{"kind": "raw", "name": "Species"}], + # A chain the structure estimate accepts but the data + # breaks: SepalLengthCm holds decimals (e.g. 5.1), which + # cannot be cast to Integer, so PreprocessingJob fails. + "converter": "TypeCast", + "params": {"new_type": "Integer", "on_error": "raise"}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], }, ], - "input_column_refs": [{"kind": "group", "step": 0}], + "input_column_refs": [{"kind": "raw", "name": "SepalLengthCm"}], }, ) assert session_response.status_code == 201, session_response.text From f6d582f35ccb8c807c3111e5e7b389dbf69370c5 Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 11:59:03 +0200 Subject: [PATCH 35/39] feat: add preprocessing structure estimation and related hooks, types, and tests --- DashAI/front/src/api/modelSession.ts | 30 +++++ .../models/modelSession/sessionColumnRefs.js | 116 +++++++++++++--- .../modelSession/sessionColumnRefs.test.js | 124 ++++++++++++++++++ .../modelSession/usePreprocessingStructure.js | 78 +++++++++++ .../usePreprocessingStructure.test.js | 80 +++++++++++ DashAI/front/src/types/modelSession.ts | 48 +++++++ 6 files changed, 457 insertions(+), 19 deletions(-) create mode 100644 DashAI/front/src/components/models/modelSession/usePreprocessingStructure.js create mode 100644 DashAI/front/src/components/models/modelSession/usePreprocessingStructure.test.js diff --git a/DashAI/front/src/api/modelSession.ts b/DashAI/front/src/api/modelSession.ts index 0c85e6f9e..5fd319ac7 100644 --- a/DashAI/front/src/api/modelSession.ts +++ b/DashAI/front/src/api/modelSession.ts @@ -3,6 +3,7 @@ import type { IColumnRef, IConverterStep, IModelSession, + IStructureResult, } from "../types/modelSession"; const endpointURL = "/v1/model-session"; @@ -100,3 +101,32 @@ export const validateColumns = async ( ); return response.data; }; + +// Estimate the dataset state after every step of a preprocessing chain, +// without fitting anything (see the backend's infer_structure). +export const getPreprocessingStructure = async ({ + datasetId, + candidates, + outputColumns, + steps, +}: { + datasetId: number; + candidates: string[]; + outputColumns: string[]; + steps: IConverterStep[]; +}): Promise => { + const response = await api.post( + `${endpointURL}/preprocessing/structure`, + { + dataset_id: datasetId, + candidates, + output_columns: outputColumns, + steps: steps.map(({ converter, params, scope }) => ({ + converter, + params, + scope, + })), + }, + ); + return response.data; +}; diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js index 97c8f7268..cd403a03d 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js @@ -20,45 +20,123 @@ const GROUP_KEY_PREFIX = "__group__"; const SLOT_SEPARATOR = "__slot__"; +const NAME_SEPARATOR = "__name__"; +// The step, then at most one of a slot or a column name. Only the first +// separator after the step counts, so a slot or column name that itself +// contains a separator stays intact. +const GROUP_KEY_PATTERN = new RegExp( + `^${GROUP_KEY_PREFIX}(\\d+)(?:${SLOT_SEPARATOR}(.*)|${NAME_SEPARATOR}(.*))?$`, + "s", +); export const groupKey = (step, slot = null) => slot == null ? `${GROUP_KEY_PREFIX}${step}` : `${GROUP_KEY_PREFIX}${step}${SLOT_SEPARATOR}${slot}`; +// A generated column whose name is known before fit (e.g. DateFeatures' +// "date_month"), mirroring the backend's GroupColumnRef.name. +export const namedGroupKey = (step, name) => + `${GROUP_KEY_PREFIX}${step}${NAME_SEPARATOR}${name}`; + export const isGroupKey = (key) => typeof key === "string" && key.startsWith(GROUP_KEY_PREFIX); -export const stepFromGroupKey = (key) => { - const withoutPrefix = key.slice(GROUP_KEY_PREFIX.length); - const separatorIndex = withoutPrefix.indexOf(SLOT_SEPARATOR); - const stepPart = - separatorIndex === -1 - ? withoutPrefix - : withoutPrefix.slice(0, separatorIndex); - return Number(stepPart); +const parseGroupKey = (key) => { + const match = GROUP_KEY_PATTERN.exec(key); + return { + step: Number(match[1]), + slot: match[2] ?? null, + name: match[3] ?? null, + }; }; -export const slotFromGroupKey = (key) => { - const withoutPrefix = key.slice(GROUP_KEY_PREFIX.length); - const separatorIndex = withoutPrefix.indexOf(SLOT_SEPARATOR); - return separatorIndex === -1 - ? null - : withoutPrefix.slice(separatorIndex + SLOT_SEPARATOR.length); -}; +export const stepFromGroupKey = (key) => parseGroupKey(key).step; + +export const slotFromGroupKey = (key) => parseGroupKey(key).slot; /** ColumnRef -> synthetic key */ -export const refToKey = (ref) => - ref.kind === "raw" ? ref.name : groupKey(ref.step, ref.slot ?? null); +export const refToKey = (ref) => { + if (ref.kind === "raw") return ref.name; + if (ref.name != null) return namedGroupKey(ref.step, ref.name); + return groupKey(ref.step, ref.slot ?? null); +}; /** synthetic key -> ColumnRef */ export const keyToRef = (key) => { if (!isGroupKey(key)) return { kind: "raw", name: key }; - const step = stepFromGroupKey(key); - const slot = slotFromGroupKey(key); + const { step, slot, name } = parseGroupKey(key); + if (name != null) return { kind: "group", step, name }; return slot == null ? { kind: "group", step } : { kind: "group", step, slot }; }; +/** + * The ColumnRef that points at an item of the estimated dataset structure + * (see the backend's infer_structure): an original column by name, a + * generated column by its step and name, a block by its step and slot (a + * lone block has no slot: it is its step's whole group). + */ +export const itemToRef = (item) => { + if (item.kind === "column") { + return item.origin == null + ? { kind: "raw", name: item.name } + : { kind: "group", step: item.origin, name: item.name }; + } + return item.slot == null + ? { kind: "group", step: item.step } + : { kind: "group", step: item.step, slot: item.slot }; +}; + +const blockLabel = (block, stepName) => { + const base = + block.label && block.label !== "output" + ? `${stepName}: ${block.label}` + : `${stepName}: output`; + // "N" when the column count is only known after fit. + return `${base} (${block.count ?? "N"})`; +}; + +/** + * Selector options for the items of an estimated dataset state: one key per + * item (the synthetic key of the ColumnRef pointing at it), its type, and a + * label for every item that is not an original column. Plugs into the same + * `{allKeys, columnTypes, optionLabels}` shape ColumnSelector and + * DivideDatasetColumns already take. + * + * @param {Array} state items from a StructureResult state + * @param {string[]} stepDisplayNames one name per step (see + * buildStepDisplayNames) + */ +export function stateToOptions(state, stepDisplayNames = []) { + const allKeys = []; + const columnTypes = {}; + const optionLabels = {}; + (state || []).forEach((item) => { + const key = refToKey(itemToRef(item)); + allKeys.push(key); + columnTypes[key] = { type: item.type ?? null, dtype: item.dtype ?? null }; + if (item.kind === "block") { + optionLabels[key] = blockLabel(item, stepDisplayNames[item.step] ?? ""); + } else if (item.origin != null) { + optionLabels[key] = item.name; + } + }); + return { allKeys, columnTypes, optionLabels }; +} + +/** + * A label for a ColumnRef without the estimated structure, for sessions + * that are already created (e.g. the session info panel). + */ +export function labelForRef(ref, stepDisplayNames = []) { + if (ref.kind === "raw") return ref.name; + if (ref.name != null) return ref.name; + const stepName = stepDisplayNames[ref.step] ?? `${ref.step}`; + return ref.slot == null + ? `${stepName}: output` + : `${stepName}: output (${ref.slot})`; +} + // A step's declared output, normalized to a list of slots — even a // homogeneous step (the common case) is one "slot" with slot: null, so // every caller iterates the same shape regardless of how many there are. diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js index 43aea6c70..eaf36cf26 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -9,6 +9,9 @@ import { buildStepDisplayNames, resolveDeclaredOutputSlots, rawColumnsNeededFor, + itemToRef, + stateToOptions, + labelForRef, } from "./sessionColumnRefs"; describe("sessionColumnRefs", () => { @@ -395,4 +398,125 @@ describe("sessionColumnRefs", () => { expect(rawColumnsNeededFor(refs, [])).toEqual(["age"]); }); }); + + describe("named group refs", () => { + it("round-trips a named group ColumnRef through a key", () => { + const ref = { kind: "group", step: 1, name: "date_month" }; + const key = refToKey(ref); + expect(isGroupKey(key)).toBe(true); + expect(stepFromGroupKey(key)).toBe(1); + expect(slotFromGroupKey(key)).toBeNull(); + expect(keyToRef(key)).toEqual(ref); + }); + + it("keeps a column name containing the separators intact", () => { + const ref = { kind: "group", step: 0, name: "a__slot__b" }; + expect(keyToRef(refToKey(ref))).toEqual(ref); + }); + }); + + describe("itemToRef", () => { + it("refs an original column by name", () => { + const item = { kind: "column", name: "age", origin: null }; + expect(itemToRef(item)).toEqual({ kind: "raw", name: "age" }); + }); + + it("refs a generated column by its step and name", () => { + const item = { kind: "column", name: "date_month", origin: 2 }; + expect(itemToRef(item)).toEqual({ + kind: "group", + step: 2, + name: "date_month", + }); + }); + + it("refs a lone block as its step's whole group", () => { + const item = { kind: "block", step: 0, slot: null }; + expect(itemToRef(item)).toEqual({ kind: "group", step: 0 }); + }); + + it("refs one of several blocks by its slot", () => { + const item = { kind: "block", step: 0, slot: "Float" }; + expect(itemToRef(item)).toEqual({ + kind: "group", + step: 0, + slot: "Float", + }); + }); + }); + + describe("stateToOptions", () => { + const state = [ + { + kind: "column", + name: "age", + type: "Integer", + dtype: "int64", + origin: null, + }, + { + kind: "column", + name: "date_month", + type: "Integer", + dtype: "int64", + origin: 1, + }, + { + kind: "block", + step: 0, + slot: null, + label: "output", + type: "Float", + dtype: "float64", + count: 2, + }, + { + kind: "block", + step: 2, + slot: null, + label: "ohe_*", + type: "Integer", + dtype: "int64", + count: null, + }, + ]; + const stepNames = ["PCA", "Date Features", "One Hot Encoder"]; + + it("keys every item as the ref it stands for, typed", () => { + const { allKeys, columnTypes } = stateToOptions(state, stepNames); + + expect(allKeys.map(keyToRef)).toEqual(state.map(itemToRef)); + expect(columnTypes.age).toEqual({ type: "Integer", dtype: "int64" }); + expect(columnTypes[allKeys[2]]).toEqual({ + type: "Float", + dtype: "float64", + }); + }); + + it("labels blocks with their step and size, N when unknown", () => { + const { allKeys, optionLabels } = stateToOptions(state, stepNames); + + expect(optionLabels[allKeys[0]]).toBeUndefined(); + expect(optionLabels[allKeys[1]]).toBe("date_month"); + expect(optionLabels[allKeys[2]]).toBe("PCA: output (2)"); + expect(optionLabels[allKeys[3]]).toBe("One Hot Encoder: ohe_* (N)"); + }); + }); + + describe("labelForRef", () => { + const stepNames = ["PCA", "Date Features"]; + + it("labels a ref without needing the estimated structure", () => { + expect(labelForRef({ kind: "raw", name: "age" }, stepNames)).toBe("age"); + expect( + labelForRef({ kind: "group", step: 1, name: "date_month" }, stepNames), + ).toBe("date_month"); + expect(labelForRef({ kind: "group", step: 0 }, stepNames)).toBe( + "PCA: output", + ); + expect( + labelForRef({ kind: "group", step: 0, slot: "Float" }, stepNames), + ).toBe("PCA: output (Float)"); + }); + }); }); diff --git a/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.js b/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.js new file mode 100644 index 000000000..1811fe832 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.js @@ -0,0 +1,78 @@ +import { useEffect, useRef, useState } from "react"; +import { getPreprocessingStructure } from "../../../api/modelSession"; + +const DEBOUNCE_MS = 300; + +/** + * The estimated dataset structure along a session's preprocessing chain, + * from the backend's infer_structure: the state after every step, their + * errors and warnings, and the final state. + * + * Refetched on every change to the inputs, debounced so quick edits send + * one request. Only the response to the latest inputs is kept: a slower + * response to an earlier edit never overwrites a newer one. + * + * @param {object} args + * @param {number} args.datasetId the session's dataset; nothing is fetched + * without one + * @param {string[]} args.candidates original columns fed into the chain + * @param {string[]} args.outputColumns the target columns + * @param {Array} args.steps the chain's `{converter, params, scope}` + * @returns {{structure: object|null, loading: boolean, error: Error|null}} + */ +export default function usePreprocessingStructure({ + datasetId, + candidates, + outputColumns, + steps, +}) { + const [structure, setStructure] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const latestRequest = useRef(0); + + // A stable value to depend on: the caller may rebuild these arrays on + // every render, and a step may carry extra UI-only fields. + const payloadKey = JSON.stringify({ + datasetId, + candidates: candidates || [], + outputColumns: outputColumns || [], + steps: (steps || []).map(({ converter, params, scope }) => ({ + converter, + params, + scope, + })), + }); + + useEffect(() => { + const payload = JSON.parse(payloadKey); + const requestId = latestRequest.current + 1; + latestRequest.current = requestId; + if (!payload.datasetId) { + setStructure(null); + setLoading(false); + return undefined; + } + setLoading(true); + const timer = setTimeout(async () => { + try { + const result = await getPreprocessingStructure(payload); + if (requestId === latestRequest.current) { + setStructure(result); + setError(null); + } + } catch (requestError) { + if (requestId === latestRequest.current) { + setError(requestError); + } + } finally { + if (requestId === latestRequest.current) { + setLoading(false); + } + } + }, DEBOUNCE_MS); + return () => clearTimeout(timer); + }, [payloadKey]); + + return { structure, loading, error }; +} diff --git a/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.test.js b/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.test.js new file mode 100644 index 000000000..4708233c6 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.test.js @@ -0,0 +1,80 @@ +import { act, renderHook } from "@testing-library/react"; +import usePreprocessingStructure from "./usePreprocessingStructure"; +import { getPreprocessingStructure } from "../../../api/modelSession"; + +jest.mock("../../../api/modelSession", () => ({ + getPreprocessingStructure: jest.fn(), +})); + +const deferred = () => { + let resolve; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +}; + +const args = (steps) => ({ + datasetId: 1, + candidates: ["age"], + outputColumns: ["label"], + steps, +}); + +describe("usePreprocessingStructure", () => { + beforeEach(() => { + jest.useFakeTimers(); + getPreprocessingStructure.mockReset(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it("sends one request for quick successive edits", async () => { + getPreprocessingStructure.mockResolvedValue({ valid: true }); + const { rerender } = renderHook( + (props) => usePreprocessingStructure(props), + { + initialProps: args([]), + }, + ); + + rerender(args([{ converter: "PCA", params: {}, scope: [] }])); + await act(async () => { + jest.advanceTimersByTime(300); + }); + + expect(getPreprocessingStructure).toHaveBeenCalledTimes(1); + expect(getPreprocessingStructure.mock.calls[0][0].steps).toHaveLength(1); + }); + + it("never lets a slower, older response overwrite a newer one", async () => { + const older = deferred(); + const newer = deferred(); + getPreprocessingStructure + .mockReturnValueOnce(older.promise) + .mockReturnValueOnce(newer.promise); + const { result, rerender } = renderHook( + (props) => usePreprocessingStructure(props), + { initialProps: args([]) }, + ); + + await act(async () => { + jest.advanceTimersByTime(300); + }); + rerender(args([{ converter: "PCA", params: {}, scope: [] }])); + await act(async () => { + jest.advanceTimersByTime(300); + }); + await act(async () => { + newer.resolve({ id: "newer" }); + }); + await act(async () => { + older.resolve({ id: "older" }); + }); + + expect(result.current.structure).toEqual({ id: "newer" }); + expect(result.current.loading).toBe(false); + }); +}); diff --git a/DashAI/front/src/types/modelSession.ts b/DashAI/front/src/types/modelSession.ts index 0048d86b2..f944f39f8 100644 --- a/DashAI/front/src/types/modelSession.ts +++ b/DashAI/front/src/types/modelSession.ts @@ -9,6 +9,10 @@ export interface IRawColumnRef { export interface IGroupColumnRef { kind: "group"; step: number; + // One type's columns of the step's output (mutually exclusive with name). + slot?: string | null; + // One column of the step's output whose name is known before fit. + name?: string | null; } export type IColumnRef = IRawColumnRef | IGroupColumnRef; @@ -19,6 +23,50 @@ export interface IConverterStep { scope: IColumnRef[]; } +// Estimated dataset structure along a session's converter chain, as returned +// by the backend's infer_structure (DashAI/back/preprocessing/structure.py). +export interface IColumnItem { + kind: "column"; + name: string; + type: string | null; + dtype: string | null; + // null for an original dataset column; otherwise the step that created it. + origin: number | null; +} + +export interface IBlockItem { + kind: "block"; + step: number; + slot: string | null; + label: string; + type: string | null; + dtype: string | null; + // null when the column count is only known after fit. + count: number | null; +} + +export type IStateItem = IColumnItem | IBlockItem; + +export interface IStructureMessage { + code: string; + params: Record; +} + +export interface IStepStructure { + status: "ok" | "error" | "blocked"; + state: IStateItem[]; + added: IStateItem[]; + error: IStructureMessage | null; + warnings: IStructureMessage[]; +} + +export interface IStructureResult { + initial: IStateItem[]; + steps: IStepStructure[]; + final: IStateItem[]; + valid: boolean; +} + export type IPreprocessingStatus = "ready" | "pending" | "failed"; export interface IModelSession { From 18f296496a01fd479dddd71ca7df376eb76a5f3c Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 12:01:27 +0200 Subject: [PATCH 36/39] feat: add BaseColumnsStep component and integrate it into session creation workflow --- .../components/models/CreateSessionSteps.jsx | 33 ++++++- .../models/modelSession/BaseColumnsStep.jsx | 96 +++++++++++++++++++ .../modelSession/DivideDatasetColumns.jsx | 5 +- .../src/utils/i18n/locales/de/models.json | 3 + .../src/utils/i18n/locales/en/models.json | 3 + .../src/utils/i18n/locales/es/models.json | 3 + .../src/utils/i18n/locales/pt/models.json | 3 + .../src/utils/i18n/locales/zh/models.json | 3 + 8 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 DashAI/front/src/components/models/modelSession/BaseColumnsStep.jsx diff --git a/DashAI/front/src/components/models/CreateSessionSteps.jsx b/DashAI/front/src/components/models/CreateSessionSteps.jsx index 05d5a0da9..aeb20ae5e 100644 --- a/DashAI/front/src/components/models/CreateSessionSteps.jsx +++ b/DashAI/front/src/components/models/CreateSessionSteps.jsx @@ -6,6 +6,7 @@ import { useFormik } from "formik"; import { useTourContext } from "../tour/TourProvider"; import SetNameAndDatasetStep from "./SetNameAndDatasetStep"; import PrepareDatasetStep from "./modelSession/PrepareDatasetStep"; +import BaseColumnsStep from "./modelSession/BaseColumnsStep"; import PreprocessingStep from "./modelSession/PreprocessingStep"; import SelectColumnsStep from "./modelSession/SelectColumnsStep"; import DatasetAutocomplete from "../notebooks/notebookCreation/DatasetAutocomplete"; @@ -26,6 +27,7 @@ import StepperNavigationFooter from "../shared/StepperNavigationFooter"; import { hasPartition } from "../../utils/splitsPayload"; const STEP_PREPARE_DATASET = "prepareDataset"; +const STEP_BASE_COLUMNS = "baseColumns"; const STEP_PREPROCESSING = "preprocessing"; const STEP_SELECT_COLUMNS = "selectColumns"; @@ -66,6 +68,9 @@ function CreateSessionSteps({ splits: {}, runs: [], applyPreprocessing: false, + // Original columns the preprocessing chain may work on (see + // BaseColumnsStep); only used when applyPreprocessing is on. + candidate_columns: [], preprocessing: [], input_column_refs: [], }); @@ -111,7 +116,12 @@ function CreateSessionSteps({ const steps = useMemo( () => [ STEP_PREPARE_DATASET, - ...(newExp.applyPreprocessing ? [STEP_PREPROCESSING] : []), + // With preprocessing, the output and candidate columns come first, so + // the chain is built knowing what it is for and never touches the + // output. + ...(newExp.applyPreprocessing + ? [STEP_BASE_COLUMNS, STEP_PREPROCESSING] + : []), STEP_SELECT_COLUMNS, ], [newExp.applyPreprocessing], @@ -128,6 +138,7 @@ function CreateSessionSteps({ dataset: newDataset, input_columns: [], output_columns: [], + candidate_columns: [], input_column_refs: [], splits: {}, })); @@ -324,12 +335,14 @@ function CreateSessionSteps({ const stepTitle = { [STEP_PREPARE_DATASET]: t("models:label.prepareDataset"), + [STEP_BASE_COLUMNS]: t("models:label.baseColumnsTitle"), [STEP_PREPROCESSING]: t("models:label.preprocessingOptional"), [STEP_SELECT_COLUMNS]: t("models:label.selectColumnsTitle"), }[currentStep]; const stepSubtitle = { [STEP_PREPARE_DATASET]: t("models:label.selectDatasetAndPrepare"), + [STEP_BASE_COLUMNS]: t("models:label.baseColumnsDescription"), [STEP_PREPROCESSING]: t("models:label.preprocessingOptionalDescription"), [STEP_SELECT_COLUMNS]: t("models:label.selectColumnsDescription"), }[currentStep]; @@ -387,6 +400,24 @@ function CreateSessionSteps({ )} + {currentStep === STEP_BASE_COLUMNS && selectedDataset && ( + <> + {infoLoading ? ( + + + + ) : ( + + )} + + )} + {currentStep === STEP_PREPROCESSING && selectedDataset && ( !outputColumnNames.includes(name), + ); + const candidates = candidateSelection.filter( + (name) => !outputColumnNames.includes(name), + ); + + useLayoutEffect(() => { + // Default the first time the dataset's columns load: the last column is + // the output and every other one is a candidate. + if (rawColumnNames.length === 0) return; + if (outputColumnNames.length === 0 && candidateSelection.length === 0) { + const output = rawColumnNames[rawColumnNames.length - 1]; + setOutputColumnNames([output]); + setCandidateSelection(rawColumnNames.filter((name) => name !== output)); + } + }, [rawColumnNames.join(",")]); + + const ready = candidates.length >= 1 && outputColumnNames.length >= 1; + + useEffect(() => { + setNewExp((prev) => ({ + ...prev, + candidate_columns: candidates, + output_columns: outputColumnNames, + })); + setNextEnabled(ready); + }, [candidates.join(","), outputColumnNames.join(",")]); + + return ( + + + + ); +} + +BaseColumnsStep.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + setNextEnabled: PropTypes.func.isRequired, + datasetInfo: PropTypes.object, + datasetTypes: PropTypes.object, +}; + +export default BaseColumnsStep; diff --git a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx index 1508d1dc5..f3ad8bebd 100644 --- a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx +++ b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx @@ -26,6 +26,7 @@ function DivideDatasetColumns({ outputError = false, outputHelperText = "", disabled = false, + inputLabel, }) { const { t } = useTranslation(["experiments", "common", "models"]); const theme = useTheme(); @@ -145,7 +146,7 @@ function DivideDatasetColumns({ Date: Mon, 28 Sep 2026 12:09:00 +0200 Subject: [PATCH 37/39] Refactor session preprocessing components to improve structure estimation - Updated FormSessionConverterSection to utilize final state and step display names for converter scoping. - Enhanced PreprocessingStep to integrate preprocessing structure estimation and error handling. - Modified ScopeStepSessionConverter to derive column options from the estimated dataset state. - Revamped SessionConvertersRightBar to align with new structure handling and converter validation. - Removed obsolete resolveDeclaredOutputSlots function and related tests. - Introduced structureMessages for user-friendly error messages based on backend responses. - Added translations for new structure-related messages in multiple languages. --- .../components/models/CreateSessionSteps.jsx | 1 - .../modelSession/AppliedConvertersView.jsx | 234 ++++++++++-------- .../AppliedConvertersView.test.jsx | 177 +++++++++---- .../FormSessionConverterSection.jsx | 35 ++- .../models/modelSession/PreprocessingStep.jsx | 58 +++-- .../ScopeStepSessionConverter.jsx | 36 ++- .../SessionConvertersRightBar.jsx | 74 +++--- .../models/modelSession/sessionColumnRefs.js | 87 ------- .../modelSession/sessionColumnRefs.test.js | 136 ---------- .../models/modelSession/structureMessages.js | 27 ++ .../src/utils/i18n/locales/de/models.json | 20 ++ .../src/utils/i18n/locales/en/models.json | 20 ++ .../src/utils/i18n/locales/es/models.json | 20 ++ .../src/utils/i18n/locales/pt/models.json | 20 ++ .../src/utils/i18n/locales/zh/models.json | 20 ++ 15 files changed, 493 insertions(+), 472 deletions(-) create mode 100644 DashAI/front/src/components/models/modelSession/structureMessages.js diff --git a/DashAI/front/src/components/models/CreateSessionSteps.jsx b/DashAI/front/src/components/models/CreateSessionSteps.jsx index aeb20ae5e..cf4bb6c5a 100644 --- a/DashAI/front/src/components/models/CreateSessionSteps.jsx +++ b/DashAI/front/src/components/models/CreateSessionSteps.jsx @@ -424,7 +424,6 @@ function CreateSessionSteps({ setNewExp={setNewExp} setNextEnabled={setNextEnabled} dataset={selectedDataset} - datasetTypes={datasetTypes} /> )} diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx index 266abc4ce..533acecea 100644 --- a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx @@ -1,6 +1,9 @@ import React, { useEffect, useMemo, useState } from "react"; import PropTypes from "prop-types"; import { + Accordion, + AccordionDetails, + AccordionSummary, Box, Paper, Typography, @@ -13,6 +16,7 @@ import { useMaterialReactTable, } from "material-react-table"; import DeleteIcon from "@mui/icons-material/Delete"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; import Transform from "@mui/icons-material/Transform"; import { useTheme } from "@mui/material/styles"; import { useTranslation } from "react-i18next"; @@ -23,11 +27,11 @@ import DeleteConfirmationModal from "../../threeSectionLayout/DeleteConfirmation import ItemsToDeleteList from "../../notebooks/converter/ItemsToDeleteList"; import { useExplorersAndConverters } from "../../notebooks/context/ExplorersAndConvertersContext"; import { - buildColumnKeysAndTypes, buildStepDisplayNames, - groupKey, - refToKey, + labelForRef, + stateToOptions, } from "./sessionColumnRefs"; +import { formatStructureMessage } from "./structureMessages"; function TypeChip({ type }) { if (!type) return null; @@ -49,14 +53,14 @@ function TypeChip({ type }) { TypeChip.propTypes = { type: PropTypes.string }; -function RefChip({ refKey, label, columnTypes }) { +function RefChip({ label, type }) { return ( {label} - + } /> @@ -64,23 +68,49 @@ function RefChip({ refKey, label, columnTypes }) { } RefChip.propTypes = { - refKey: PropTypes.string.isRequired, label: PropTypes.string.isRequired, - columnTypes: PropTypes.object.isRequired, + type: PropTypes.string, +}; + +/** + * One chip per item of an estimated dataset state: an original column shows + * its name, anything a step produced shows its label (a block also shows its + * column count, "N" when only known after fit). + */ +function StateChips({ items, stepDisplayNames }) { + const { allKeys, columnTypes, optionLabels } = stateToOptions( + items, + stepDisplayNames, + ); + return ( + + {allKeys.map((key) => ( + + ))} + + ); +} + +StateChips.propTypes = { + items: PropTypes.array.isRequired, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, }; /** * Mirrors the notebook's own ConverterParametersTable (same columns, same * MaterialReactTable setup): a compact two-column table with one row per - * converter attribute, values rendered as wrapped chips (each with a - * colored type badge) instead of plain text, since a session converter's - * scope/output are references to columns or groups, not literal strings. + * converter attribute. The scope is plain text, like Notebooks shows an + * applied converter's scope; the output is one chip per column or block the + * step produces, each with its type. */ function SessionConverterParametersTable({ step, - optionLabels, - columnTypes, - outputEntries, + added, + stepDisplayNames, t, localization, }) { @@ -91,34 +121,14 @@ function SessionConverterParametersTable({ const paramRows = [ { - // Plain comma-joined text, matching the notebook's own - // ConverterParametersTable (ConverterBox.jsx) — no chips here, since - // that's how Notebooks shows an already-applied converter's scope. key: t("datasets:label.scopeColumns"), value: step.scope - .map((ref) => { - const key = refToKey(ref); - return optionLabels[key] || key; - }) + .map((ref) => labelForRef(ref, stepDisplayNames)) .join(", "), }, { - // Usually one chip; more than one when this step's scope mixed - // column types (e.g. SimpleImputer preserving both a categorical and - // a numeric column), so each declared slot gets its own chip. key: t("datasets:label.converterOutput"), - value: ( - - {outputEntries.map(({ key, label }) => ( - - ))} - - ), + value: , }, ]; @@ -142,14 +152,8 @@ function SessionConverterParametersTable({ SessionConverterParametersTable.propTypes = { step: PropTypes.object.isRequired, - optionLabels: PropTypes.object.isRequired, - columnTypes: PropTypes.object.isRequired, - outputEntries: PropTypes.arrayOf( - PropTypes.shape({ - key: PropTypes.string.isRequired, - label: PropTypes.string.isRequired, - }), - ).isRequired, + added: PropTypes.array.isRequired, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, t: PropTypes.func.isRequired, localization: PropTypes.object.isRequired, }; @@ -157,12 +161,11 @@ SessionConverterParametersTable.propTypes = { /** * A single applied-converter card, styled after the notebook's own * ConverterBox (icon + real component display name + description + - * parameters table), but built against the session's preprocessing step - * shape (`{converter, scope, outputSlots}`) instead of the notebook's - * (`{parameters: {scope, target}}`), and with a "Salida" row showing the - * converter's declared output type(s) — usually one chip, more than one - * when the step's scope mixed column types — since that group doesn't - * exist as a real column until the session is created. + * parameters table), built against the session's preprocessing step shape + * (`{converter, params, scope}`). Its output and status come from the + * chain's estimated structure: a step that cannot work is outlined in red + * with the reason, a step after it is dimmed (it cannot be checked until + * the earlier one is fixed), and warnings are shown under the table. */ function SessionConverterCard({ step, @@ -170,54 +173,29 @@ function SessionConverterCard({ displayName, description, onDelete, - datasetTypes, - preprocessing, + stepStructure, stepDisplayNames, }) { const theme = useTheme(); const { t } = useTranslation(["datasets", "models", "common"]); const localization = useTableLocalization(); - - const { columnTypes } = buildColumnKeysAndTypes({ - datasetTypes, - preprocessing, - }); - // Every earlier step's group label(s) use its resolved display name (e.g. - // "Bag of Words: output"), matching this card's own output label(s) - // below — not the raw registry name buildColumnKeysAndTypes falls back - // to when it has no display-name lookup of its own. One label per - // declared slot, so a step whose scope mixed types (more than one slot) - // gets one distinguishable label per slot. - const optionLabels = {}; - preprocessing.forEach((s, i) => { - const name = stepDisplayNames[i]; - const slots = s.outputSlots?.length > 0 ? s.outputSlots : [{ slot: null }]; - // No "(slot)" suffix here: RefChip already shows the slot's type as its - // own colored badge right next to this label, so repeating it as text - // would be redundant (a step with several slots gets several chips - // with identical text but different badges, which is enough to tell - // them apart). - slots.forEach(({ slot }) => { - const key = groupKey(i, slot); - optionLabels[key] = `${name}: output`; - }); - }); - const ownSlots = - step.outputSlots?.length > 0 ? step.outputSlots : [{ slot: null }]; - const outputEntries = ownSlots.map(({ slot }) => { - const key = groupKey(index, slot); - return { key, label: optionLabels[key] || `${displayName}: output` }; - }); + const status = stepStructure?.status; return ( + {status === "error" && ( + + {formatStructureMessage(t, stepStructure.error)} + + )} + {status === "blocked" && ( + + {t("models:structure.blocked")} + + )} + {(stepStructure?.warnings || []).map((warning, i) => ( + + {formatStructureMessage(t, warning)} + + ))} ); @@ -274,26 +271,51 @@ SessionConverterCard.propTypes = { displayName: PropTypes.string.isRequired, description: PropTypes.string, onDelete: PropTypes.func.isRequired, - datasetTypes: PropTypes.object, - preprocessing: PropTypes.array, + stepStructure: PropTypes.object, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, +}; + +/** + * The estimated dataset state at the end of the chain: what the model + * inputs will be picked from in the next step. + */ +function DatasetStatePanel({ finalState, stepDisplayNames }) { + const { t } = useTranslation(["models"]); + return ( + + }> + + {t("models:structure.currentState")} + + + + + + + ); +} + +DatasetStatePanel.propTypes = { + finalState: PropTypes.array.isRequired, stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, }; /** * Cards for every converter already added to the session's preprocessing - * sequence, mirroring the notebook module's applied-tool cards. + * sequence, mirroring the notebook module's applied-tool cards, under a + * panel with the estimated dataset state after the whole chain. * - * Deleting a converter cascades: any later converter whose scope - * references its output group would be left pointing at a step that no - * longer exists, so every converter configured after it is removed too. - * Confirmed first via the same `DeleteConfirmationModal` + - * `ItemsToDeleteList` the notebook's own converter deletion uses, so the - * user sees exactly what else is about to go before confirming. + * Deleting a converter cascades: any later converter could reference what + * it produces and would be left pointing at a step that no longer exists, + * so every converter configured after it is removed too. Confirmed first + * via the same `DeleteConfirmationModal` + `ItemsToDeleteList` the + * notebook's own converter deletion uses, so the user sees exactly what + * else is about to go before confirming. */ export default function AppliedConvertersView({ newExp, setNewExp, - datasetTypes, + structure, }) { const { t } = useTranslation(["experiments", "models", "datasets", "common"]); const theme = useTheme(); @@ -307,7 +329,7 @@ export default function AppliedConvertersView({ // A converter is draggable from SessionConvertersRightBar's ToolList/ // ToolGrid for free (drag-start lives in the shared ToolListItem/ // ToolGridItem, not in those wrappers), so this view only needs to be a - // drop target — same "application/x-dashai-tool" payload and the same + // drop target: same "application/x-dashai-tool" payload and the same // setPendingDropTool hand-off Notebooks' own NotebookView.jsx uses, which // ToolList/ToolGrid already resolve through their normal click-to-add path. useEffect(() => { @@ -380,7 +402,7 @@ export default function AppliedConvertersView({ // Numbers duplicate converter types ("Simple Imputer" / "Simple Imputer // (2)") so two steps of the same type never render with identical names - // — see buildStepDisplayNames. + // (see buildStepDisplayNames). const stepDisplayNames = buildStepDisplayNames(steps, convertersMeta); const itemsToDelete = useMemo(() => { @@ -408,7 +430,7 @@ export default function AppliedConvertersView({ onDrop={handleDrop} sx={{ position: "relative", - // A percentage minHeight is unreliable here — this Box is a flex + // A percentage minHeight is unreliable here: this Box is a flex // item inside CreateSessionSteps' flex-column scroll container, and // percentage heights don't resolve dependably through it. flex: 1 // makes it actually fill the remaining vertical space (with content @@ -456,6 +478,13 @@ export default function AppliedConvertersView({ )} + {structure && ( + + )} + {steps.length === 0 ? ( {t("experiments:label.noConverterAdded")} @@ -473,8 +502,7 @@ export default function AppliedConvertersView({ meta?.description || meta?.metadata?.short_description } onDelete={(i) => setDeleteIndex(i)} - datasetTypes={datasetTypes} - preprocessing={steps} + stepStructure={structure?.steps?.[index]} stepDisplayNames={stepDisplayNames} /> ); @@ -503,5 +531,7 @@ export default function AppliedConvertersView({ AppliedConvertersView.propTypes = { newExp: PropTypes.object.isRequired, setNewExp: PropTypes.func.isRequired, - datasetTypes: PropTypes.object, + // The chain's estimated structure (see usePreprocessingStructure); null + // until the first estimate arrives. + structure: PropTypes.object, }; diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx index 6b8f360bf..71cd18d81 100644 --- a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx @@ -38,7 +38,33 @@ const AVAILABLE_CONVERTERS = [ }, ]; -const datasetTypes = { age: { type: "Integer" }, text: { type: "Text" } }; +const column = (name, type, origin) => ({ + kind: "column", + name, + type, + dtype: null, + origin, +}); + +const block = (step, slot, type, count) => ({ + kind: "block", + step, + slot, + label: "output", + type, + dtype: null, + count, +}); + +const okStep = (added) => ({ + status: "ok", + state: [], + added, + error: null, + warnings: [], +}); + +const structureOf = (steps) => ({ initial: [], steps, final: [], valid: true }); describe("AppliedConvertersView", () => { let setPendingDropTool; @@ -53,117 +79,176 @@ describe("AppliedConvertersView", () => { renderView({ newExp: { preprocessing: [] }, setNewExp: () => {}, - datasetTypes, }); expect(screen.getByText("No converter was added.")).toBeInTheDocument(); }); - it("shows a converter's real display name, scope and output type chip", async () => { + it("shows a converter's real display name, scope and estimated output", async () => { const newExp = { preprocessing: [ { converter: "Binarizer", params: { threshold: 0.5 }, scope: [{ kind: "raw", name: "age" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; + const structure = structureOf([okStep([column("bin_age", "Integer", 0)])]); - renderView({ newExp, setNewExp: () => {}, datasetTypes }); + renderView({ newExp, setNewExp: () => {}, structure }); expect(await screen.findByText("Binarizador")).toBeInTheDocument(); expect(screen.getByText("age")).toBeInTheDocument(); - expect(screen.getByText("Binarizador: output")).toBeInTheDocument(); + expect(screen.getAllByText("bin_age").length).toBeGreaterThan(0); expect(screen.getAllByText("Integer").length).toBeGreaterThan(0); }); - it("shows a chained converter's scope as the earlier converter's output group", async () => { + it("shows a chained converter's scope as the earlier converter's output", async () => { const newExp = { preprocessing: [ { converter: "BagOfWordsConverter", params: {}, scope: [{ kind: "raw", name: "text" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, { converter: "Binarizer", params: {}, scope: [{ kind: "group", step: 0 }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; + const structure = structureOf([ + okStep([block(0, null, "Integer", null)]), + okStep([block(1, null, "Integer", null)]), + ]); - renderView({ newExp, setNewExp: () => {}, datasetTypes }); + renderView({ newExp, setNewExp: () => {}, structure }); await screen.findByText("Bag of Words"); - // The second card's scope shows the first converter's output label. - expect(screen.getAllByText("Bag of Words: output").length).toBe(2); + // The second card's scope names the first converter's output. + expect(screen.getByText("Bag of Words: output")).toBeInTheDocument(); + // Each card's output block shows its size, N when unknown. + expect(screen.getAllByText("Bag of Words: output (N)").length).toBe(1); }); - it("shows one chip per declared slot when a step's scope mixed column types", async () => { + it("shows one chip per estimated block when a step's output mixes types", async () => { const newExp = { preprocessing: [ { - converter: "SimpleImputer", - params: { strategy: "most_frequent" }, + converter: "SelectKBest", + params: { k: 1 }, scope: [ { kind: "raw", name: "age" }, - { kind: "raw", name: "text" }, - ], - outputSlots: [ - { slot: "Integer", type: "Integer", dtype: "int64" }, - { slot: "Categorical", type: "Categorical", dtype: null }, + { kind: "raw", name: "score" }, ], }, ], }; - - renderView({ - newExp, - setNewExp: () => {}, - datasetTypes: { - ...datasetTypes, - // "SimpleImputer" isn't in AVAILABLE_CONVERTERS, so its display - // name falls back to the raw converter name — fine here, this - // test only cares about the output chips. - }, - }); - - // No "(slot)" suffix: each slot already gets its own colored type chip - // (asserted below), so both chips share the same label text. - expect(await screen.findAllByText("SimpleImputer: output")).toHaveLength(2); - expect(screen.getByText("Integer")).toBeInTheDocument(); - expect(screen.getByText("Categorical")).toBeInTheDocument(); + const structure = structureOf([ + okStep([ + block(0, "Integer", "Integer", null), + block(0, "Float", "Float", null), + ]), + ]); + + renderView({ newExp, setNewExp: () => {}, structure }); + + expect(await screen.findAllByText("SelectKBest: output (N)")).toHaveLength( + 2, + ); + expect(screen.getByText("Float")).toBeInTheDocument(); }); - it("numbers two converters of the same type so their cards and output labels don't collide", async () => { + it("numbers two converters of the same type so their cards don't collide", async () => { const newExp = { preprocessing: [ { converter: "Binarizer", params: { threshold: 0.5 }, scope: [{ kind: "raw", name: "age" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, { converter: "Binarizer", params: { threshold: 1 }, scope: [{ kind: "raw", name: "text" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; - renderView({ newExp, setNewExp: () => {}, datasetTypes }); + renderView({ newExp, setNewExp: () => {}, structure: null }); expect(await screen.findByText("Binarizador")).toBeInTheDocument(); expect(screen.getByText("Binarizador (2)")).toBeInTheDocument(); - expect(screen.getByText("Binarizador: output")).toBeInTheDocument(); - expect(screen.getByText("Binarizador (2): output")).toBeInTheDocument(); + }); + + it("flags a step that cannot work and dims the ones after it", async () => { + const newExp = { + preprocessing: [ + { + converter: "Binarizer", + params: {}, + scope: [{ kind: "raw", name: "age" }], + }, + { + converter: "Binarizer", + params: {}, + scope: [{ kind: "raw", name: "text" }], + }, + ], + }; + const structure = { + ...structureOf([]), + valid: false, + steps: [ + { + status: "error", + state: [], + added: [], + error: { code: "missing_ref", params: { ref: "age" } }, + warnings: [], + }, + { status: "blocked", state: [], added: [], error: null, warnings: [] }, + ], + }; + + renderView({ newExp, setNewExp: () => {}, structure }); + + expect( + await screen.findByText( + '"age" does not exist at this point of the chain: an earlier step may have consumed it.', + ), + ).toBeInTheDocument(); + expect(screen.getByTestId("session-converter-card-0")).toHaveAttribute( + "data-status", + "error", + ); + expect(screen.getByTestId("session-converter-card-1")).toHaveAttribute( + "data-status", + "blocked", + ); + expect( + screen.getByText("Fix the previous step to check this one."), + ).toBeInTheDocument(); + }); + + it("shows the dataset state after the whole chain", async () => { + const structure = { + ...structureOf([]), + final: [column("age", "Integer", null), block(0, null, "Float", 2)], + }; + + renderView({ + newExp: { preprocessing: [] }, + setNewExp: () => {}, + structure, + }); + + expect( + await screen.findByText("Dataset after preprocessing"), + ).toBeInTheDocument(); + expect(screen.getByText("age")).toBeInTheDocument(); }); it("cascades deletion to every converter configured after the deleted one", async () => { @@ -174,18 +259,16 @@ describe("AppliedConvertersView", () => { converter: "BagOfWordsConverter", params: {}, scope: [{ kind: "raw", name: "text" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, { converter: "Binarizer", params: {}, scope: [{ kind: "group", step: 0 }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; - renderView({ newExp, setNewExp, datasetTypes }); + renderView({ newExp, setNewExp, structure: null }); await screen.findByText("Bag of Words"); const deleteButtons = screen.getAllByRole("button", { name: "Remove" }); @@ -211,7 +294,6 @@ describe("AppliedConvertersView", () => { renderView({ newExp: { preprocessing: [] }, setNewExp: () => {}, - datasetTypes, }); const dropZone = screen.getByText("No converter was added.").parentElement; @@ -225,7 +307,6 @@ describe("AppliedConvertersView", () => { renderView({ newExp: { preprocessing: [] }, setNewExp: () => {}, - datasetTypes, }); const dropZone = screen.getByText("No converter was added.").parentElement; diff --git a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx index d6520bdc0..46dcfaf32 100644 --- a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx +++ b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx @@ -3,16 +3,16 @@ import PropTypes from "prop-types"; import { Box } from "@mui/material"; import ScopeStepSessionConverter from "./ScopeStepSessionConverter"; import ParameterStepConverter from "../../notebooks/converterCreation/ParameterStepConverter"; -import { resolveDeclaredOutputSlots } from "./sessionColumnRefs"; /** - * "Add a converter" form for the session wizard's preprocessing step — - * mirrors the notebook's FormConverterSection (same two-step scope/params - * flow, same ParameterStepConverter for the second step), but a session + * "Add a converter" form for the session wizard's preprocessing step, + * mirroring the notebook's FormConverterSection (same two-step scope/params + * flow, same ParameterStepConverter for the second step). A session * converter is never fit immediately: saving here only appends * `{converter, params, scope}` to the session's local `preprocessing` list. - * The actual fit happens once, later, when the session is created (see the - * backend's PreprocessingJob) — never from this form. + * What the step produces is estimated by the backend afterwards (see + * usePreprocessingStructure), and the actual fit happens once, when the + * session is created (see the backend's PreprocessingJob). */ export default function FormSessionConverterSection({ step, @@ -21,26 +21,18 @@ export default function FormSessionConverterSection({ tool, newExp, setNewExp, - datasetTypes, + finalState, + stepDisplayNames, filePath, hideButtons = false, - convertersMeta, }) { const [scope, setScope] = useState([]); const handleSaveConverter = async (params) => { - const outputSlots = resolveDeclaredOutputSlots({ - tool, - params, - scope, - datasetTypes, - preprocessing: newExp.preprocessing, - }); const newStep = { converter: tool.name, params: params || {}, scope, - outputSlots, }; setNewExp({ ...newExp, @@ -63,12 +55,11 @@ export default function FormSessionConverterSection({ {step === 0 && ( 0 ? () => setStep((s) => s + 1) @@ -99,8 +90,10 @@ FormSessionConverterSection.propTypes = { tool: PropTypes.object.isRequired, newExp: PropTypes.object.isRequired, setNewExp: PropTypes.func.isRequired, - datasetTypes: PropTypes.object.isRequired, + // The estimated dataset state at the end of the chain, where the new + // converter will be appended. + finalState: PropTypes.array.isRequired, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, filePath: PropTypes.string, hideButtons: PropTypes.bool, - convertersMeta: PropTypes.object, }; diff --git a/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx b/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx index 41a6d2051..d9f05e5c9 100644 --- a/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx +++ b/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx @@ -1,33 +1,41 @@ import React, { useEffect } from "react"; import PropTypes from "prop-types"; +import { Alert } from "@mui/material"; +import { useTranslation } from "react-i18next"; import { useModels } from "../ModelsContext"; import SessionConvertersRightBar from "./SessionConvertersRightBar"; import AppliedConvertersView from "./AppliedConvertersView"; +import usePreprocessingStructure from "./usePreprocessingStructure"; /** - * Optional preprocessing step of the session wizard: pick converters to fit - * on training data only (see the backend's PreprocessingJob), and configure - * each one's scope. A converter's scope may include the not-yet-materialized - * output group of an earlier converter in the same list, which is how - * chaining works before any real fit exists. + * Optional preprocessing step of the session wizard: build a chain of + * converters, fit on training data only (see the backend's + * PreprocessingJob), over the candidate columns picked in BaseColumnsStep. + * + * The dataset structure along the chain is estimated by the backend + * (usePreprocessingStructure) on every change: a new converter can only be + * scoped on columns that exist at the end of the chain, each card shows + * what its step produces, and a step that cannot work is flagged and blocks + * moving on. * * Styled like the notebook module's own converter picker: the catalog * (search + category list/grid) lives in the right bar * (SessionConvertersRightBar), the already-added converters are shown as * cards in the main content area (AppliedConvertersView). */ -function PreprocessingStep({ - newExp, - setNewExp, - setNextEnabled, - dataset, - datasetTypes, -}) { +function PreprocessingStep({ newExp, setNewExp, setNextEnabled, dataset }) { + const { t } = useTranslation(["models"]); const { setSessionRightContent } = useModels(); + const { structure, loading, error } = usePreprocessingStructure({ + datasetId: dataset?.id, + candidates: newExp.candidate_columns, + outputColumns: newExp.output_columns, + steps: newExp.preprocessing, + }); useEffect(() => { - setNextEnabled(true); - }, [setNextEnabled]); + setNextEnabled(Boolean(structure?.valid) && !loading && !error); + }, [structure, loading, error, setNextEnabled]); useEffect(() => { setSessionRightContent( @@ -35,18 +43,25 @@ function PreprocessingStep({ newExp={newExp} setNewExp={setNewExp} dataset={dataset} - datasetTypes={datasetTypes} + structure={structure} />, ); return () => setSessionRightContent(null); - }, [newExp, setNewExp, dataset, datasetTypes]); + }, [newExp, setNewExp, dataset, structure]); return ( - + <> + {error && ( + + {t("models:structure.loadError")} + + )} + + ); } @@ -55,7 +70,6 @@ PreprocessingStep.propTypes = { setNewExp: PropTypes.func.isRequired, setNextEnabled: PropTypes.func.isRequired, dataset: PropTypes.object, - datasetTypes: PropTypes.object, }; export default PreprocessingStep; diff --git a/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx index e2363e960..989d12d3d 100644 --- a/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx +++ b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx @@ -5,32 +5,32 @@ import { useTheme } from "@mui/material/styles"; import { useTranslation } from "react-i18next"; import ColumnSelector from "../../notebooks/ColumnSelector"; import FormSchemaButtonGroup from "../../shared/FormSchemaButtonGroup"; -import { buildColumnKeysAndTypes, keyToRef } from "./sessionColumnRefs"; +import { keyToRef, stateToOptions } from "./sessionColumnRefs"; /** * Scope step for a converter being added to a session's preprocessing - * sequence: column selection only (there's no row-level scope — the + * sequence: column selection only. There's no row-level scope: the * train/test split doesn't exist yet at config time, and a session * converter never sees rows outside its own fold's training partition at - * fit time regardless of what's picked here). + * fit time regardless of what's picked here. * - * The scope offered isn't just the dataset's raw columns: it's every raw - * column plus one synthetic key per output group of every converter - * already configured *before* this one in the sequence (chaining). The - * shared `ColumnSelector` only knows a plain `{name: {type, dtype}}` map, - * so groups are represented as synthetic keys built by `sessionColumnRefs` - * and translated back into real ColumnRef scope entries once the user - * picks a selection. + * The scope offered is the estimated dataset state at the end of the chain, + * where this converter is appended (chaining): the candidate columns that + * are still there, plus every column or block earlier converters produced, + * and never a column an earlier step consumed or the output column. The + * shared `ColumnSelector` only knows a plain `{name: {type, dtype}}` map, so + * each state item is represented by the synthetic key of the ColumnRef + * pointing at it (see `sessionColumnRefs`) and translated back into that + * ColumnRef once the user picks a selection. */ export default function ScopeStepSessionConverter({ tool, - datasetTypes, - preprocessing, + finalState, + stepDisplayNames, filePath, scope, setScope, nextStep, - convertersMeta, }) { const theme = useTheme(); const { t } = useTranslation(["common", "datasets", "models"]); @@ -42,9 +42,8 @@ export default function ScopeStepSessionConverter({ const inputCardinality = tool?.metadata?.input_cardinality || {}; const { columnTypes: columnTypesForSelector, optionLabels } = useMemo( - () => - buildColumnKeysAndTypes({ datasetTypes, preprocessing, convertersMeta }), - [datasetTypes, preprocessing, convertersMeta], + () => stateToOptions(finalState, stepDisplayNames), + [finalState, stepDisplayNames], ); const handleSelectionChange = (selected) => { @@ -102,11 +101,10 @@ export default function ScopeStepSessionConverter({ ScopeStepSessionConverter.propTypes = { tool: PropTypes.object.isRequired, - datasetTypes: PropTypes.object.isRequired, - preprocessing: PropTypes.array, + finalState: PropTypes.array.isRequired, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, filePath: PropTypes.string, scope: PropTypes.array.isRequired, setScope: PropTypes.func.isRequired, nextStep: PropTypes.func.isRequired, - convertersMeta: PropTypes.object, }; diff --git a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx index 3a71c5c6c..aab8cdf77 100644 --- a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx +++ b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx @@ -16,21 +16,27 @@ import ToolGrid from "../../notebooks/tool/ToolGrid"; import { getComponents } from "../../../api/component"; import { evaluateColumnEligibility } from "../../../utils/columnEligibility"; import FormSessionConverterSection from "./FormSessionConverterSection"; -import { buildColumnKeysAndTypes } from "./sessionColumnRefs"; +import { buildStepDisplayNames, stateToOptions } from "./sessionColumnRefs"; /** * Converters-only sidebar for the session wizard's preprocessing step, * styled like the notebook module's own converters picker (same SearchBar, * ToolList/ToolGrid, category accordion, list/grid toggle). Unlike the * notebook flow, picking a converter here never fits anything or POSTs - * anywhere — see FormSessionConverterSection, which just appends the - * configured step to newExp.preprocessing. + * anywhere: FormSessionConverterSection just appends the configured step to + * newExp.preprocessing. + * + * A new converter always lands at the end of the chain, so it is offered + * (and scoped) against the estimated final dataset state: the surviving + * candidate columns plus everything earlier steps produced, never the + * output column. Converters that change the number of rows are hidden, + * since sessions do not support them yet. */ export default function SessionConvertersRightBar({ newExp, setNewExp, dataset, - datasetTypes, + structure, }) { const theme = useTheme(); const { t } = useTranslation(["models", "datasets", "common"]); @@ -56,20 +62,22 @@ export default function SessionConvertersRightBar({ }; }, [t]); - // A new converter can be scoped on any raw dataset column OR the output - // group (any declared slot) of any converter already configured — it - // always lands at the end of the sequence, so every existing step is - // "before" it and fair game to chain off. Gating this on raw columns - // alone wrongly blocked, e.g., PCA on a text-only dataset (no raw - // Integer/Float columns) even after adding Bag of Words, whose Integer - // output group PCA could legitimately scope on. + // Type-level friendly names (e.g. "Simple Imputer" for "SimpleImputer"), + // used to label earlier steps' output blocks, numbered the same way + // AppliedConvertersView numbers repeated converter types. + const convertersMeta = useMemo( + () => Object.fromEntries(converters.map((c) => [c.name, c])), + [converters], + ); + const stepDisplayNames = useMemo( + () => buildStepDisplayNames(newExp.preprocessing, convertersMeta), + [newExp.preprocessing, convertersMeta], + ); + const finalState = structure?.final; + const { columnTypes: allColumnTypes } = useMemo( - () => - buildColumnKeysAndTypes({ - datasetTypes, - preprocessing: newExp.preprocessing, - }), - [datasetTypes, newExp.preprocessing], + () => stateToOptions(finalState, stepDisplayNames), + [finalState, stepDisplayNames], ); const datasetColumns = useMemo( @@ -120,21 +128,14 @@ export default function SessionConvertersRightBar({ return { disabled, tooltip, validColumns }; }; - // Type-level friendly names (e.g. "Simple Imputer" for "SimpleImputer"), - // reused when scoping a new converter over an earlier one's output group - // — see FormSessionConverterSection -> ScopeStepSessionConverter, which - // disambiguates repeated types the same way AppliedConvertersView does. - const convertersMeta = useMemo( - () => Object.fromEntries(converters.map((c) => [c.name, c])), - [converters], - ); - const validatedConverters = useMemo( () => - converters.map((converter) => { - const validation = validateConverter(converter); - return { ...converter, ...validation }; - }), + converters + .filter((converter) => converter.metadata?.column_operation !== "rows") + .map((converter) => { + const validation = validateConverter(converter); + return { ...converter, ...validation }; + }), [converters, datasetColumns, t], ); @@ -149,8 +150,8 @@ export default function SessionConvertersRightBar({ }, [searchQuery, validatedConverters]); // Stable identity across re-renders (e.g. every keystroke in the search - // box) so ConfigureToolModal never remounts FormSessionConverterSection - // — and loses in-progress scope/parameter input — while it's open. + // box) so ConfigureToolModal never remounts FormSessionConverterSection, + // and loses in-progress scope/parameter input, while it's open. const SessionFormSection = useMemo(() => { function Wrapped(sectionProps) { return ( @@ -158,14 +159,14 @@ export default function SessionConvertersRightBar({ {...sectionProps} newExp={newExp} setNewExp={setNewExp} - datasetTypes={datasetTypes} + finalState={finalState || []} + stepDisplayNames={stepDisplayNames} filePath={dataset?.file_path} - convertersMeta={convertersMeta} /> ); } return Wrapped; - }, [newExp, setNewExp, datasetTypes, dataset, convertersMeta]); + }, [newExp, setNewExp, finalState, stepDisplayNames, dataset]); return ( columnTypes[refToKey(ref)]); - if (scopeEntries.some((entry) => !entry?.type)) return fallback; - - const distinctTypes = [...new Set(scopeEntries.map((entry) => entry.type))]; - - const dtypeFor = (type) => { - const dtypes = [ - ...new Set( - scopeEntries - .filter((entry) => entry.type === type) - .map((entry) => entry.dtype) - .filter(Boolean), - ), - ]; - return dtypes.length === 1 ? dtypes[0] : null; - }; - - if (distinctTypes.length === 1) { - return [ - { slot: null, type: distinctTypes[0], dtype: dtypeFor(distinctTypes[0]) }, - ]; - } - - return distinctTypes.map((type) => ({ - slot: type, - type, - dtype: dtypeFor(type), - })); -} - /** * Every RAW dataset column name needed to compute a list of ColumnRef, * walking group refs back to their step's own scope recursively (a group diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js index eaf36cf26..9b4f6e0c1 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -7,7 +7,6 @@ import { keyToRef, buildColumnKeysAndTypes, buildStepDisplayNames, - resolveDeclaredOutputSlots, rawColumnsNeededFor, itemToRef, stateToOptions, @@ -149,141 +148,6 @@ describe("sessionColumnRefs", () => { expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); }); - describe("resolveDeclaredOutputSlots", () => { - const datasetTypes = { - age: { type: "Integer", dtype: "int64" }, - score: { type: "Float", dtype: "float64" }, - name: { type: "Text", dtype: "string" }, - }; - - it("falls back to the converter's declared metadata by default", () => { - const tool = { - name: "Binarizer", - metadata: { output_type: "Integer", output_dtype: "int64" }, - }; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [{ kind: "raw", name: "age" }], - datasetTypes, - preprocessing: [], - }); - expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); - }); - - it("uses the scope's real type when the converter preserves input type", () => { - const tool = { - name: "SelectKBest", - metadata: { output_type: "Float", preserves_input_type: true }, - }; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [{ kind: "raw", name: "age" }], - datasetTypes, - preprocessing: [], - }); - expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); - }); - - it("splits into one slot per distinct type when a preserving converter's scope mixes types", () => { - const tool = { - name: "VarianceThreshold", - metadata: { output_type: "Float", preserves_input_type: true }, - }; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [ - { kind: "raw", name: "age" }, - { kind: "raw", name: "name" }, - ], - datasetTypes, - preprocessing: [], - }); - expect(result).toEqual( - expect.arrayContaining([ - { slot: "Integer", type: "Integer", dtype: "int64" }, - { slot: "Text", type: "Text", dtype: "string" }, - ]), - ); - expect(result).toHaveLength(2); - }); - - it("treats SimpleImputer as type-preserving only for most_frequent/constant", () => { - const tool = { - name: "SimpleImputer", - metadata: { output_type: "Float", output_dtype: null }, - }; - const scope = [{ kind: "raw", name: "name" }]; - - const mostFrequent = resolveDeclaredOutputSlots({ - tool, - params: { strategy: "most_frequent" }, - scope, - datasetTypes, - preprocessing: [], - }); - expect(mostFrequent).toEqual([ - { slot: null, type: "Text", dtype: "string" }, - ]); - - const mean = resolveDeclaredOutputSlots({ - tool, - params: { strategy: "mean" }, - scope, - datasetTypes, - preprocessing: [], - }); - expect(mean).toEqual([{ slot: null, type: "Float", dtype: null }]); - }); - - it("resolves a preserving converter's scope through a chained group ref", () => { - const tool = { - name: "SelectKBest", - metadata: { output_type: "Float", preserves_input_type: true }, - }; - const preprocessing = [ - { - converter: "BagOfWordsConverter", - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], - }, - ]; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [{ kind: "group", step: 0 }], - datasetTypes, - preprocessing, - }); - expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); - }); - - it("resolves a preserving converter's scope through one specific slot of a chained group ref", () => { - const tool = { - name: "SelectKBest", - metadata: { output_type: "Float", preserves_input_type: true }, - }; - const preprocessing = [ - { - converter: "SimpleImputer", - outputSlots: [ - { slot: "Integer", type: "Integer", dtype: "int64" }, - { slot: "Categorical", type: "Categorical", dtype: null }, - ], - }, - ]; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [{ kind: "group", step: 0, slot: "Integer" }], - datasetTypes, - preprocessing, - }); - expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); - }); - }); - describe("buildStepDisplayNames", () => { it("leaves a converter type's name unchanged when it appears only once", () => { const steps = [ diff --git a/DashAI/front/src/components/models/modelSession/structureMessages.js b/DashAI/front/src/components/models/modelSession/structureMessages.js new file mode 100644 index 000000000..d25fcf795 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/structureMessages.js @@ -0,0 +1,27 @@ +/** + * Translate a structure message from the backend's infer_structure + * (`{code, params}`) into user-facing text, from `models:structure.`. + * + * A few codes need their params shaped first: a cardinality error names + * whichever bound was broken, and a list of accepted types is joined. + * + * @param {Function} t the i18next translation function + * @param {{code: string, params?: object}} message + * @returns {string} + */ +export function formatStructureMessage(t, message) { + if (!message) return ""; + const params = { ...(message.params || {}) }; + let key = message.code; + if (key === "cardinality") { + const tooFew = params.min != null && params.selected < params.min; + key = tooFew ? "cardinalityMin" : "cardinalityMax"; + } + if (Array.isArray(params.allowed)) { + params.allowed = params.allowed.join(", "); + } + return t(`models:structure.${key}`, { + ...params, + interpolation: { escapeValue: false }, + }); +} diff --git a/DashAI/front/src/utils/i18n/locales/de/models.json b/DashAI/front/src/utils/i18n/locales/de/models.json index 1cb70a7ba..86782b126 100644 --- a/DashAI/front/src/utils/i18n/locales/de/models.json +++ b/DashAI/front/src/utils/i18n/locales/de/models.json @@ -349,5 +349,25 @@ "skippedUndownloadedRuns": "Durchläufe mit nicht heruntergeladenem Modell wurden übersprungen. Lade das Modell zuerst herunter.", "skippedUnavailableRuns": "Durchläufe, deren Modell noch einen Download oder Anmeldedaten benötigt, wurden übersprungen. Behebe dies zuerst.", "tabAvailableAfterFinish": "Verfügbar, sobald der Durchlauf das Training abgeschlossen hat." + }, + "structure": { + "currentState": "Datensatz nach der Vorverarbeitung", + "columnsCount": "{{count}} Spalten", + "unknownCount": "N Spalten, erst nach dem Training bekannt", + "blocked": "Korrigieren Sie den vorherigen Schritt, um diesen zu prüfen.", + "loadError": "Das Ergebnis der Vorverarbeitung konnte nicht geschätzt werden.", + "missing_ref": "\"{{ref}}\" existiert an dieser Stelle der Kette nicht: Ein früherer Schritt hat sie möglicherweise verbraucht.", + "target_in_scope": "\"{{column}}\" ist die Ausgabespalte, und die Ausgabe wird nie transformiert.", + "type_not_allowed": "\"{{column}}\" ist {{type}}, was dieser Converter nicht akzeptiert (akzeptiert: {{allowed}}).", + "cardinalityMin": "Dieser Converter benötigt mindestens {{min}} Spalten, ausgewählt sind {{selected}}.", + "cardinalityMax": "Dieser Converter akzeptiert höchstens {{max}} Spalten, ausgewählt sind {{selected}}.", + "rows_not_supported": "Converter, die die Zeilenanzahl ändern, sind in Sitzungen noch nicht verfügbar.", + "n_components_exceeds": "Es werden {{n_components}} Komponenten angefordert, aber nur {{columns}} Spalten sind ausgewählt.", + "unknown_converter": "Der Converter \"{{converter}}\" ist nicht verfügbar.", + "invalid_params": "Ungültige Parameter: {{detail}}", + "converter_rejected": "Der Converter lehnt diese Konfiguration ab: {{detail}}", + "inference_fallback": "Das Ergebnis dieses Converters lässt sich nicht vorhersehen; seine Ausgabe wird als ein Block unbekannter Größe angezeigt.", + "count_unknown_bound": "Es werden {{n_components}} Komponenten angefordert; ob genug Spalten vorhanden sind, zeigt sich erst beim Training.", + "cast_may_skip": "Eine Spalte, die nicht in {{new_type}} umgewandelt werden kann, behält ihren ursprünglichen Typ." } } diff --git a/DashAI/front/src/utils/i18n/locales/en/models.json b/DashAI/front/src/utils/i18n/locales/en/models.json index d24aaa1d7..e7047744c 100644 --- a/DashAI/front/src/utils/i18n/locales/en/models.json +++ b/DashAI/front/src/utils/i18n/locales/en/models.json @@ -349,5 +349,25 @@ "skippedUndownloadedRuns": "Skipped runs whose model is not downloaded. Download the model first.", "skippedUnavailableRuns": "Skipped runs whose model still needs a download or credentials. Resolve those first.", "tabAvailableAfterFinish": "Available once the run finishes training." + }, + "structure": { + "currentState": "Dataset after preprocessing", + "columnsCount": "{{count}} columns", + "unknownCount": "N columns, known once trained", + "blocked": "Fix the previous step to check this one.", + "loadError": "Could not estimate the preprocessing result.", + "missing_ref": "\"{{ref}}\" does not exist at this point of the chain: an earlier step may have consumed it.", + "target_in_scope": "\"{{column}}\" is the output column, and the output is never transformed.", + "type_not_allowed": "\"{{column}}\" is {{type}}, which this converter does not accept (accepts: {{allowed}}).", + "cardinalityMin": "This converter needs at least {{min}} columns, but {{selected}} are selected.", + "cardinalityMax": "This converter accepts at most {{max}} columns, but {{selected}} are selected.", + "rows_not_supported": "Converters that change the number of rows are not available in sessions yet.", + "n_components_exceeds": "It asks for {{n_components}} components, but only {{columns}} columns are selected.", + "unknown_converter": "The converter \"{{converter}}\" is not available.", + "invalid_params": "Invalid parameters: {{detail}}", + "converter_rejected": "The converter rejects this configuration: {{detail}}", + "inference_fallback": "The result of this converter cannot be anticipated; its output is shown as one block of unknown size.", + "count_unknown_bound": "Asks for {{n_components}} components; whether there are enough columns is only known once trained.", + "cast_may_skip": "A column that cannot be converted to {{new_type}} keeps its original type." } } diff --git a/DashAI/front/src/utils/i18n/locales/es/models.json b/DashAI/front/src/utils/i18n/locales/es/models.json index a6b92ddbe..a27149260 100644 --- a/DashAI/front/src/utils/i18n/locales/es/models.json +++ b/DashAI/front/src/utils/i18n/locales/es/models.json @@ -358,5 +358,25 @@ "skippedUndownloadedRuns": "Se omitieron las ejecuciones cuyo modelo no está descargado. Descarga el modelo primero.", "skippedUnavailableRuns": "Se omitieron las ejecuciones cuyo modelo aún requiere descarga o credenciales. Resuélvelo primero.", "tabAvailableAfterFinish": "Estará disponible cuando el run termine de entrenar." + }, + "structure": { + "currentState": "Dataset después del preprocesamiento", + "columnsCount": "{{count}} columnas", + "unknownCount": "N columnas, se sabe al entrenar", + "blocked": "Corrige el paso anterior para revisar este.", + "loadError": "No se pudo estimar el resultado del preprocesamiento.", + "missing_ref": "\"{{ref}}\" no existe en este punto de la cadena: un paso anterior pudo haberla consumido.", + "target_in_scope": "\"{{column}}\" es la columna de salida, y la salida nunca se transforma.", + "type_not_allowed": "\"{{column}}\" es {{type}}, que este converter no acepta (acepta: {{allowed}}).", + "cardinalityMin": "Este converter necesita al menos {{min}} columnas, pero hay {{selected}} seleccionadas.", + "cardinalityMax": "Este converter acepta como máximo {{max}} columnas, pero hay {{selected}} seleccionadas.", + "rows_not_supported": "Los converters que cambian la cantidad de filas todavía no están disponibles en sesiones.", + "n_components_exceeds": "Pide {{n_components}} componentes, pero solo hay {{columns}} columnas seleccionadas.", + "unknown_converter": "El converter \"{{converter}}\" no está disponible.", + "invalid_params": "Parámetros inválidos: {{detail}}", + "converter_rejected": "El converter rechaza esta configuración: {{detail}}", + "inference_fallback": "No se puede anticipar el resultado de este converter; su salida se muestra como un bloque de tamaño desconocido.", + "count_unknown_bound": "Pide {{n_components}} componentes; si alcanzan las columnas se sabe recién al entrenar.", + "cast_may_skip": "Una columna que no se pueda convertir a {{new_type}} conserva su tipo original." } } diff --git a/DashAI/front/src/utils/i18n/locales/pt/models.json b/DashAI/front/src/utils/i18n/locales/pt/models.json index e5c8aaf46..3d0a74311 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/models.json +++ b/DashAI/front/src/utils/i18n/locales/pt/models.json @@ -358,5 +358,25 @@ "skippedUndownloadedRuns": "Execuções cujo modelo não está baixado foram ignoradas. Baixe o modelo primeiro.", "skippedUnavailableRuns": "Execuções cujo modelo ainda precisa de download ou credenciais foram ignoradas. Resolva isso primeiro.", "tabAvailableAfterFinish": "Estará disponível quando a execução terminar o treinamento." + }, + "structure": { + "currentState": "Dataset após o pré-processamento", + "columnsCount": "{{count}} colunas", + "unknownCount": "N colunas, conhecido após o treinamento", + "blocked": "Corrija a etapa anterior para verificar esta.", + "loadError": "Não foi possível estimar o resultado do pré-processamento.", + "missing_ref": "\"{{ref}}\" não existe neste ponto da cadeia: uma etapa anterior pode tê-la consumido.", + "target_in_scope": "\"{{column}}\" é a coluna de saída, e a saída nunca é transformada.", + "type_not_allowed": "\"{{column}}\" é {{type}}, que este conversor não aceita (aceita: {{allowed}}).", + "cardinalityMin": "Este conversor precisa de pelo menos {{min}} colunas, mas há {{selected}} selecionadas.", + "cardinalityMax": "Este conversor aceita no máximo {{max}} colunas, mas há {{selected}} selecionadas.", + "rows_not_supported": "Conversores que alteram o número de linhas ainda não estão disponíveis em sessões.", + "n_components_exceeds": "Pede {{n_components}} componentes, mas há apenas {{columns}} colunas selecionadas.", + "unknown_converter": "O conversor \"{{converter}}\" não está disponível.", + "invalid_params": "Parâmetros inválidos: {{detail}}", + "converter_rejected": "O conversor rejeita esta configuração: {{detail}}", + "inference_fallback": "Não é possível antecipar o resultado deste conversor; sua saída é mostrada como um bloco de tamanho desconhecido.", + "count_unknown_bound": "Pede {{n_components}} componentes; se há colunas suficientes só se sabe após o treinamento.", + "cast_may_skip": "Uma coluna que não puder ser convertida para {{new_type}} mantém seu tipo original." } } diff --git a/DashAI/front/src/utils/i18n/locales/zh/models.json b/DashAI/front/src/utils/i18n/locales/zh/models.json index e73db6b1d..656d12f67 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/models.json +++ b/DashAI/front/src/utils/i18n/locales/zh/models.json @@ -348,5 +348,25 @@ "skippedUndownloadedRuns": "已跳过模型未下载的运行。请先下载模型。", "skippedUnavailableRuns": "已跳过模型仍需下载或验证的运行。请先解决这些问题。", "tabAvailableAfterFinish": "运行训练完成后即可使用。" + }, + "structure": { + "currentState": "预处理后的数据集", + "columnsCount": "{{count}} 列", + "unknownCount": "N 列，训练后才能确定", + "blocked": "请先修正上一步，再检查这一步。", + "loadError": "无法估计预处理结果。", + "missing_ref": "“{{ref}}”在链的这一位置不存在：之前的某一步可能已将其消耗。", + "target_in_scope": "“{{column}}”是输出列，输出永远不会被转换。", + "type_not_allowed": "“{{column}}”的类型是 {{type}}，此转换器不接受（接受：{{allowed}}）。", + "cardinalityMin": "此转换器至少需要 {{min}} 列，但只选择了 {{selected}} 列。", + "cardinalityMax": "此转换器最多接受 {{max}} 列，但选择了 {{selected}} 列。", + "rows_not_supported": "会改变行数的转换器暂时无法在会话中使用。", + "n_components_exceeds": "要求 {{n_components}} 个成分，但只选择了 {{columns}} 列。", + "unknown_converter": "转换器“{{converter}}”不可用。", + "invalid_params": "参数无效：{{detail}}", + "converter_rejected": "转换器拒绝此配置：{{detail}}", + "inference_fallback": "无法预先确定此转换器的结果；其输出显示为一个大小未知的块。", + "count_unknown_bound": "要求 {{n_components}} 个成分；列数是否足够要到训练时才能确定。", + "cast_may_skip": "无法转换为 {{new_type}} 的列会保留其原始类型。" } } From 0ee3f79122d8a76218b4134beff9ba177fc95d1f Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 28 Sep 2026 12:14:42 +0200 Subject: [PATCH 38/39] feat: update session preprocessing structure and validation logic --- .../api/api_v1/endpoints/model_sessions.py | 29 +--- .../api_v1/schemas/model_sessions_params.py | 5 +- DashAI/front/src/api/modelSession.ts | 10 +- .../components/models/CreateSessionSteps.jsx | 10 +- .../components/models/SessionInfoContent.jsx | 16 +-- .../modelSession/DivideDatasetColumns.jsx | 5 +- .../models/modelSession/SelectColumnsStep.jsx | 136 ++++++++++-------- .../models/modelSession/sessionColumnRefs.js | 100 +++---------- .../modelSession/sessionColumnRefs.test.js | 107 -------------- tests/back/api/test_model_session_api.py | 60 +------- 10 files changed, 122 insertions(+), 356 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index d20c00cb9..9326cdc1c 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -307,29 +307,12 @@ async def validate_columns( ), } elif group_refs: - declared_types = params.converter_output_types or {} - task_metadata = task.get_metadata() - allowed_input_types = set(task_metadata.get("inputs_types", [])) - for ref in group_refs: - # A step with a heterogeneous scope (see SessionPreprocessor. - # _classify_by_type) can declare more than one type, one per - # slot — "{step}:{slot}" disambiguates which one a slotted ref - # means; an unslotted ref (the whole step) keeps the plain - # "{step}" key, unchanged from before slots existed. - key = str(ref.step) if ref.slot is None else f"{ref.step}:{ref.slot}" - declared_type = declared_types.get(key) - type_ok = declared_type is not None and _type_allowed( - declared_type, allowed_input_types - ) - if allowed_input_types and not type_ok: - return { - "dataset_status": "invalid", - "error": ( - f"Converter step {ref.step} declares output type " - f"'{declared_type}', which is not one of the task's " - f"allowed input types {sorted(allowed_input_types)}." - ), - } + # A ref to a converter's output only has a type through the chain's + # estimated structure, which needs the preprocessing steps. + return { + "dataset_status": "invalid", + "error": "References to converter outputs need the preprocessing steps.", + } validation_response = {} diff --git a/DashAI/back/api/api_v1/schemas/model_sessions_params.py b/DashAI/back/api/api_v1/schemas/model_sessions_params.py index ed70e1d66..983d836fe 100644 --- a/DashAI/back/api/api_v1/schemas/model_sessions_params.py +++ b/DashAI/back/api/api_v1/schemas/model_sessions_params.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import List, Optional from pydantic import BaseModel, Field @@ -29,9 +29,6 @@ class ColumnsValidationParams(BaseModel): # With preprocessing, the type of every input ref is taken from the # chain's estimated structure (see infer_structure). preprocessing: Optional[List[ConverterStep]] = None - # Legacy: group ref types computed by the frontend, used only when no - # preprocessing is sent. - converter_output_types: Optional[Dict[str, str]] = None class PreprocessingStructureParams(BaseModel): diff --git a/DashAI/front/src/api/modelSession.ts b/DashAI/front/src/api/modelSession.ts index 5fd319ac7..d495b1d24 100644 --- a/DashAI/front/src/api/modelSession.ts +++ b/DashAI/front/src/api/modelSession.ts @@ -85,7 +85,9 @@ export const validateColumns = async ( inputColumns: string[], outputColumns: string[], inputRefs?: IColumnRef[], - converterOutputTypes?: Record, + // With preprocessing, the backend types every input ref from the chain's + // estimated structure. + preprocessing?: IConverterStep[], ): Promise => { const formData = { task_name: taskName, @@ -93,7 +95,11 @@ export const validateColumns = async ( inputs_columns: inputColumns, outputs_columns: outputColumns, input_refs: inputRefs, - converter_output_types: converterOutputTypes, + preprocessing: preprocessing?.map(({ converter, params, scope }) => ({ + converter, + params, + scope, + })), }; const response = await api.post( "/v1/model-session/validation", diff --git a/DashAI/front/src/components/models/CreateSessionSteps.jsx b/DashAI/front/src/components/models/CreateSessionSteps.jsx index cf4bb6c5a..ec47701cf 100644 --- a/DashAI/front/src/components/models/CreateSessionSteps.jsx +++ b/DashAI/front/src/components/models/CreateSessionSteps.jsx @@ -264,6 +264,12 @@ function CreateSessionSteps({ const hasValidation = hasPartition(newExp.splits, "validation"); const hasTest = hasPartition(newExp.splits, "test"); + // Steps added while the switch was on and then abandoned by turning + // it off must not be sent: the session would still preprocess. + const preprocessing = newExp.applyPreprocessing + ? newExp.preprocessing + : []; + let effectiveName = sessionName; let response; try { @@ -278,7 +284,7 @@ function CreateSessionSteps({ hasTest ? allMetricNames : [], newExp.evaluation_strategy, JSON.stringify(newExp.splits), - newExp.preprocessing, + preprocessing, newExp.input_column_refs, ); } catch (createError) { @@ -296,7 +302,7 @@ function CreateSessionSteps({ hasTest ? allMetricNames : [], newExp.evaluation_strategy, JSON.stringify(newExp.splits), - newExp.preprocessing, + preprocessing, newExp.input_column_refs, ); } else { diff --git a/DashAI/front/src/components/models/SessionInfoContent.jsx b/DashAI/front/src/components/models/SessionInfoContent.jsx index 8b79df8ae..a937b2333 100644 --- a/DashAI/front/src/components/models/SessionInfoContent.jsx +++ b/DashAI/front/src/components/models/SessionInfoContent.jsx @@ -7,8 +7,7 @@ import { getComponents } from "../../api/component"; import ParamInfoList, { ParamInfoBox } from "./ParamInfoBox"; import { buildStepDisplayNames, - buildColumnKeysAndTypes, - refToKey, + labelForRef, } from "./modelSession/sessionColumnRefs"; const SPLIT_TYPE_LABEL_KEYS = { @@ -56,14 +55,6 @@ export default function SessionInfoContent({ preprocessingSteps, convertersMeta, ); - // datasetTypes isn't needed here: a raw scope ref is shown as its own - // column name regardless, and a group scope ref's label only depends on - // the earlier step it points to (see buildColumnKeysAndTypes). - const { optionLabels: scopeOptionLabels } = buildColumnKeysAndTypes({ - datasetTypes: {}, - preprocessing: preprocessingSteps, - convertersMeta, - }); const getDatasetName = () => { if (!session.dataset_id || !datasets.length) return t("common:unknown"); @@ -199,10 +190,7 @@ export default function SessionInfoContent({ key={`${step.converter}-${index}`} label={stepDisplayNames[index]} value={(step.scope || []) - .map((ref) => { - const key = refToKey(ref); - return scopeOptionLabels[key] || key; - }) + .map((ref) => labelForRef(ref, stepDisplayNames)) .join(", ")} /> ))} diff --git a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx index f3ad8bebd..52424b698 100644 --- a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx +++ b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx @@ -27,6 +27,7 @@ function DivideDatasetColumns({ outputHelperText = "", disabled = false, inputLabel, + outputDisabled = false, }) { const { t } = useTranslation(["experiments", "common", "models"]); const theme = useTheme(); @@ -187,7 +188,7 @@ function DivideDatasetColumns({ /> )} sx={{ mb: 8 }} - disabled={disabled || allColumnNames.length === 0} + disabled={disabled || outputDisabled || allColumnNames.length === 0} /> ); @@ -212,6 +213,8 @@ DivideDatasetColumns.propTypes = { disabled: PropTypes.bool, // Overrides the input selector's label (defaults to "Input columns"). inputLabel: PropTypes.string, + // Shows the output as fixed, e.g. when it was chosen in an earlier step. + outputDisabled: PropTypes.bool, }; export default DivideDatasetColumns; diff --git a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx index af54e0bfb..846514d26 100644 --- a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx +++ b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx @@ -19,16 +19,22 @@ import { isGroupKey, refToKey, keyToRef, - groupKey, - buildColumnKeysAndTypes, + buildStepDisplayNames, + stateToOptions, } from "./sessionColumnRefs"; +import usePreprocessingStructure from "./usePreprocessingStructure"; /** - * Step of the session wizard: pick input/output columns. A column can be a - * raw dataset column, or (when preprocessing is configured) the whole output - * group of one of the session's converter steps, represented here as a - * synthetic key (e.g. "__group__0") so the underlying Autocomplete only - * ever deals with plain strings — the same shape a raw column name has. + * Last step of the session wizard: pick the model's input columns (and, + * without preprocessing, its output column). + * + * With preprocessing, the inputs are picked from the estimated dataset + * state after the whole chain (see usePreprocessingStructure): surviving + * candidate columns plus what the converters produced, each represented by + * the synthetic key of the ColumnRef pointing at it, so the underlying + * Autocomplete only ever deals with plain strings. The output was already + * chosen in BaseColumnsStep and is shown fixed. Every item of the final + * state is selected by default. */ function SelectColumnsStep({ newExp, @@ -45,18 +51,29 @@ function SelectColumnsStep({ const [convertersMeta, setConvertersMeta] = useState({}); const rawColumnNames = datasetInfo.column_names || []; + const withPreprocessing = Boolean(newExp.applyPreprocessing); + const steps = newExp.preprocessing || []; - const { - allKeys: inputOptionNames, - columnTypes: columnTypesForSelector, - optionLabels, - } = buildColumnKeysAndTypes({ - datasetTypes, - preprocessing: newExp.preprocessing, - convertersMeta, + const { structure } = usePreprocessingStructure({ + datasetId: withPreprocessing ? dataset.id : undefined, + candidates: newExp.candidate_columns, + outputColumns: newExp.output_columns, + steps, }); - const [inputSelection, setInputSelection] = useState(() => + const stepDisplayNames = buildStepDisplayNames(steps, convertersMeta); + const finalOptions = stateToOptions(structure?.final, stepDisplayNames); + const inputOptionNames = withPreprocessing + ? finalOptions.allKeys + : rawColumnNames; + // The output keeps its dataset type; an input takes its type after the + // chain (e.g. a column cast in place by TypeCast). + const columnTypesForSelector = withPreprocessing + ? { ...datasetTypes, ...finalOptions.columnTypes } + : datasetTypes; + const optionLabels = withPreprocessing ? finalOptions.optionLabels : {}; + + const [inputSelectionState, setInputSelection] = useState(() => newExp.input_column_refs && newExp.input_column_refs.length > 0 ? newExp.input_column_refs.map(refToKey) : newExp.input_columns || [], @@ -65,41 +82,40 @@ function SelectColumnsStep({ newExp.output_columns || [], ); + // Only what can still be picked: going back and changing the chain can + // consume a column picked here before, and turning preprocessing off + // leaves only original columns. + const optionsKnown = withPreprocessing + ? Boolean(structure) + : rawColumnNames.length > 0; + const inputSelection = optionsKnown + ? inputSelectionState.filter((key) => inputOptionNames.includes(key)) + : inputSelectionState; + const inputColumnRefs = inputSelection.map(keyToRef); const rawInputNames = inputSelection.filter((key) => !isGroupKey(key)); const columnsReady = - inputSelection.length >= 1 && outputColumnNames.length >= 1; + inputSelection.length >= 1 && + outputColumnNames.length >= 1 && + (!withPreprocessing || Boolean(structure)); const [columnsAreValid, setColumnsAreValid] = useState(false); const [validationPending, setValidationPending] = useState(true); useLayoutEffect(() => { - // Auto-select sensible defaults the first time the dataset's columns load. - if (rawColumnNames.length === 0) return; + // Auto-select sensible defaults the first time the dataset's columns + // load. With preprocessing, the output comes from BaseColumnsStep and + // the inputs default once the estimated structure arrives (below). + if (rawColumnNames.length === 0 || withPreprocessing) return; if ( - inputSelection.length === 0 && + inputSelectionState.length === 0 && (!newExp.input_columns || newExp.input_columns.length === 0) ) { - const steps = newExp.preprocessing || []; - if (steps.length > 0) { - // With preprocessing configured, default to just the last step's - // output group(s) — the point of building a chain is usually to - // end up using its final result, not the raw columns it started - // from. Every raw column and every other group stays available to - // pick instead; this is only the starting default. - const lastIndex = steps.length - 1; - const slots = - steps[lastIndex].outputSlots?.length > 0 - ? steps[lastIndex].outputSlots - : [{ slot: null }]; - setInputSelection(slots.map(({ slot }) => groupKey(lastIndex, slot))); - } else { - setInputSelection( - rawColumnNames.length > 1 - ? rawColumnNames.slice(0, -1) - : [rawColumnNames[0]], - ); - } + setInputSelection( + rawColumnNames.length > 1 + ? rawColumnNames.slice(0, -1) + : [rawColumnNames[0]], + ); } if ( outputColumnNames.length === 0 && @@ -109,6 +125,15 @@ function SelectColumnsStep({ } }, [rawColumnNames.join(",")]); + useLayoutEffect(() => { + // With preprocessing, default to everything the chain leaves: the model + // is usually meant to use the chain's whole result. + if (!withPreprocessing || !structure) return; + if (inputSelection.length === 0) { + setInputSelection(finalOptions.allKeys); + } + }, [structure]); + const getTaskRequirements = async () => { try { const taskComponents = await getComponentsRequest({ @@ -146,29 +171,15 @@ function SelectColumnsStep({ setColumnsAreValid(false); return; } - const hasGroupRef = inputSelection.some(isGroupKey); - // Keyed "{step}" for a step with one declared type (unslotted, the - // common case) or "{step}:{slot}" for one of several — matches the - // key scheme validate_columns looks declared_type up by backend-side - // (see model_sessions.py), which mirrors GroupColumnRef.slot. - const converterOutputTypes = {}; - (newExp.preprocessing || []).forEach((step, index) => { - const slots = - step.outputSlots?.length > 0 - ? step.outputSlots - : [{ slot: null, type: step.outputType || null }]; - slots.forEach(({ slot, type }) => { - const key = slot == null ? String(index) : `${index}:${slot}`; - converterOutputTypes[key] = type || null; - }); - }); + // With preprocessing, the backend types every input from the chain's + // estimated structure, the same one this step's options come from. const validation = await validateColumnsRequest( newExp.task_name, dataset.id, - hasGroupRef ? rawInputNames : inputSelection, + rawInputNames, outputColumnNames, - hasGroupRef ? inputColumnRefs : undefined, - hasGroupRef ? converterOutputTypes : undefined, + withPreprocessing ? inputColumnRefs : undefined, + withPreprocessing ? steps : undefined, ); setColumnsAreValid(validation.dataset_status === "valid"); } catch (error) { @@ -216,9 +227,9 @@ function SelectColumnsStep({ }, []); useEffect(() => { - // Only needed to disambiguate two steps of the same converter type in - // the option labels below (e.g. "Simple Imputer" / "Simple Imputer - // (2)") — see buildStepDisplayNames. + // Only needed to name the steps in the option labels, disambiguating two + // of the same converter type (e.g. "Simple Imputer" / "Simple Imputer + // (2)"), see buildStepDisplayNames. let cancelled = false; getComponentsRequest({ selectTypes: ["Converter"] }) .then((data) => { @@ -375,6 +386,7 @@ function SelectColumnsStep({ outputColumnNames.length === 0 ? t("common:required") : "" } disabled={rawColumnNames.length === 0} + outputDisabled={withPreprocessing} /> diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js index 83b681fcb..abedcfb74 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js @@ -1,21 +1,20 @@ /** * Synthetic-key representation of a session's ColumnRef (see the backend's - * DashAI.back.preprocessing.column_ref module). A raw dataset column is - * just its own name; a converter's not-yet-materialized output group is - * represented as a synthetic string key, so every UI piece that already - * knows how to work with a flat `{name: string}` column list (Autocomplete - * options, MaterialReactTable rows, columnTypes maps) can represent a group - * without knowing groups exist at all. + * DashAI.back.preprocessing.column_ref module), so every UI piece that + * already works with a flat list of string column names (Autocomplete + * options, MaterialReactTable rows, columnTypes maps) can also represent + * what a converter produces, without knowing converters exist: * - * A step's output isn't always one homogeneous type — e.g. SimpleImputer - * with "most_frequent"/"constant" just preserves each scope column's own - * type, so a scope mixing a categorical and a numeric column produces both - * kinds of columns. `slot` (a DashAI type's display_name(), e.g. - * "Categorical") picks out just the columns of one declared type from a - * step's output, mirroring the backend's GroupColumnRef.slot / - * SessionPreprocessor.resolved_slots. A step with only one declared type — - * the common case, and everything before slots existed — has no slot - * (`slot: null`), which means "the whole group," unchanged from before. + * - an original dataset column is just its own name; + * - a generated column whose name is known before fit (e.g. DateFeatures' + * "date_month") is "__group__{step}__name__{name}"; + * - a block of columns only known after fit (e.g. one-hot columns) is + * "__group__{step}", or "__group__{step}__slot__{type}" when the step + * outputs blocks of several types, mirroring the backend's + * GroupColumnRef.slot and SessionPreprocessor.resolved_slots. + * + * Which of these exist at each point of a chain comes from the backend's + * estimated structure (see usePreprocessingStructure and stateToOptions). */ const GROUP_KEY_PREFIX = "__group__"; @@ -137,31 +136,13 @@ export function labelForRef(ref, stepDisplayNames = []) { : `${stepName}: output (${ref.slot})`; } -// A step's declared output, normalized to a list of slots — even a -// homogeneous step (the common case) is one "slot" with slot: null, so -// every caller iterates the same shape regardless of how many there are. -// Falls back defensively for a step that predates outputSlots. -function stepOutputSlots(step) { - if (Array.isArray(step?.outputSlots) && step.outputSlots.length > 0) { - return step.outputSlots; - } - return [ - { - slot: null, - type: step?.outputType ?? null, - dtype: step?.outputDtype ?? null, - }, - ]; -} - /** * One display name per step, disambiguated when the sequence has more than * one step of the same converter type (same registry name, or same - * fallback string when `convertersMeta` hasn't loaded yet) — the first + * fallback string when `convertersMeta` hasn't loaded yet): the first * occurrence keeps the bare name, later ones get " (2)", " (3)", etc., by - * order of appearance. Computed over the FULL `steps` array regardless of - * any later truncation (e.g. buildColumnKeysAndTypes's `uptoStep`), so a - * step's numbering never shifts depending on which view is asking. + * order of appearance. Computed over the full `steps` array, so a step's + * numbering never shifts depending on which view is asking. */ export function buildStepDisplayNames(steps, convertersMeta = {}) { const baseNames = (steps || []).map( @@ -181,56 +162,11 @@ export function buildStepDisplayNames(steps, convertersMeta = {}) { }); } -/** - * Every column key a session's preprocessing sequence can be scoped over, - * up to (and not including) `uptoStep`: every raw dataset column, plus one - * group key per declared slot of every converter step before it (usually - * one key per step; more than one only when that step's scope mixed - * column types). Passing no `uptoStep` includes every configured step - * (used once a sequence is final and being displayed, e.g. in - * SelectColumnsStep, where every step is already "before" the - * column-selection step that comes after all of them). - * - * `convertersMeta` (registry name -> component, e.g. from getComponents) - * is optional: when supplied, option labels use each step's disambiguated - * display name (see buildStepDisplayNames) instead of its raw registry - * name. - */ -export function buildColumnKeysAndTypes({ - datasetTypes, - preprocessing, - uptoStep, - convertersMeta = {}, -}) { - const steps = preprocessing || []; - const limit = uptoStep === undefined ? steps.length : uptoStep; - const displayNames = buildStepDisplayNames(steps, convertersMeta); - - const columnTypes = { ...datasetTypes }; - const optionLabels = {}; - const allKeys = Object.keys(datasetTypes || {}); - - for (let index = 0; index < limit; index += 1) { - const step = steps[index]; - const name = displayNames[index]; - stepOutputSlots(step).forEach(({ slot, type, dtype }) => { - const key = groupKey(index, slot); - columnTypes[key] = { type: type || null, dtype: dtype || null }; - optionLabels[key] = slot - ? `${name}: output (${slot})` - : `${name}: output`; - allKeys.push(key); - }); - } - - return { allKeys, columnTypes, optionLabels }; -} - /** * Every RAW dataset column name needed to compute a list of ColumnRef, * walking group refs back to their step's own scope recursively (a group * ref's step may itself reference an earlier group, chained arbitrarily - * deep). This is what a caller must actually supply values for — e.g. + * deep). This is what a caller must actually supply values for, e.g. * manual prediction, where the backend only ever accepts real dataset * columns as input (see BaseTask.process_manual_input), never a * converter's resolved output name like "pca_1": it runs the raw values diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js index 9b4f6e0c1..ec3f3077c 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -5,7 +5,6 @@ import { slotFromGroupKey, refToKey, keyToRef, - buildColumnKeysAndTypes, buildStepDisplayNames, rawColumnsNeededFor, itemToRef, @@ -42,112 +41,6 @@ describe("sessionColumnRefs", () => { expect(slotFromGroupKey(groupKey(2))).toBeNull(); }); - it("builds keys/types/labels for raw columns plus every prior step", () => { - const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ - datasetTypes: { age: { type: "Integer" }, text: { type: "Text" } }, - preprocessing: [ - { - converter: "BagOfWordsConverter", - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], - }, - { - converter: "Binarizer", - outputSlots: [{ slot: null, type: "Integer", dtype: null }], - }, - ], - uptoStep: 1, - }); - - expect(allKeys).toEqual(["age", "text", "__group__0"]); - expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); - expect(optionLabels.__group__0).toBe("BagOfWordsConverter: output"); - // step 1 (Binarizer) is excluded: uptoStep=1 only includes steps before it - expect(allKeys).not.toContain(groupKey(1)); - }); - - it("includes every step when uptoStep is omitted", () => { - const { allKeys } = buildColumnKeysAndTypes({ - datasetTypes: { age: { type: "Integer" } }, - preprocessing: [ - { - converter: "BagOfWordsConverter", - outputSlots: [{ slot: null, type: "Integer" }], - }, - { - converter: "Binarizer", - outputSlots: [{ slot: null, type: "Integer" }], - }, - ], - }); - - expect(allKeys).toEqual(["age", "__group__0", "__group__1"]); - }); - - it("offers one key per declared slot for a step with a heterogeneous scope", () => { - const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ - datasetTypes: {}, - preprocessing: [ - { - converter: "SimpleImputer", - outputSlots: [ - { slot: "Integer", type: "Integer", dtype: "int64" }, - { slot: "Categorical", type: "Categorical", dtype: null }, - ], - }, - ], - }); - - const integerKey = groupKey(0, "Integer"); - const categoricalKey = groupKey(0, "Categorical"); - expect(allKeys).toEqual([integerKey, categoricalKey]); - expect(columnTypes[integerKey]).toEqual({ - type: "Integer", - dtype: "int64", - }); - expect(columnTypes[categoricalKey]).toEqual({ - type: "Categorical", - dtype: null, - }); - expect(optionLabels[integerKey]).toBe("SimpleImputer: output (Integer)"); - expect(optionLabels[categoricalKey]).toBe( - "SimpleImputer: output (Categorical)", - ); - }); - - it("disambiguates option labels for two steps of the same converter type", () => { - const { optionLabels } = buildColumnKeysAndTypes({ - datasetTypes: {}, - preprocessing: [ - { - converter: "SimpleImputer", - outputSlots: [{ slot: null, type: "Float" }], - }, - { - converter: "SimpleImputer", - outputSlots: [{ slot: null, type: "Float" }], - }, - ], - convertersMeta: { - SimpleImputer: { display_name: "Simple Imputer" }, - }, - }); - - expect(optionLabels[groupKey(0)]).toBe("Simple Imputer: output"); - expect(optionLabels[groupKey(1)]).toBe("Simple Imputer (2): output"); - }); - - it("falls back to a single unslotted key for a step predating outputSlots", () => { - const { allKeys, columnTypes } = buildColumnKeysAndTypes({ - datasetTypes: {}, - preprocessing: [ - { converter: "Binarizer", outputType: "Integer", outputDtype: "int64" }, - ], - }); - - expect(allKeys).toEqual(["__group__0"]); - expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); - }); - describe("buildStepDisplayNames", () => { it("leaves a converter type's name unchanged when it appears only once", () => { const steps = [ diff --git a/tests/back/api/test_model_session_api.py b/tests/back/api/test_model_session_api.py index 5c81e53f4..d9b3fdeae 100644 --- a/tests/back/api/test_model_session_api.py +++ b/tests/back/api/test_model_session_api.py @@ -127,28 +127,7 @@ def test_create_model_session_with_preprocessing_requires_input_column_refs( assert response.status_code == 422, response.text -def test_validate_columns_accepts_a_group_ref_matching_the_task_type( - client: TestClient, dataset_1: Dataset -) -> None: - response = client.post( - "/api/v1/model-session/validation", - json={ - "task_name": "TabularClassificationTask", - "dataset_id": dataset_1.id, - "inputs_columns": ["SepalWidthCm"], - "outputs_columns": ["Species"], - "input_refs": [ - {"kind": "raw", "name": "SepalWidthCm"}, - {"kind": "group", "step": 0}, - ], - "converter_output_types": {"0": "Integer"}, - }, - ) - assert response.status_code == 200, response.text - assert response.json()["dataset_status"] == "valid" - - -def test_validate_columns_rejects_a_group_ref_with_an_incompatible_type( +def test_validate_columns_rejects_group_refs_without_preprocessing( client: TestClient, dataset_1: Dataset ) -> None: response = client.post( @@ -159,43 +138,6 @@ def test_validate_columns_rejects_a_group_ref_with_an_incompatible_type( "inputs_columns": [], "outputs_columns": ["Species"], "input_refs": [{"kind": "group", "step": 0}], - "converter_output_types": {"0": "Text"}, - }, - ) - assert response.status_code == 200, response.text - assert response.json()["dataset_status"] == "invalid" - - -def test_validate_columns_accepts_a_slotted_group_ref_matching_the_task_type( - client: TestClient, dataset_1: Dataset -) -> None: - response = client.post( - "/api/v1/model-session/validation", - json={ - "task_name": "TabularClassificationTask", - "dataset_id": dataset_1.id, - "inputs_columns": [], - "outputs_columns": ["Species"], - "input_refs": [{"kind": "group", "step": 0, "slot": "Integer"}], - "converter_output_types": {"0:Integer": "Integer", "0:Categorical": "Text"}, - }, - ) - assert response.status_code == 200, response.text - assert response.json()["dataset_status"] == "valid" - - -def test_validate_columns_rejects_a_slotted_group_ref_with_an_incompatible_type( - client: TestClient, dataset_1: Dataset -) -> None: - response = client.post( - "/api/v1/model-session/validation", - json={ - "task_name": "TabularClassificationTask", - "dataset_id": dataset_1.id, - "inputs_columns": [], - "outputs_columns": ["Species"], - "input_refs": [{"kind": "group", "step": 0, "slot": "Categorical"}], - "converter_output_types": {"0:Integer": "Integer", "0:Categorical": "Text"}, }, ) assert response.status_code == 200, response.text From 0b5246eec1b701db05746b1f8fe740f1b3b08274 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 29 Sep 2026 18:20:17 +0200 Subject: [PATCH 39/39] feat: add leakage warning toggle to ParameterStepConverter and update FormSessionConverterSection --- .../models/modelSession/FormSessionConverterSection.jsx | 1 + .../notebooks/converterCreation/ParameterStepConverter.jsx | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx index 46dcfaf32..c6093685f 100644 --- a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx +++ b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx @@ -77,6 +77,7 @@ export default function FormSessionConverterSection({ handleSaveConverter={handleSaveConverter} setStep={setStep} hideButtons={hideButtons} + warnAboutLeakage={false} /> )} diff --git a/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx b/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx index 5e37f2fbb..cf872bdd4 100644 --- a/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx +++ b/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx @@ -13,6 +13,10 @@ export default function ParameterStepConverter({ handleSaveConverter, setStep, hideButtons = false, + // The notebook fits a converter on the whole dataset, so one that learns + // from the data can leak test information. Session preprocessing fits on + // the training split only, so it turns this warning off. + warnAboutLeakage = true, }) { const tourContext = useTourContext(); const { t } = useTranslation(["common", "datasets"]); @@ -72,7 +76,8 @@ export default function ParameterStepConverter({ nColumnsSelected > 0 && currentNComponents > nColumnsSelected; - const showLeakageWarning = tool?.metadata?.learns_from_data === true; + const showLeakageWarning = + warnAboutLeakage && tool?.metadata?.learns_from_data === true; return (