diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index aad2cc72b..9326cdc1c 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -11,11 +11,17 @@ ColumnsValidationParams, ModelSessionBulkDeleteParams, ModelSessionParams, + PreprocessingStructureParams, ) from DashAI.back.api.utils import remove_path from DashAI.back.dependencies.database.models import Dataset, ModelSession, Run from DashAI.back.job.preprocessing_job import PreprocessingJob from DashAI.back.preprocessing.column_ref import ConverterSequence, RawColumnRef +from DashAI.back.preprocessing.structure import ( + StructureError, + infer_structure, + resolve_state_refs, +) from DashAI.back.splitters.splits_payload import ( META_KEYS, normalize_splits_payload, @@ -121,14 +127,26 @@ async def get_model_session( return model_session -@router.post("/validation") -@inject -async def validate_columns( - params: ColumnsValidationParams, - component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """Validate if dataset columns are compatible with a task.""" +def _load_sample_dataset(db, dataset_id: int): + """Load the first rows of a dataset, enough for its columns and types. + + Parameters + ---------- + db : Session + An open database session. + dataset_id : int + The dataset to load. + + Returns + ------- + DashAIDataset + At most 5 rows of the dataset, with every column and its type. + + Raises + ------ + HTTPException + 404 if the dataset is not registered. + """ import os import pyarrow as pa @@ -136,25 +154,92 @@ async def validate_columns( from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + dataset = db.get(Dataset, dataset_id) + if not dataset: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Dataset not found", + ) + data_filepath = os.path.join(f"{dataset.file_path}/dataset", "data.arrow") + with pa.OSFile(data_filepath, "rb") as source: + reader = ipc.open_file(source) + batch = reader.get_batch(0) + sample_batch = batch.slice(0, min(5, batch.num_rows)) + return DashAIDataset(pa.Table.from_batches([sample_batch])) + + +def _type_allowed(type_name, allowed_types) -> bool: + """Whether a column type is one of a task's allowed input types. + + A task declaring the DashAIValue wildcard accepts every concrete value + type; an unknown type (None) is let through, since only a fit can tell. + """ + if not allowed_types or type_name is None: + return True + return type_name in allowed_types or ( + "DashAIValue" in allowed_types and type_name in _DASHAI_VALUE_TYPE_NAMES + ) + + +@router.post("/preprocessing/structure") +@inject +async def get_preprocessing_structure( + params: PreprocessingStructureParams, + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Estimate the dataset structure after every step of a converter chain. + + Read only: nothing is fit and nothing is stored. The session wizard calls + it on every change to the chain, to offer only columns that exist at each + step and to flag steps that cannot work. + + Parameters + ---------- + params : PreprocessingStructureParams + The dataset, the candidate and target columns, and the chain. + + Returns + ------- + StructureResult + The state after every step, their errors and warnings, and the final + state. + """ with session_factory() as db: try: - dataset = db.get(Dataset, params.dataset_id) - if not dataset: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Dataset not found", - ) + sample = _load_sample_dataset(db, params.dataset_id) + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + return infer_structure( + dataset_types=sample.types, + candidates=params.candidates, + target=params.output_columns, + steps=params.steps, + component_registry=component_registry, + ) - dataset_path = f"{dataset.file_path}/dataset" - data_filepath = os.path.join(dataset_path, "data.arrow") - with pa.OSFile(data_filepath, "rb") as source: - reader = ipc.open_file(source) - batch = reader.get_batch(0) - sample_size = min(5, batch.num_rows) - sample_batch = batch.slice(0, sample_size) - table = pa.Table.from_batches([sample_batch]) - minimal_dataset = DashAIDataset(table) +@router.post("/validation") +@inject +async def validate_columns( + params: ColumnsValidationParams, + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Validate if dataset columns are compatible with a task. + + With `preprocessing`, every input ref is resolved against the chain's + estimated final structure and its type checked against the task's input + types; the original columns among them also go through the task's own + validation, as without preprocessing. + """ + with session_factory() as db: + try: + minimal_dataset = _load_sample_dataset(db, params.dataset_id) column_names = minimal_dataset.column_names @@ -192,31 +277,42 @@ async def validate_columns( task: "BaseTask" = component_registry[params.task_name]["class"]() - if group_refs: - declared_types = params.converter_output_types or {} - task_metadata = task.get_metadata() - allowed_input_types = set(task_metadata.get("inputs_types", [])) - for ref in group_refs: - # A step with a heterogeneous scope (see SessionPreprocessor. - # _classify_by_type) can declare more than one type, one per - # slot — "{step}:{slot}" disambiguates which one a slotted ref - # means; an unslotted ref (the whole step) keeps the plain - # "{step}" key, unchanged from before slots existed. - key = str(ref.step) if ref.slot is None else f"{ref.step}:{ref.slot}" - declared_type = declared_types.get(key) - type_ok = declared_type in allowed_input_types or ( - "DashAIValue" in allowed_input_types - and declared_type in _DASHAI_VALUE_TYPE_NAMES + if params.preprocessing: + allowed_input_types = set(task.get_metadata().get("inputs_types", [])) + structure = infer_structure( + dataset_types=minimal_dataset.types, + candidates=list(column_names), + target=outputs_names, + steps=params.preprocessing, + component_registry=component_registry, + ) + if not structure.valid: + return { + "dataset_status": "invalid", + "error": "The preprocessing chain has an invalid step.", + } + try: + items = resolve_state_refs( + params.input_refs or [], structure.final, set(outputs_names) ) - if allowed_input_types and not type_ok: + except StructureError as e: + return {"dataset_status": "invalid", "error": e.message.code} + for item in items: + if not _type_allowed(item.type, allowed_input_types): return { "dataset_status": "invalid", "error": ( - f"Converter step {ref.step} declares output type " - f"'{declared_type}', which is not one of the task's " + f"Input of type '{item.type}' is not one of the task's " f"allowed input types {sorted(allowed_input_types)}." ), } + elif group_refs: + # A ref to a converter's output only has a type through the chain's + # estimated structure, which needs the preprocessing steps. + return { + "dataset_status": "invalid", + "error": "References to converter outputs need the preprocessing steps.", + } validation_response = {} @@ -286,6 +382,52 @@ def _validate_splits(splits: str, component_registry: "ComponentRegistry") -> No ) from e +def _check_preprocessing_structure( + db, params, sequence, input_column_refs, component_registry +) -> None: + """Reject a session whose chain or inputs cannot work, before creating it. + + Without this, the session would be created and its PreprocessingJob + would only fail afterwards, with an error like a bare missing column + name. + + Raises + ------ + HTTPException + 422 with the estimated structure when a step is invalid, or with the + error when an input ref points at something the chain does not + produce (or consumed). + """ + sample = _load_sample_dataset(db, params.dataset_id) + structure = infer_structure( + dataset_types=sample.types, + candidates=list(sample.column_names), + target=params.output_columns, + steps=sequence.steps, + component_registry=component_registry, + ) + if not structure.valid: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={ + "message": "The preprocessing chain has an invalid step.", + "structure": structure.model_dump(mode="json"), + }, + ) + try: + resolve_state_refs( + input_column_refs, structure.final, set(params.output_columns) + ) + except StructureError as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={ + "message": "An input column does not exist after preprocessing.", + "error": e.message.model_dump(mode="json"), + }, + ) from e + + @router.post("/", status_code=status.HTTP_201_CREATED) @inject async def create_model_session( @@ -362,6 +504,11 @@ async def create_model_session( schema = reader.schema column_names = schema.names + if has_preprocessing: + _check_preprocessing_structure( + db, params, sequence, input_column_refs, component_registry + ) + # When preprocessing is configured, input_columns is only a # placeholder until PreprocessingJob resolves the real ones, so # it cannot be checked against the raw dataset's column count. diff --git a/DashAI/back/api/api_v1/schemas/model_sessions_params.py b/DashAI/back/api/api_v1/schemas/model_sessions_params.py index f5e35e415..983d836fe 100644 --- a/DashAI/back/api/api_v1/schemas/model_sessions_params.py +++ b/DashAI/back/api/api_v1/schemas/model_sessions_params.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import List, Optional from pydantic import BaseModel, Field @@ -26,7 +26,17 @@ class ColumnsValidationParams(BaseModel): inputs_columns: List[str] outputs_columns: List[str] input_refs: Optional[List[ColumnRef]] = None - converter_output_types: Optional[Dict[str, str]] = None + # With preprocessing, the type of every input ref is taken from the + # chain's estimated structure (see infer_structure). + preprocessing: Optional[List[ConverterStep]] = None + + +class PreprocessingStructureParams(BaseModel): + dataset_id: int + # Original columns the user may feed into the chain. + candidates: List[str] + output_columns: List[str] + steps: List[ConverterStep] = Field(default_factory=list) class ModelSessionBulkDeleteParams(BaseModel): diff --git a/DashAI/back/converters/base_converter.py b/DashAI/back/converters/base_converter.py index 3eda0d764..ba794cd01 100644 --- a/DashAI/back/converters/base_converter.py +++ b/DashAI/back/converters/base_converter.py @@ -1,10 +1,18 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Dict, Final, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Type, Union from DashAI.back.config_object import ConfigObject from DashAI.back.core.schema_fields.base_schema import BaseSchema +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + ColumnItem, + RowsNotSupportedError, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.static.icons import Icon from DashAI.back.types.dashai_data_type import DashAIDataType @@ -61,6 +69,17 @@ class BaseConverter(ConfigObject, ABC): # which — called on a bare unfitted instance — has no idea what column # it will actually run on. PRESERVES_INPUT_TYPE: bool = False + # What the converter does to the columns in its scope, used to estimate + # the dataset structure of a session's converter chain before any fit + # (see infer_output_columns): + # - "replace": same columns, values (and maybe types) changed + # - "add": keeps its scope columns and adds new ones + # - "expand": consumes its scope columns and outputs new ones + # - "select": keeps a subset of its scope columns, unchanged + # - "rows": changes the row count (not supported in sessions yet) + # None (e.g. an older plugin) is treated as "expand" with an unknown + # column count, which never promises a column that may not exist. + COLUMN_OPERATION: Optional[str] = None SCHEMA: BaseConverterSchema @classmethod @@ -95,6 +114,7 @@ def get_metadata(cls) -> Dict[str, Any]: meta["changes_row_count"] = cls.CHANGES_ROW_COUNT meta["preserves_input_type"] = cls.PRESERVES_INPUT_TYPE meta["learns_from_data"] = cls.LEARNS_FROM_DATA + meta["column_operation"] = cls.COLUMN_OPERATION meta["n_components_features_bounded"] = getattr( cls, "N_COMPONENTS_FEATURES_BOUNDED", False ) @@ -135,19 +155,11 @@ def get_metadata(cls) -> Dict[str, Any]: # require constructor params with no default), so this is # best-effort: None means "unknown until configured". try: - output_type = cls().get_output_type() - meta["output_type"] = ( - output_type.display_name() - if output_type is not None and hasattr(output_type, "display_name") - else None - ) - # The concrete storage dtype (e.g. "int64"), so a group column can - # show one instead of "unknown" before any real fit exists — same - # best-effort default-constructed instance as output_type above. - meta["output_dtype"] = ( - output_type.to_string().get("dtype") - if output_type is not None and hasattr(output_type, "to_string") - else None + # output_dtype is the concrete storage dtype (e.g. "int64"), so a + # group column can show one instead of "unknown" before any real + # fit exists. + meta["output_type"], meta["output_dtype"] = type_fields( + cls().get_output_type() ) except Exception: meta["output_type"] = None @@ -155,6 +167,91 @@ def get_metadata(cls) -> Dict[str, Any]: return meta + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate what this converter does to its scope, without any data. + + Called on an instance built with the user's params but never fit, so + a session wizard can show which columns exist after each step of a + converter chain. The default follows COLUMN_OPERATION; converters + that know more (e.g. an exact column count from `n_components`, or + output names built from their inputs) override it. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The scope items that survive (maybe retyped) and the new items. + + Raises + ------ + RowsNotSupportedError + If the converter changes the row count. + """ + operation = type(self).COLUMN_OPERATION or "expand" + if operation == "rows": + raise RowsNotSupportedError(type(self).__name__) + if operation == "replace": + return StructureDelta(kept=[self._retype(item) for item in inputs]) + if operation == "add": + return StructureDelta(kept=list(inputs), added=self._default_blocks()) + if operation == "select": + return StructureDelta(added=self._selection_blocks(inputs)) + return StructureDelta(added=self._default_blocks()) + + def _retype(self, item: StateItem) -> StateItem: + """Copy an item replaced in place with this converter's output type.""" + if type(self).PRESERVES_INPUT_TYPE: + return item + column_name = item.name if isinstance(item, ColumnItem) else None + type_name, dtype = type_fields(self.get_output_type(column_name)) + return item.model_copy(update={"type": type_name, "dtype": dtype}) + + def _default_blocks(self, count: Optional[int] = None) -> List[StateItem]: + """One block typed with this converter's output type. + + Parameters + ---------- + count : int, optional + The block's column count, when params determine it. Defaults to + None (known only after fit). + """ + type_name, dtype = type_fields(self.get_output_type()) + return [BlockItem(type=type_name, dtype=dtype, count=count)] + + def _selected_count(self, inputs: List[StateItem]) -> Optional[int]: + """How many scope columns a "select" converter keeps, if params say. + + None (the default) means it depends on the data, e.g. a statistical + test threshold. Overridden by selectors with a fixed count (`k`). + """ + return None + + def _selection_blocks(self, inputs: List[StateItem]) -> List[StateItem]: + """One block per distinct input type, since selection keeps types. + + A selector says at most how many columns survive, never which ones, + so its output is always symbolic, even with a known count. + """ + by_type: Dict[Optional[str], List[StateItem]] = {} + for item in inputs: + by_type.setdefault(item.type, []).append(item) + count = self._selected_count(inputs) if len(by_type) == 1 else None + blocks = [] + for type_name, items in by_type.items(): + dtypes = {item.dtype for item in items} + blocks.append( + BlockItem( + type=type_name, + dtype=dtypes.pop() if len(dtypes) == 1 else None, + count=count, + ) + ) + return blocks + @abstractmethod def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a given column. diff --git a/DashAI/back/converters/category/advanced_preprocessing.py b/DashAI/back/converters/category/advanced_preprocessing.py index cba5f6ba7..b541df481 100644 --- a/DashAI/back/converters/category/advanced_preprocessing.py +++ b/DashAI/back/converters/category/advanced_preprocessing.py @@ -25,3 +25,4 @@ class AdvancedPreprocessingConverter(BaseConverter): ) ICON: Final[str] = Icon.Psychology.value COLOR: Final[str] = "rgb(70, 130, 180)" + COLUMN_OPERATION = "add" diff --git a/DashAI/back/converters/category/basic_preprocessing.py b/DashAI/back/converters/category/basic_preprocessing.py index 3a0dc8d56..dcb6ac98c 100644 --- a/DashAI/back/converters/category/basic_preprocessing.py +++ b/DashAI/back/converters/category/basic_preprocessing.py @@ -25,3 +25,4 @@ class BasicPreprocessingConverter(BaseConverter): ) ICON: Final[str] = Icon.Build.value COLOR: Final[str] = "rgb(60, 179, 113)" + COLUMN_OPERATION = "replace" diff --git a/DashAI/back/converters/category/dimensionality_reduction.py b/DashAI/back/converters/category/dimensionality_reduction.py index 378296343..6dd14bc51 100644 --- a/DashAI/back/converters/category/dimensionality_reduction.py +++ b/DashAI/back/converters/category/dimensionality_reduction.py @@ -26,4 +26,5 @@ class DimensionalityReductionConverter(BaseConverter): ) ICON: Final[str] = Icon.Layers.value COLOR: Final[str] = "rgb(255, 99, 132)" + COLUMN_OPERATION = "expand" N_COMPONENTS_FEATURES_BOUNDED: bool = True diff --git a/DashAI/back/converters/category/encoding.py b/DashAI/back/converters/category/encoding.py index dc812159e..c61bb83b3 100644 --- a/DashAI/back/converters/category/encoding.py +++ b/DashAI/back/converters/category/encoding.py @@ -1,7 +1,13 @@ -from typing import TYPE_CHECKING, Final, Union +from typing import TYPE_CHECKING, Final, List, Union from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.static.icons import Icon if TYPE_CHECKING: @@ -28,7 +34,47 @@ class EncodingConverter(BaseConverter): ) ICON: Final[str] = Icon.Dns.value COLOR: Final[str] = "rgb(138, 43, 226)" + COLUMN_OPERATION = "add" PREFIX: str = "encoded_" + # Whether the converter appends exactly one PREFIX column per encoded + # scope column (ordinal, label, binarizer). One-hot appends one column + # per category instead, a count only known after fit. + ONE_COLUMN_PER_INPUT: bool = False + + def _encodes(self, item: ColumnItem) -> bool: + """Whether a scope column gets an encoded copy. Defaults to all.""" + return True + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus one encoded copy each. + + Only exact for ONE_COLUMN_PER_INPUT converters over concrete + columns; otherwise the new columns are one block of unknown size. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the encoded columns. + """ + concrete = all(isinstance(item, ColumnItem) for item in inputs) + if not (type(self).ONE_COLUMN_PER_INPUT and concrete): + return super().infer_output_columns(inputs) + added = [] + for item in inputs: + if not self._encodes(item): + continue + type_name, dtype = type_fields(self.get_output_type(item.name)) + added.append( + ColumnItem( + name=f"{self.PREFIX}{item.name}", type=type_name, dtype=dtype + ) + ) + return StructureDelta(kept=list(inputs), added=added) def transform( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/converters/category/feature_engineering.py b/DashAI/back/converters/category/feature_engineering.py index 5dca3da00..8f03cd9ae 100644 --- a/DashAI/back/converters/category/feature_engineering.py +++ b/DashAI/back/converters/category/feature_engineering.py @@ -28,3 +28,4 @@ class FeatureEngineeringConverter(BaseConverter): ) ICON: Final[str] = Icon.Functions.value COLOR: Final[str] = "rgb(0, 188, 212)" + COLUMN_OPERATION = "add" diff --git a/DashAI/back/converters/category/feature_selection.py b/DashAI/back/converters/category/feature_selection.py index a8c9919e2..e3cd5a52e 100644 --- a/DashAI/back/converters/category/feature_selection.py +++ b/DashAI/back/converters/category/feature_selection.py @@ -34,6 +34,7 @@ class FeatureSelectionConverter(BaseConverter): ) ICON: Final[str] = Icon.FilterList.value COLOR: Final[str] = "rgb(255, 206, 86)" + COLUMN_OPERATION = "select" PRESERVES_INPUT_TYPE = True def fit( diff --git a/DashAI/back/converters/category/polynomial_kernel.py b/DashAI/back/converters/category/polynomial_kernel.py index 96dd1cf7d..827c0f65a 100644 --- a/DashAI/back/converters/category/polynomial_kernel.py +++ b/DashAI/back/converters/category/polynomial_kernel.py @@ -26,3 +26,4 @@ class PolynomialKernelConverter(BaseConverter): ) ICON: Final[str] = Icon.Functions.value COLOR: Final[str] = "rgb(153, 102, 255)" + COLUMN_OPERATION = "expand" diff --git a/DashAI/back/converters/category/sampling.py b/DashAI/back/converters/category/sampling.py index 77cd19062..93f8de27b 100644 --- a/DashAI/back/converters/category/sampling.py +++ b/DashAI/back/converters/category/sampling.py @@ -25,3 +25,4 @@ class SamplingConverter(BaseConverter): ) ICON: Final[str] = Icon.Casino.value COLOR: Final[str] = "rgb(255, 159, 64)" + COLUMN_OPERATION = "rows" diff --git a/DashAI/back/converters/category/scaling_and_normalization.py b/DashAI/back/converters/category/scaling_and_normalization.py index a2878740c..776eb7819 100644 --- a/DashAI/back/converters/category/scaling_and_normalization.py +++ b/DashAI/back/converters/category/scaling_and_normalization.py @@ -25,3 +25,4 @@ class ScalingAndNormalizationConverter(BaseConverter): ) ICON: Final[str] = Icon.TrendingUp.value COLOR: Final[str] = "rgb(255, 165, 0)" + COLUMN_OPERATION = "replace" diff --git a/DashAI/back/converters/dataset_columns.py b/DashAI/back/converters/dataset_columns.py index cad8406c5..51f5026d7 100644 --- a/DashAI/back/converters/dataset_columns.py +++ b/DashAI/back/converters/dataset_columns.py @@ -3,12 +3,47 @@ preprocessing (SessionPreprocessor). """ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, Dict, Iterable, List if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset +def plan_new_column_names( + taken_names: Iterable[str], new_columns: List[str] +) -> Dict[str, str]: + """Name every new column so it never collides with an existing one. + + A new column whose name is already taken gets the first free + ``_1``, ``_2``... suffix. Shared by the dataset rebuild below + and by the structure estimate of a session's converter chain, so both + always agree on the final names. + + Parameters + ---------- + taken_names : Iterable[str] + Names already present once the converter's removed columns are gone. + new_columns : List[str] + The converter's new column names, in output order. + + Returns + ------- + Dict[str, str] + Each new column name mapped to its final, unique name. + """ + seen = set(taken_names) + mapping = {} + for col in new_columns: + unique_col = col + counter = 1 + while unique_col in seen: + unique_col = f"{col}_{counter}" + counter += 1 + seen.add(unique_col) + mapping[col] = unique_col + return mapping + + def rebuild_dataset_with_transformed_columns( base: "DashAIDataset", transformed: "DashAIDataset", @@ -70,16 +105,8 @@ def rebuild_dataset_with_transformed_columns( new_columns_order.append(col) seen_cols.add(col) - col_name_mapping = {} - for col in new_cols: - unique_col = col - counter = 1 - while unique_col in seen_cols: - unique_col = f"{col}_{counter}" - counter += 1 - new_columns_order.append(unique_col) - seen_cols.add(unique_col) - col_name_mapping[col] = unique_col + col_name_mapping = plan_new_column_names(seen_cols, new_cols) + new_columns_order.extend(col_name_mapping.values()) updated_arrays = {} for col in replacement_cols: diff --git a/DashAI/back/converters/hugging_face/embedding.py b/DashAI/back/converters/hugging_face/embedding.py index f0666d7fc..58eaa9197 100644 --- a/DashAI/back/converters/hugging_face/embedding.py +++ b/DashAI/back/converters/hugging_face/embedding.py @@ -105,6 +105,7 @@ class Embedding(AdvancedPreprocessingConverter, HuggingFaceWrapper): SCHEMA = EmbeddingSchema LEARNS_FROM_DATA = False + COLUMN_OPERATION = "expand" DESCRIPTION = MultilingualString( en="Convert text to embeddings using HuggingFace transformer models.", es="Convierte texto a embeddings usando modelos de HuggingFace.", diff --git a/DashAI/back/converters/hugging_face/image_embedding.py b/DashAI/back/converters/hugging_face/image_embedding.py index c529b8646..1f6711c48 100644 --- a/DashAI/back/converters/hugging_face/image_embedding.py +++ b/DashAI/back/converters/hugging_face/image_embedding.py @@ -1,6 +1,6 @@ """HuggingFace image embedding converter""" -from typing import TYPE_CHECKING, Callable, Dict, Optional +from typing import TYPE_CHECKING, Callable, Dict, List, Optional from DashAI.back.converters.category.advanced_preprocessing import ( AdvancedPreprocessingConverter, @@ -19,6 +19,7 @@ DEVICE_PLACEHOLDER, DEVICE_TO_IDX, ) +from DashAI.back.preprocessing.structure_types import StateItem, StructureDelta from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.dashai_image import DashAIImage from DashAI.back.types.value_types import Float @@ -264,6 +265,28 @@ def fit( return self + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: one block of embedding columns. + + The number of embedding columns is the encoder's hidden size, only + known once the model is loaded. The source image columns survive + only when ``keep_source_column`` is true. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The source columns (if kept), plus the embedding block. + """ + return StructureDelta( + kept=list(inputs) if self.keep_source_column else [], + added=self._default_blocks(), + ) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI type produced for a given output column. diff --git a/DashAI/back/converters/hugging_face/tokenizer.py b/DashAI/back/converters/hugging_face/tokenizer.py index 33a213a06..189f0ea4b 100644 --- a/DashAI/back/converters/hugging_face/tokenizer.py +++ b/DashAI/back/converters/hugging_face/tokenizer.py @@ -82,6 +82,7 @@ class TokenizerConverter(AdvancedPreprocessingConverter, HuggingFaceWrapper): SCHEMA = TokenizerSchema LEARNS_FROM_DATA = False + COLUMN_OPERATION = "expand" DESCRIPTION = MultilingualString( en=( "Tokenize text into input IDs; each token ID goes into its own column. " diff --git a/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py b/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py index d131b9bc3..c87a6a0b9 100644 --- a/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py +++ b/DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py @@ -1,3 +1,5 @@ +from typing import List + from sklearn.kernel_approximation import ( AdditiveChi2Sampler as AdditiveChi2SamplerOperation, ) @@ -12,6 +14,11 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + StateItem, + StructureDelta, + known_width, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -88,6 +95,26 @@ class AdditiveChi2Sampler( metadata = {"allowed_types": [Float, Integer], "allowed_dtypes": []} + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: 2 * sample_steps - 1 Float columns per input. + + Each input column is expanded into one square-root term plus a cosine + and a sine term per extra sample step. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The inputs are consumed; one block replaces them. + """ + width = known_width(inputs) + count = None if width is None else width * (2 * self.sample_steps - 1) + return StructureDelta(added=self._default_blocks(count=count)) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a column. diff --git a/DashAI/back/converters/scikit_learn/binarizer.py b/DashAI/back/converters/scikit_learn/binarizer.py index 4f738640c..f4e70f426 100644 --- a/DashAI/back/converters/scikit_learn/binarizer.py +++ b/DashAI/back/converters/scikit_learn/binarizer.py @@ -72,6 +72,7 @@ class Binarizer(EncodingConverter, SklearnWrapper, BinarizerOperation): IMAGE_PREVIEW = "binarizer.png" PREFIX = "bin_" + ONE_COLUMN_PER_INPUT = True metadata = { "allowed_types": [Float, Integer], diff --git a/DashAI/back/converters/scikit_learn/fast_ica.py b/DashAI/back/converters/scikit_learn/fast_ica.py index 0e0ab35dd..71ba72dcc 100644 --- a/DashAI/back/converters/scikit_learn/fast_ica.py +++ b/DashAI/back/converters/scikit_learn/fast_ica.py @@ -9,6 +9,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( bool_field, enum_field, @@ -173,7 +174,12 @@ class FastICASchema(BaseSchema): ) # type: ignore -class FastICA(DimensionalityReductionConverter, SklearnWrapper, FastICAOperation): +class FastICA( + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + FastICAOperation, +): """Decompose features into statistically independent components using FastICA. Independent Component Analysis (ICA) models the observed data X as a linear diff --git a/DashAI/back/converters/scikit_learn/incremental_pca.py b/DashAI/back/converters/scikit_learn/incremental_pca.py index 9761051f8..650256fe1 100644 --- a/DashAI/back/converters/scikit_learn/incremental_pca.py +++ b/DashAI/back/converters/scikit_learn/incremental_pca.py @@ -4,6 +4,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( bool_field, int_field, @@ -71,7 +72,10 @@ class IncrementalPCASchema(BaseSchema): class IncrementalPCA( - DimensionalityReductionConverter, SklearnWrapper, IncrementalPCAOperation + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + IncrementalPCAOperation, ): """Reduce dimensionality using PCA computed incrementally over mini-batches. diff --git a/DashAI/back/converters/scikit_learn/knn_imputer.py b/DashAI/back/converters/scikit_learn/knn_imputer.py index 16ca91abb..4697f5666 100644 --- a/DashAI/back/converters/scikit_learn/knn_imputer.py +++ b/DashAI/back/converters/scikit_learn/knn_imputer.py @@ -1,3 +1,5 @@ +from typing import List + from sklearn.impute import KNNImputer as KNNImputerOperation from DashAI.back.converters.category.basic_preprocessing import ( @@ -12,6 +14,10 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + StateItem, + StructureDelta, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -150,6 +156,28 @@ def __init__(self, **kwargs): """ super().__init__(**kwargs) + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope imputed in place as Float. + + With ``add_indicator``, one Float indicator column is added per + column that had missing values during fit, a count only known after + fit. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The imputed inputs, plus an indicator block if requested. + """ + delta = super().infer_output_columns(inputs) + if self.add_indicator: + delta.added = self._default_blocks() + return delta + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a column. diff --git a/DashAI/back/converters/scikit_learn/label_encoder.py b/DashAI/back/converters/scikit_learn/label_encoder.py index 8d743cf31..5912ce8a9 100644 --- a/DashAI/back/converters/scikit_learn/label_encoder.py +++ b/DashAI/back/converters/scikit_learn/label_encoder.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + from DashAI.back.preprocessing.structure_types import ColumnItem class LabelEncoderSchema(BaseSchema): @@ -66,6 +67,7 @@ class LabelEncoder(EncodingConverter, SklearnWrapper): IMAGE_PREVIEW = "label_encoder.png" PREFIX = "le_" + ONE_COLUMN_PER_INPUT = True metadata = { "changes_data_types": True, @@ -90,6 +92,14 @@ def __init__(self, **kwargs): self.encoders = {} self.fitted_columns = [] + def _encodes(self, item: "ColumnItem") -> bool: + """Whether ``fit`` will encode a scope column. + + ``fit`` only encodes columns stored as strings, so a Categorical + column of numeric labels (e.g. dtype int64) gets no encoded copy. + """ + return item.type == "Text" or item.dtype == "string" + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a column. diff --git a/DashAI/back/converters/scikit_learn/missing_indicator.py b/DashAI/back/converters/scikit_learn/missing_indicator.py index 8b1297201..133dc71c0 100644 --- a/DashAI/back/converters/scikit_learn/missing_indicator.py +++ b/DashAI/back/converters/scikit_learn/missing_indicator.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, List, Union from sklearn.impute import MissingIndicator as MissingIndicatorOperation @@ -8,6 +8,12 @@ from DashAI.back.converters.sklearn_wrapper import SklearnWrapper from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Integer @@ -52,6 +58,7 @@ class MissingIndicator( """ SCHEMA = MissingIndicatorSchema + COLUMN_OPERATION = "add" DESCRIPTION = MultilingualString( en="Binary indicators for missing values.", es="Indicadores binarios para valores faltantes.", @@ -73,6 +80,33 @@ class MissingIndicator( "allowed_dtypes": [], } + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus one indicator per column. + + ``features="all"`` is forced in ``__init__``, so every scope column + gets its ``missingindicator_`` column, missing values or not. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the indicator columns. + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + type_name, dtype = type_fields(self.get_output_type()) + added = [ + ColumnItem( + name=f"missingindicator_{item.name}", type=type_name, dtype=dtype + ) + for item in inputs + ] + return StructureDelta(kept=list(inputs), added=added) + def __init__(self, **kwargs): """Initialize the MissingIndicator converter. diff --git a/DashAI/back/converters/scikit_learn/nystroem.py b/DashAI/back/converters/scikit_learn/nystroem.py index 7aef4b009..c3600af6b 100644 --- a/DashAI/back/converters/scikit_learn/nystroem.py +++ b/DashAI/back/converters/scikit_learn/nystroem.py @@ -5,6 +5,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( enum_field, float_field, @@ -132,7 +133,12 @@ class NystroemSchema(BaseSchema): ) # type: ignore -class Nystroem(DimensionalityReductionConverter, SklearnWrapper, NystroemOperation): +class Nystroem( + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + NystroemOperation, +): """Approximate a kernel feature map using the Nystroem method. The Nystroem method constructs an explicit low-dimensional feature map diff --git a/DashAI/back/converters/scikit_learn/ordinal_encoder.py b/DashAI/back/converters/scikit_learn/ordinal_encoder.py index 04f936f89..410a9da73 100644 --- a/DashAI/back/converters/scikit_learn/ordinal_encoder.py +++ b/DashAI/back/converters/scikit_learn/ordinal_encoder.py @@ -161,6 +161,7 @@ class OrdinalEncoder(EncodingConverter, SklearnWrapper, OrdinalEncoderOperation) IMAGE_PREVIEW = "ordinal_encoder.png" PREFIX = "oe_" + ONE_COLUMN_PER_INPUT = True metadata = { "allowed_types": [Categorical], diff --git a/DashAI/back/converters/scikit_learn/pca.py b/DashAI/back/converters/scikit_learn/pca.py index a55042624..193933825 100644 --- a/DashAI/back/converters/scikit_learn/pca.py +++ b/DashAI/back/converters/scikit_learn/pca.py @@ -5,6 +5,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( bool_field, enum_field, @@ -200,7 +201,12 @@ class PCASchema(BaseSchema): ) # type: ignore -class PCA(DimensionalityReductionConverter, SklearnWrapper, PCAOPERATION): +class PCA( + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + PCAOPERATION, +): """Reduce dimensionality using Principal Component Analysis (PCA). PCA finds a set of orthogonal axes (principal components) that successively diff --git a/DashAI/back/converters/scikit_learn/polynomial_features.py b/DashAI/back/converters/scikit_learn/polynomial_features.py index 3788b6274..4be63d11c 100644 --- a/DashAI/back/converters/scikit_learn/polynomial_features.py +++ b/DashAI/back/converters/scikit_learn/polynomial_features.py @@ -1,3 +1,6 @@ +import math +from typing import List + from sklearn.preprocessing import PolynomialFeatures as PolynomialFeaturesOperation from DashAI.back.converters.category.polynomial_kernel import PolynomialKernelConverter @@ -10,6 +13,11 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + StateItem, + StructureDelta, + known_width, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -176,6 +184,40 @@ class PolynomialFeatures( metadata = {"allowed_types": [Float, Integer], "allowed_dtypes": []} + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: inputs kept as Float, plus the other terms. + + The degree-1 terms keep their input column's name, so the inputs + survive (as Float); the bias column and higher-degree terms are new. + With n inputs there are C(n + degree, degree) monomials of degree at + most `degree`, or sum(C(n, k) for k <= degree) products of distinct + features when `interaction_only`, one less without the bias. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The retyped inputs, plus one block for the new terms. + """ + width = known_width(inputs) + count = None + if width is not None and isinstance(self.degree, int): + if self.interaction_only: + total = sum(math.comb(width, k) for k in range(self.degree + 1)) + else: + total = math.comb(width + self.degree, self.degree) + if not self.include_bias: + total -= 1 + count = total - width + return StructureDelta( + kept=[self._retype(item) for item in inputs], + added=self._default_blocks(count=count), + ) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return ``Float64`` as the output type for all polynomial feature columns. diff --git a/DashAI/back/converters/scikit_learn/rbf_sampler.py b/DashAI/back/converters/scikit_learn/rbf_sampler.py index 509306e06..d0f328a9e 100644 --- a/DashAI/back/converters/scikit_learn/rbf_sampler.py +++ b/DashAI/back/converters/scikit_learn/rbf_sampler.py @@ -3,6 +3,7 @@ from DashAI.back.api.utils import create_random_state from DashAI.back.converters.category.polynomial_kernel import PolynomialKernelConverter from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( enum_field, float_field, @@ -80,7 +81,12 @@ class RBFSamplerSchema(BaseSchema): ) # type: ignore -class RBFSampler(PolynomialKernelConverter, SklearnWrapper, RBFSamplerOperation): +class RBFSampler( + ComponentsOutputMixin, + PolynomialKernelConverter, + SklearnWrapper, + RBFSamplerOperation, +): """Approximate the RBF (Gaussian) kernel feature map via random Fourier features. The Radial Basis Function (RBF) kernel is one of the most widely used diff --git a/DashAI/back/converters/scikit_learn/select_k_best.py b/DashAI/back/converters/scikit_learn/select_k_best.py index 1bffc79f7..14601934c 100644 --- a/DashAI/back/converters/scikit_learn/select_k_best.py +++ b/DashAI/back/converters/scikit_learn/select_k_best.py @@ -1,3 +1,5 @@ +from typing import List, Optional + from sklearn.feature_selection import SelectKBest as SelectKBestOperation from DashAI.back.converters.category.feature_selection import FeatureSelectionConverter @@ -10,6 +12,7 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import StateItem, known_width from DashAI.back.types.value_types import Float, Integer @@ -92,3 +95,11 @@ def __init__(self, **kwargs): schema fields. Forwarded to the underlying scikit-learn class. """ super().__init__(**kwargs) + + def _selected_count(self, inputs: List[StateItem]) -> Optional[int]: + """Keep exactly ``k`` columns, or every column for ``k="all"``.""" + if self.k == "all": + return known_width(inputs) + if isinstance(self.k, int) and not isinstance(self.k, bool): + return self.k + return None diff --git a/DashAI/back/converters/scikit_learn/simple_imputer.py b/DashAI/back/converters/scikit_learn/simple_imputer.py index d3c4a1017..3f5833405 100644 --- a/DashAI/back/converters/scikit_learn/simple_imputer.py +++ b/DashAI/back/converters/scikit_learn/simple_imputer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, List, Union from sklearn.impute import SimpleImputer as SimpleImputerOperation @@ -24,6 +24,12 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.categorical import Categorical from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -181,12 +187,11 @@ class SimpleImputer( ``MissingIndicator`` binary matrix is stacked onto the output. Output typing preserves the original column type whenever the strategy - does not force a fractional result: ``"most_frequent"`` and - ``"constant"`` never perform arithmetic, so the source type (Integer, - Float, or Categorical) is kept. For ``"mean"``/``"median"`` the computed - per-column statistic is inspected, and an originally-Integer column - stays ``Integer`` if that statistic happens to be a whole number (e.g. a - median over an odd count of integers); otherwise it becomes ``Float64``. + does not perform arithmetic: ``"most_frequent"`` and ``"constant"`` only + copy existing values, so the source type (Integer, Float, or + Categorical) is kept. ``"mean"``/``"median"`` always produce ``Float64``, + even when the statistic happens to be a whole number, so the output type + follows from the params alone and never from the data. Wraps ``sklearn.impute.SimpleImputer``. @@ -254,10 +259,10 @@ def __init__(self, **kwargs): def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None ) -> "SimpleImputer": - """Fit the imputer, remembering input types and column order. + """Fit the imputer, remembering the input column types. - These are needed by ``get_output_type`` to preserve the original - column type instead of always coercing to ``Float64``. + ``get_output_type`` needs them to preserve the original column type + for the "most_frequent" and "constant" strategies. Parameters ---------- @@ -273,9 +278,42 @@ def fit( """ if hasattr(x, "types") and x.types is not None: self._input_types = dict(x.types) - self._input_columns = {name: idx for idx, name in enumerate(x.column_names)} return super().fit(x, y) + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope imputed in place, plus indicators. + + "most_frequent" and "constant" only copy existing values, so every + column keeps its type; "mean" and "median" produce Float. With + ``add_indicator``, one indicator column is added per column that had + missing values during fit, a count only known after fit. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The imputed inputs, plus an indicator block if requested. + """ + import pyarrow as pa + + if self.strategy in ("most_frequent", "constant"): + kept = list(inputs) + else: + type_name, dtype = type_fields(Float(arrow_type=pa.float64())) + kept = [ + item.model_copy(update={"type": type_name, "dtype": dtype}) + for item in inputs + ] + added = [] + if self.add_indicator: + type_name, dtype = type_fields(Integer(arrow_type=pa.int64())) + added = [BlockItem(label="missingindicator_*", type=type_name, dtype=dtype)] + return StructureDelta(kept=kept, added=added) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI data type produced by this converter for a column. @@ -290,10 +328,10 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType: ``Integer`` for the binary ``MissingIndicator`` columns appended when ``add_indicator=True``. Otherwise, the original column type for ``"most_frequent"``/``"constant"`` (no arithmetic is performed - on the values), or for ``"mean"``/``"median"``, ``Integer`` if the - source column was an Integer and the computed statistic is a - whole number — otherwise a Float type backed by - ``pyarrow.float64()``. + on the values), and a Float type backed by ``pyarrow.float64()`` + for ``"mean"``/``"median"``. The type never depends on the fitted + statistics, so it can be known before fitting (see + ``infer_output_columns``). """ import pyarrow as pa @@ -308,16 +346,4 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType: return input_type return Float(arrow_type=pa.float64()) - if isinstance(input_type, Integer): - columns = getattr(self, "_input_columns", None) - statistics = getattr(self, "statistics_", None) - if ( - columns is not None - and statistics is not None - and column_name in columns - ): - value = statistics[columns[column_name]] - if float(value).is_integer(): - return Integer(arrow_type=pa.int64()) - return Float(arrow_type=pa.float64()) diff --git a/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py b/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py index e97e1733e..cf519365f 100644 --- a/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py +++ b/DashAI/back/converters/scikit_learn/skewed_chi_2_sampler.py @@ -3,6 +3,7 @@ from DashAI.back.api.utils import create_random_state from DashAI.back.converters.category.polynomial_kernel import PolynomialKernelConverter from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( enum_field, float_field, @@ -94,7 +95,10 @@ class SkewedChi2SamplerSchema(BaseSchema): class SkewedChi2Sampler( - PolynomialKernelConverter, SklearnWrapper, SkewedChi2SamplerOperation + ComponentsOutputMixin, + PolynomialKernelConverter, + SklearnWrapper, + SkewedChi2SamplerOperation, ): """Approximate the skewed chi-squared kernel feature map via random Fourier features. diff --git a/DashAI/back/converters/scikit_learn/truncated_svd.py b/DashAI/back/converters/scikit_learn/truncated_svd.py index b25195168..59d53ad0e 100644 --- a/DashAI/back/converters/scikit_learn/truncated_svd.py +++ b/DashAI/back/converters/scikit_learn/truncated_svd.py @@ -5,6 +5,7 @@ DimensionalityReductionConverter, ) from DashAI.back.converters.sklearn_wrapper import SklearnWrapper +from DashAI.back.converters.structure_mixins import ComponentsOutputMixin from DashAI.back.core.schema_fields import ( enum_field, float_field, @@ -127,7 +128,10 @@ class TruncatedSVDSchema(BaseSchema): class TruncatedSVD( - DimensionalityReductionConverter, SklearnWrapper, TruncatedSVDOperation + ComponentsOutputMixin, + DimensionalityReductionConverter, + SklearnWrapper, + TruncatedSVDOperation, ): """Reduce dimensionality using Truncated Singular Value Decomposition (LSA). diff --git a/DashAI/back/converters/scikit_learn/variance_threshold.py b/DashAI/back/converters/scikit_learn/variance_threshold.py index 335893904..916f140b0 100644 --- a/DashAI/back/converters/scikit_learn/variance_threshold.py +++ b/DashAI/back/converters/scikit_learn/variance_threshold.py @@ -93,6 +93,7 @@ class VarianceThreshold( zh="方差阈值", ) PRESERVES_INPUT_TYPE = True + COLUMN_OPERATION = "select" def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/converters/segmentation/sam3_segment_converter.py b/DashAI/back/converters/segmentation/sam3_segment_converter.py index 12639d1cf..e14ec2358 100644 --- a/DashAI/back/converters/segmentation/sam3_segment_converter.py +++ b/DashAI/back/converters/segmentation/sam3_segment_converter.py @@ -1,7 +1,7 @@ """Promptable instance segmentation converter that appends ranked columns.""" import io -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Dict, List, Optional from kink import di @@ -27,6 +27,12 @@ DEVICE_PLACEHOLDER, DEVICE_TO_IDX, ) +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.segmenters.rendering import render_binary_mask, render_segment from DashAI.back.segmenters.sam3_segmenter import SAM3Segmenter from DashAI.back.types.dashai_data_type import DashAIDataType @@ -816,6 +822,36 @@ def transform( return DashAIDataset(result_table, splits=x.splits, types=output_types) + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus one column set per rank. + + ``max_masks`` fixes how many ``segment_i`` and ``seg_score_i`` + columns (and ``mask_i`` with ``keep_binary_mask``) are appended, + whatever the model detects, so every name is known ahead of fit. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the segment, score and mask columns. + """ + names = [ + name + for rank in range(1, self.max_masks + 1) + for name in (_segment_column(rank), _score_column(rank)) + ] + if self.keep_binary_mask: + names += [_mask_column(rank) for rank in range(1, self.max_masks + 1)] + added = [] + for name in names: + type_name, dtype = type_fields(self.get_output_type(name)) + added.append(ColumnItem(name=name, type=type_name, dtype=dtype)) + return StructureDelta(kept=list(inputs), added=added) + def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the DashAI type produced for a given output column. diff --git a/DashAI/back/converters/simple_converters/character_replacer.py b/DashAI/back/converters/simple_converters/character_replacer.py index c02bd41eb..739dc9603 100644 --- a/DashAI/back/converters/simple_converters/character_replacer.py +++ b/DashAI/back/converters/simple_converters/character_replacer.py @@ -9,7 +9,7 @@ from DashAI.back.core.utils import MultilingualString from DashAI.back.types.categorical import Categorical from DashAI.back.types.dashai_data_type import DashAIDataType -from DashAI.back.types.value_types import Integer, Text +from DashAI.back.types.value_types import Text if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset @@ -61,13 +61,16 @@ class CharacterReplacer(BasicPreprocessingConverter, BaseConverter): """Replace or remove a character or substring in all selected text columns. Scans each value in the configured string columns and substitutes every - occurrence of ``char_to_replace`` with ``replacement_char``. If the - replacement produces a column of pure integers, the column type is promoted - to ``Integer``. + occurrence of ``char_to_replace`` with ``replacement_char``. Every column + keeps its type, even when the replacement leaves only digits: follow up + with TypeCast to turn such a column into ``Integer``. Deciding that from + the values would make the output type depend on the data (and on each + batch), so it could not be known before fitting. """ SCHEMA = CharacterReplacerSchema LEARNS_FROM_DATA = False + PRESERVES_INPUT_TYPE = True DESCRIPTION = MultilingualString( en=( "Replaces or removes specified characters/substrings in selected " @@ -182,37 +185,15 @@ def transform( Returns ------- DashAIDataset - A new dataset with ``char_to_replace`` substituted in all text columns. - If replacement yields only integer-like values, the column type is - promoted to ``Integer``. + A new dataset with ``char_to_replace`` substituted in all text + columns, each keeping its type. """ - import pyarrow as pa - from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset if not self._target_columns: # if no target columns were set, return the dataset unchanged return x - def try_convert_to_int(value): - """Try to convert a value to integer, return the original if not possible. - - Parameters - ---------- - value : object - The value to convert. - - Returns - ------- - int or object - ``int(value)`` on success, or the original ``value`` unchanged - if a ``ValueError`` or ``TypeError`` is raised. - """ - try: - return int(value) - except (ValueError, TypeError): - return value - new_types = x.types.copy() categorical_new_values: dict = { col: set() @@ -225,40 +206,16 @@ def replace_function(batch): processed_batch = {} for column_name, values in batch.items(): if column_name in self._target_columns: - if isinstance(x.types[column_name], Text): - replaced_values = [ - ( - val.replace(self.char_to_replace, self.replacement_char) - if isinstance(val, str) - else val - ) - for val in values - ] - - all_numeric = all( - isinstance(val, str) and val.strip().isdigit() - for val in replaced_values + replaced_values = [ + ( + val.replace(self.char_to_replace, self.replacement_char) if isinstance(val, str) + else val ) - - if all_numeric: - processed_batch[column_name] = [ - try_convert_to_int(val) for val in replaced_values - ] - new_types[column_name] = Integer(arrow_type=pa.int64()) - else: - processed_batch[column_name] = replaced_values - new_types[column_name] = Text(arrow_type=pa.string()) - else: - replaced_values = [ - ( - val.replace(self.char_to_replace, self.replacement_char) - if isinstance(val, str) - else val - ) - for val in values - ] - processed_batch[column_name] = replaced_values + for val in values + ] + processed_batch[column_name] = replaced_values + if column_name in categorical_new_values: categorical_new_values[column_name].update( v for v in replaced_values if v is not None ) @@ -285,8 +242,8 @@ def replace_function(batch): def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the default output type for a transformed column. - The actual type may be ``Integer`` if all values become numeric during - ``transform``, but the static default is ``Text``. + A transformed column keeps its own input type (``Text`` or + ``Categorical``); ``Text`` is only the default with no column given. Parameters ---------- diff --git a/DashAI/back/converters/simple_converters/column_arithmetic.py b/DashAI/back/converters/simple_converters/column_arithmetic.py index e0b065b4c..f156b15c0 100644 --- a/DashAI/back/converters/simple_converters/column_arithmetic.py +++ b/DashAI/back/converters/simple_converters/column_arithmetic.py @@ -1,5 +1,5 @@ import math -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.converters.category.feature_engineering import ( @@ -15,6 +15,12 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -154,13 +160,13 @@ class ColumnArithmetic(FeatureEngineeringConverter, BaseConverter): ``__`` or ``__``. The output column is ``Integer`` when both operands are ``Integer`` (a - whole-number ``constant`` counts as ``Integer``), the operation is + whole-number ``constant`` counts as ``Integer``) and the operation is ``add``, ``subtract``, or ``multiply`` (all of which stay exact on - integers), and neither operand column has missing values (since a - missing value has no exact integer representation). ``divide`` always + integers); a missing operand gives a missing result. ``divide`` always produces a ``Float`` column, since integer division is not exact in - general, and any operation involving a ``Float`` operand, or an - operand column with missing values, also produces a ``Float`` column. + general, and any operation involving a ``Float`` operand also produces a + ``Float`` column. The type never depends on the values, so it is known + before fitting (see ``infer_output_columns``). """ SCHEMA = ColumnArithmeticSchema @@ -307,7 +313,46 @@ def fit( selected column is not numeric (Float or Integer), or if a single column is selected without a valid ``constant``. """ - columns = list(x.column_names) + types = {name: type(x.types.get(name)).__name__ for name in x.column_names} + ( + self.operand_b_mode, + self.column_a, + self.column_b, + self._result_column_name, + self._output_is_integer, + ) = self._plan(list(x.column_names), types) + if self.operand_b_mode == "constant": + self.constant = float(self.constant) + return self + + def _plan( + self, columns: List[str], types: Dict[str, str] + ) -> Tuple[str, str, Optional[str], str, bool]: + """Work out the operands, result name and result type from the scope. + + Shared by ``fit`` and ``infer_output_columns``, so the estimated + column is exactly the one ``transform`` appends. + + Parameters + ---------- + columns : List[str] + The scope column names, in dataset order. + types : Dict[str, str] + Each scope column's DashAI type name (e.g. "Integer"). + + Returns + ------- + tuple + (operand_b_mode, column_a, column_b, result_column_name, + output_is_integer); column_b is None in "constant" mode. + + Raises + ------ + ValueError + If the number of selected columns is not one or two, if a + selected column is not numeric (Float or Integer), or if a + single column is selected without a valid ``constant``. + """ if len(columns) not in (1, 2): raise ValueError( "ColumnArithmetic requires selecting one or two columns in " @@ -315,25 +360,19 @@ def fit( ) for col in columns: - if not isinstance(x.types.get(col), (Float, Integer)): + if types.get(col) not in ("Float", "Integer"): raise ValueError( f"Column '{col}' must be numeric (Float or Integer) to be " "used in ColumnArithmetic." ) if len(columns) == 2: - self.operand_b_mode = "column" - first, second = columns + operand_b_mode = "column" + column_a, column_b = columns if self.swap_operands: - first, second = second, first - self.column_a = first - self.column_b = second - operand_b_label = self.column_b - operand_b_is_integer = isinstance(x.types.get(self.column_b), Integer) - has_missing_values = ( - x.arrow_table[self.column_a].null_count > 0 - or x.arrow_table[self.column_b].null_count > 0 - ) + column_a, column_b = column_b, column_a + operand_b_label = column_b + operand_b_is_integer = types[column_b] == "Integer" else: if not isinstance(self.constant, (int, float)) or isinstance( self.constant, bool @@ -347,24 +386,55 @@ def fit( "'constant' must be a finite number (not NaN or " "infinite) when a single column is selected in scope." ) - self.operand_b_mode = "constant" - self.constant = float(self.constant) - self.column_a = columns[0] - self.column_b = None - operand_b_label = self._format_constant(self.constant) + operand_b_mode = "constant" + column_a, column_b = columns[0], None + operand_b_label = self._format_constant(float(self.constant)) operand_b_is_integer = self.constant == int(self.constant) - has_missing_values = x.arrow_table[self.column_a].null_count > 0 - self._result_column_name = self.output_column_name or ( - f"{self.column_a}_{self.operation}_{operand_b_label}" + result_column_name = self.output_column_name or ( + f"{column_a}_{self.operation}_{operand_b_label}" ) - self._output_is_integer = ( + output_is_integer = ( self.operation != "divide" - and isinstance(x.types.get(self.column_a), Integer) + and types[column_a] == "Integer" and operand_b_is_integer - and not has_missing_values ) - return self + return ( + operand_b_mode, + column_a, + column_b, + result_column_name, + output_is_integer, + ) + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus the result column. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the single result column. + + Raises + ------ + ValueError + If the scope or ``constant`` is invalid (see ``_plan``). + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + _, _, _, name, is_integer = self._plan( + [item.name for item in inputs], {item.name: item.type for item in inputs} + ) + type_name, dtype = type_fields(self._result_type(is_integer)) + return StructureDelta( + kept=list(inputs), + added=[ColumnItem(name=name, type=type_name, dtype=dtype)], + ) def _get_operand_b(self, x_pandas, dtype: str): """Return the second operand as an array aligned with ``x_pandas``. @@ -400,23 +470,26 @@ def transform( """ import numpy as np import pyarrow as pa + import pyarrow.compute as pc from DashAI.back.dataloaders.classes.dashai_dataset import modify_table x_pandas = x.to_pandas() if self._output_is_integer: - a = x_pandas[self.column_a].to_numpy(dtype="int64") - b = self._get_operand_b(x_pandas, "int64") - - if self.operation == "add": - result = a + b - elif self.operation == "subtract": - result = a - b - else: # multiply - result = a * b - - arrow_type = pa.int64() + # Arrow arithmetic keeps a missing operand as a missing result, + # which a numpy int64 array cannot represent. + a = x.arrow_table[self.column_a].cast(pa.int64()) + if self.operand_b_mode == "column": + b = x.arrow_table[self.column_b].cast(pa.int64()) + else: + b = pa.scalar(int(self.constant), type=pa.int64()) + operation = { + "add": pc.add, + "subtract": pc.subtract, + "multiply": pc.multiply, + }[self.operation] + result = operation(a, b) else: a = x_pandas[self.column_a].to_numpy(dtype="float64") b = self._get_operand_b(x_pandas, "float64") @@ -431,21 +504,18 @@ def transform( else: # divide result = np.where(b != 0, a / b, np.nan) - arrow_type = pa.float64() + result = pa.array(result, type=pa.float64()) new_types = dict(x.types) new_types[self._result_column_name] = self.get_output_type() - return modify_table( - x, - {self._result_column_name: pa.array(result, type=arrow_type)}, - types=new_types, - ) + return modify_table(x, {self._result_column_name: result}, types=new_types) def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the output type for the arithmetic result. - Determined during ``fit``: ``Integer`` when both operands are + Determined during ``fit`` (or ``infer_output_columns``): ``Integer`` + when both operands are ``Integer`` (a whole-number ``constant`` counts as ``Integer``) and the operation isn't ``divide``, ``Float`` otherwise. @@ -461,8 +531,13 @@ def get_output_type(self, column_name: str = None) -> DashAIDataType: An ``Integer`` type backed by ``pyarrow.int64()``, or a ``Float`` type backed by ``pyarrow.float64()``. """ + return self._result_type(self._output_is_integer) + + @staticmethod + def _result_type(is_integer: bool) -> DashAIDataType: + """Return ``Integer`` or ``Float`` for the result column.""" import pyarrow as pa - if self._output_is_integer: + if is_integer: return Integer(arrow_type=pa.int64()) return Float(arrow_type=pa.float64()) diff --git a/DashAI/back/converters/simple_converters/column_concat.py b/DashAI/back/converters/simple_converters/column_concat.py index d70659808..c71731931 100644 --- a/DashAI/back/converters/simple_converters/column_concat.py +++ b/DashAI/back/converters/simple_converters/column_concat.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.converters.category.feature_engineering import ( @@ -12,6 +12,12 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.categorical import Categorical from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Text @@ -269,7 +275,43 @@ def fit( selected column is not string-like (Text or Categorical), or if a single column is selected without a valid ``constant``. """ - columns = list(x.column_names) + types = {name: type(x.types.get(name)).__name__ for name in x.column_names} + ( + self.operand_b_mode, + self.column_a, + self.column_b, + self._result_column_name, + ) = self._plan(list(x.column_names), types) + return self + + def _plan( + self, columns: List[str], types: Dict[str, str] + ) -> Tuple[str, str, Optional[str], str]: + """Work out the operands and the result name from the scope. + + Shared by ``fit`` and ``infer_output_columns``, so the estimated + column is exactly the one ``transform`` appends. + + Parameters + ---------- + columns : List[str] + The scope column names, in dataset order. + types : Dict[str, str] + Each scope column's DashAI type name (e.g. "Text"). + + Returns + ------- + tuple + (operand_b_mode, column_a, column_b, result_column_name); + column_b is None in "constant" mode. + + Raises + ------ + ValueError + If the number of selected columns is not one or two, if a + selected column is not string-like (Text or Categorical), or if + a single column is selected without a valid ``constant``. + """ if len(columns) not in (1, 2): raise ValueError( "ColumnConcat requires selecting one or two columns in " @@ -277,35 +319,61 @@ def fit( ) for col in columns: - if not isinstance(x.types.get(col), (Text, Categorical)): + if types.get(col) not in ("Text", "Categorical"): raise ValueError( f"Column '{col}' must be a string (Text or Categorical) " "to be used in ColumnConcat." ) if len(columns) == 2: - self.operand_b_mode = "column" - first, second = columns + operand_b_mode = "column" + column_a, column_b = columns if self.swap_operands: - first, second = second, first - self.column_a = first - self.column_b = second - operand_b_label = self.column_b + column_a, column_b = column_b, column_a + operand_b_label = column_b else: if not isinstance(self.constant, str): raise ValueError( "'constant' must be a string when a single column is " "selected in scope." ) - self.operand_b_mode = "constant" - self.column_a = columns[0] - self.column_b = None + operand_b_mode = "constant" + column_a, column_b = columns[0], None operand_b_label = self.constant - self._result_column_name = self.output_column_name or ( - f"{self.column_a}_concat_{operand_b_label}" + result_column_name = self.output_column_name or ( + f"{column_a}_concat_{operand_b_label}" + ) + return operand_b_mode, column_a, column_b, result_column_name + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus the Text result column. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus the single result column. + + Raises + ------ + ValueError + If the scope or ``constant`` is invalid (see ``_plan``). + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + _, _, _, name = self._plan( + [item.name for item in inputs], {item.name: item.type for item in inputs} + ) + type_name, dtype = type_fields(self.get_output_type()) + return StructureDelta( + kept=list(inputs), + added=[ColumnItem(name=name, type=type_name, dtype=dtype)], ) - return self def transform( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/converters/simple_converters/column_remover.py b/DashAI/back/converters/simple_converters/column_remover.py index 2f2558a3b..776cd9dd6 100644 --- a/DashAI/back/converters/simple_converters/column_remover.py +++ b/DashAI/back/converters/simple_converters/column_remover.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.converters.category.basic_preprocessing import ( @@ -6,6 +6,7 @@ ) from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import StateItem, StructureDelta from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Text @@ -27,6 +28,7 @@ class ColumnRemover(BasicPreprocessingConverter, BaseConverter): SCHEMA = ColumnRemoverSchema LEARNS_FROM_DATA = False + COLUMN_OPERATION = "select" DESCRIPTION = MultilingualString( en="Removes the columns selected in scope from the dataset.", es="Elimina las columnas seleccionadas en el alcance del conjunto de datos.", @@ -63,6 +65,21 @@ def __init__(self): super().__init__() self.columns = [] + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: every scope column is removed, nothing added. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + An empty delta: no input survives and no column is created. + """ + return StructureDelta() + def fit(self, x: "DashAIDataset", y: "DashAIDataset" = None) -> "ColumnRemover": """Record the column names that will be removed during ``transform``. diff --git a/DashAI/back/converters/simple_converters/date_features.py b/DashAI/back/converters/simple_converters/date_features.py index 8dd8e6298..7d7b0acb7 100644 --- a/DashAI/back/converters/simple_converters/date_features.py +++ b/DashAI/back/converters/simple_converters/date_features.py @@ -7,6 +7,12 @@ from DashAI.back.core.schema_fields import bool_field, schema_field from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Date, Float, Integer @@ -344,6 +350,57 @@ def _selected_features(self) -> List[str]: """ return [name for name in FEATURE_SUFFIXES if self.features[name]] + def _feature_columns(self, date_columns: List[str]) -> Dict[str, DashAIDataType]: + """Name and type every column added for the given date columns. + + Parameters + ---------- + date_columns : List[str] + The date columns to extract features from. + + Returns + ------- + Dict[str, DashAIDataType] + New column name to its type, in the order they are written out: + one Integer column per selected feature, plus a Float sine and + cosine column per periodic feature when cyclical encoding is on. + """ + import pyarrow as pa + + columns: Dict[str, DashAIDataType] = {} + for column in date_columns: + for feature in self._selected_features(): + suffix = FEATURE_SUFFIXES[feature] + columns[f"{column}_{suffix}"] = Integer(arrow_type=pa.int64()) + if self.cyclical and feature in CYCLICAL_PERIODS: + columns[f"{column}_{suffix}_sin"] = Float(arrow_type=pa.float64()) + columns[f"{column}_{suffix}_cos"] = Float(arrow_type=pa.float64()) + return columns + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus the calendar features. + + Only Date columns get features, exactly as ``fit`` picks them. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus one column per feature and date column. + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + date_columns = [item.name for item in inputs if item.type == "Date"] + added = [] + for name, dashai_type in self._feature_columns(date_columns).items(): + type_name, dtype = type_fields(dashai_type) + added.append(ColumnItem(name=name, type=type_name, dtype=dtype)) + return StructureDelta(kept=list(inputs), added=added) + def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None ) -> "DateFeaturesConverter": @@ -361,25 +418,10 @@ def fit( DateFeaturesConverter The fitted converter instance (self). """ - import pyarrow as pa - self._date_columns = [ name for name in x.column_names if isinstance(x.types.get(name), Date) ] - self._output_types = {} - for column in self._date_columns: - for feature in self._selected_features(): - suffix = FEATURE_SUFFIXES[feature] - self._output_types[f"{column}_{suffix}"] = Integer( - arrow_type=pa.int64() - ) - if self.cyclical and feature in CYCLICAL_PERIODS: - self._output_types[f"{column}_{suffix}_sin"] = Float( - arrow_type=pa.float64() - ) - self._output_types[f"{column}_{suffix}_cos"] = Float( - arrow_type=pa.float64() - ) + self._output_types = self._feature_columns(self._date_columns) if not self._date_columns: print( diff --git a/DashAI/back/converters/simple_converters/nan_remover.py b/DashAI/back/converters/simple_converters/nan_remover.py index 8ca5e3256..a9cb5506a 100644 --- a/DashAI/back/converters/simple_converters/nan_remover.py +++ b/DashAI/back/converters/simple_converters/nan_remover.py @@ -42,6 +42,7 @@ class NanRemover(BasicPreprocessingConverter, BaseConverter): SCHEMA = NanRemoverSchema CHANGES_ROW_COUNT = True + COLUMN_OPERATION = "rows" LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( diff --git a/DashAI/back/converters/simple_converters/numeric_expansion.py b/DashAI/back/converters/simple_converters/numeric_expansion.py index f5dd4643b..742c9e4df 100644 --- a/DashAI/back/converters/simple_converters/numeric_expansion.py +++ b/DashAI/back/converters/simple_converters/numeric_expansion.py @@ -7,6 +7,12 @@ from DashAI.back.core.schema_fields import enum_field, schema_field from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + ColumnItem, + StateItem, + StructureDelta, + type_fields, +) from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer @@ -58,13 +64,11 @@ class NumericExpansion(FeatureEngineeringConverter, BaseConverter): ``sqrt``) become ``NaN`` in the corresponding output. The original columns are left untouched. ``square`` preserves the input - column's type (``Integer`` stays ``Integer``, ``Float`` stays ``Float``) - when the source column has no missing values, since squaring is exact - for both. ``log1p`` and ``sqrt`` always produce a ``Float`` column, since - they can yield non-integer or ``NaN`` results even from integer input, - and ``square`` also falls back to ``Float`` when the source ``Integer`` - column has missing values (since a missing value has no exact integer - representation). + column's type (``Integer`` stays ``Integer``, ``Float`` stays ``Float``), + since squaring is exact for both; a missing value stays missing. ``log1p`` + and ``sqrt`` always produce a ``Float`` column, since they can yield + non-integer or ``NaN`` results even from integer input. The output type + only depends on the operation and the input type, never on the values. """ SCHEMA = NumericExpansionSchema @@ -158,24 +162,15 @@ def fit( NumericExpansion The fitted converter instance (self). """ - import pyarrow as pa - self._target_columns = [] self._output_types = {} for col_name in x.column_names: col_type = x.types.get(col_name) if isinstance(col_type, (Float, Integer)): self._target_columns.append(col_name) - new_col_name = f"{self.operation}_{col_name}" - has_missing_values = x.arrow_table[col_name].null_count > 0 - if ( - self.operation == "square" - and isinstance(col_type, Integer) - and not has_missing_values - ): - self._output_types[new_col_name] = Integer(arrow_type=pa.int64()) - else: - self._output_types[new_col_name] = Float(arrow_type=pa.float64()) + self._output_types[f"{self.operation}_{col_name}"] = ( + self._expansion_type(isinstance(col_type, Integer)) + ) else: print( f"Warning: Column '{col_name}' in scope is not numeric " @@ -188,6 +183,54 @@ def fit( ) return self + def _expansion_type(self, input_is_integer: bool) -> DashAIDataType: + """Return the type of an expanded column. + + Parameters + ---------- + input_is_integer : bool + Whether the source column is ``Integer`` (otherwise ``Float``). + + Returns + ------- + DashAIDataType + ``Integer`` for the square of an ``Integer`` column, ``Float`` + otherwise. + """ + import pyarrow as pa + + if self.operation == "square" and input_is_integer: + return Integer(arrow_type=pa.int64()) + return Float(arrow_type=pa.float64()) + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope kept, plus one column per numeric one. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The unchanged inputs, plus an ``_`` column for + every Float or Integer input, as ``fit`` picks them. + """ + if not all(isinstance(item, ColumnItem) for item in inputs): + return super().infer_output_columns(inputs) + added = [] + for item in inputs: + if item.type not in ("Float", "Integer"): + continue + type_name, dtype = type_fields(self._expansion_type(item.type == "Integer")) + added.append( + ColumnItem( + name=f"{self.operation}_{item.name}", type=type_name, dtype=dtype + ) + ) + return StructureDelta(kept=list(inputs), added=added) + def transform( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None ) -> "DashAIDataset": @@ -210,6 +253,7 @@ def transform( """ import numpy as np import pyarrow as pa + import pyarrow.compute as pc from DashAI.back.dataloaders.classes.dashai_dataset import modify_table @@ -226,9 +270,10 @@ def transform( output_type = self._output_types[new_col_name] if isinstance(output_type, Integer): - values = x_pandas[col].to_numpy(dtype="int64") - result = values**2 - arrow_type = pa.int64() + # Arrow arithmetic keeps nulls as nulls, which a numpy + # int64 array cannot represent. + values = x.arrow_table[col].cast(pa.int64()) + new_columns[new_col_name] = pc.multiply(values, values) else: values = x_pandas[col].to_numpy(dtype="float64") if self.operation == "log1p": @@ -237,9 +282,7 @@ def transform( result = values**2 else: # sqrt result = np.where(values >= 0, np.sqrt(values), np.nan) - arrow_type = pa.float64() - - new_columns[new_col_name] = pa.array(result, type=arrow_type) + new_columns[new_col_name] = pa.array(result, type=pa.float64()) new_types[new_col_name] = output_type return modify_table(x, new_columns, types=new_types) @@ -247,9 +290,8 @@ def transform( def get_output_type(self, column_name: str = None) -> DashAIDataType: """Return the output type for a given expanded column. - Determined during ``fit``: ``Integer`` when the operation is - ``square`` and the source column is ``Integer``, ``Float`` - otherwise. + ``Integer`` when the operation is ``square`` and the source column is + ``Integer``, ``Float`` otherwise. Parameters ---------- diff --git a/DashAI/back/converters/simple_converters/time_resampler.py b/DashAI/back/converters/simple_converters/time_resampler.py index 3f5cbf5ba..4a5c3d325 100644 --- a/DashAI/back/converters/simple_converters/time_resampler.py +++ b/DashAI/back/converters/simple_converters/time_resampler.py @@ -156,6 +156,7 @@ class TimeResamplerConverter(BasicPreprocessingConverter, BaseConverter): SCHEMA = TimeResamplerSchema CHANGES_ROW_COUNT = True + COLUMN_OPERATION = "rows" LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( diff --git a/DashAI/back/converters/simple_converters/time_series_window.py b/DashAI/back/converters/simple_converters/time_series_window.py index a15568102..e03b79b8e 100644 --- a/DashAI/back/converters/simple_converters/time_series_window.py +++ b/DashAI/back/converters/simple_converters/time_series_window.py @@ -93,6 +93,7 @@ class TimeSeriesWindowConverter(FeatureEngineeringConverter, BaseConverter): SCHEMA = TimeSeriesWindowConverterSchema SUPERVISED = True CHANGES_ROW_COUNT = True + COLUMN_OPERATION = "rows" LEARNS_FROM_DATA = False DESCRIPTION = MultilingualString( en=( diff --git a/DashAI/back/converters/simple_converters/type_cast.py b/DashAI/back/converters/simple_converters/type_cast.py index 2e60c0c90..64866b2a6 100644 --- a/DashAI/back/converters/simple_converters/type_cast.py +++ b/DashAI/back/converters/simple_converters/type_cast.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, List, Union from DashAI.back.converters.base_converter import BaseConverter from DashAI.back.converters.category.basic_preprocessing import ( @@ -7,6 +7,11 @@ from DashAI.back.core.schema_fields import enum_field, schema_field from DashAI.back.core.schema_fields.base_schema import BaseSchema from DashAI.back.core.utils import MultilingualString +from DashAI.back.preprocessing.structure_types import ( + StateItem, + StructureDelta, + StructureMessage, +) from DashAI.back.types.categorical import Categorical from DashAI.back.types.dashai_data_type import DashAIDataType from DashAI.back.types.value_types import Float, Integer, Text @@ -179,6 +184,32 @@ def __init__(self, new_type: str, on_error: str = "raise"): self._fit_x = None self._converted_cache: dict = {} + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + """Estimate the output: the scope cast in place to ``new_type``. + + With ``on_error="skip"`` a column whose values cannot be cast keeps + its original type, which is only known once the data is seen, so the + estimate carries a ``cast_may_skip`` warning. + + Parameters + ---------- + inputs : list of ColumnItem | BlockItem + The dataset state items in this converter's scope. + + Returns + ------- + StructureDelta + The inputs retyped to ``new_type``. + """ + delta = super().infer_output_columns(inputs) + if self.on_error == "skip": + delta.warnings.append( + StructureMessage( + code="cast_may_skip", params={"new_type": self.new_type} + ) + ) + return delta + def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None ) -> "TypeCast": diff --git a/DashAI/back/converters/structure_mixins.py b/DashAI/back/converters/structure_mixins.py new file mode 100644 index 000000000..59ec5bebb --- /dev/null +++ b/DashAI/back/converters/structure_mixins.py @@ -0,0 +1,28 @@ +"""Reusable `infer_output_columns` overrides shared by several converters.""" + +from typing import List, Optional + +from DashAI.back.preprocessing.structure_types import StateItem, StructureDelta + + +class ComponentsOutputMixin: + """Structure of a converter that outputs `n_components` new columns. + + Covers projections and kernel approximations (PCA, TruncatedSVD, + Nystroem, RBFSampler...), which consume their scope and output one + column per component. The count is known whenever `n_components` is an + integer. None, a float (explained-variance ratio) or "mle" only resolve + on fit: None means min(n_samples, n_features) for the PCA family. List + this mixin before the converter's category base so its override wins. + """ + + def _components_count(self, inputs: List[StateItem]) -> Optional[int]: + n_components = getattr(self, "n_components", None) + if isinstance(n_components, int) and not isinstance(n_components, bool): + return n_components + return None + + def infer_output_columns(self, inputs: List[StateItem]) -> StructureDelta: + return StructureDelta( + added=self._default_blocks(count=self._components_count(inputs)) + ) diff --git a/DashAI/back/job/preprocessing_job.py b/DashAI/back/job/preprocessing_job.py index 7fdd21b5f..ac695a680 100644 --- a/DashAI/back/job/preprocessing_job.py +++ b/DashAI/back/job/preprocessing_job.py @@ -153,7 +153,9 @@ def run(self) -> None: f"Fitting preprocessing for fold {i + 1}/{total_folds}", ) fold_preprocessor = SessionPreprocessor( - sequence, component_registry + sequence, + component_registry, + target_columns=model_session.output_columns, ) fold_preprocessor.fit_transform(x_folds[i]) with open(os.path.join(tmp_dir, f"fold_{i}.pkl"), "wb") as f: @@ -161,7 +163,9 @@ def run(self) -> None: self.report_progress(0.85, "Fitting final preprocessing") final_preprocessor = SessionPreprocessor( - sequence, component_registry + sequence, + component_registry, + target_columns=model_session.output_columns, ) final_transformed, final_resolved = ( final_preprocessor.fit_transform(x_folds[-1]) diff --git a/DashAI/back/preprocessing/column_ref.py b/DashAI/back/preprocessing/column_ref.py index 652860a64..4e666baa1 100644 --- a/DashAI/back/preprocessing/column_ref.py +++ b/DashAI/back/preprocessing/column_ref.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import BaseModel, Field, TypeAdapter +from pydantic import BaseModel, Field, TypeAdapter, model_validator from typing_extensions import Annotated @@ -31,6 +31,19 @@ class GroupColumnRef(BaseModel): # them (see SessionPreprocessor._classify_by_type). The slot name is a # DashAI type's display_name(), e.g. "Categorical" or "Integer". slot: Optional[str] = None + # One specific column the step produced, for converters whose output + # names are known before fit (e.g. DateFeatures' "date_month"). Keeping + # the producing step in the ref, instead of a plain RawColumnRef, lets a + # caller walk it back to the original columns it depends on. + name: Optional[str] = None + + @model_validator(mode="after") + def _slot_or_name(self) -> "GroupColumnRef": + if self.slot is not None and self.name is not None: + raise ValueError( + "A group reference takes either a slot or a column name, not both." + ) + return self ColumnRef = Annotated[Union[RawColumnRef, GroupColumnRef], Field(discriminator="kind")] @@ -85,21 +98,29 @@ def resolve_refs( Returns ------- list of str - Concrete column names, in order. A GroupColumnRef with `slot=None` - expands to every column its step produced; with a `slot` set, only - to that step's columns of that declared type. + Concrete column names, in order. A GroupColumnRef with a `name` + resolves to that one column; with a `slot`, to that step's columns + of that declared type; with neither, to every column its step + produced. Raises ------ KeyError If a GroupColumnRef names a step with no entry in resolved_columns - (or, for a slot ref, no matching entry in resolved_slots) — the step - has not been fit yet, or produced no column of that type. + (or, for a slot ref, no matching entry in resolved_slots), or a + column its step did not produce: the step has not been fit yet, or + produced no such column. """ names: List[str] = [] for ref in refs: if ref.kind == "raw": names.append(ref.name) + elif ref.name is not None: + if ref.name not in resolved_columns[ref.step]: + raise KeyError( + f"Step {ref.step} did not produce a column named '{ref.name}'." + ) + names.append(ref.name) elif ref.slot is None: names.extend(resolved_columns[ref.step]) else: diff --git a/DashAI/back/preprocessing/session_preprocessor.py b/DashAI/back/preprocessing/session_preprocessor.py index 7ca6ff801..2e9c73540 100644 --- a/DashAI/back/preprocessing/session_preprocessor.py +++ b/DashAI/back/preprocessing/session_preprocessor.py @@ -8,9 +8,10 @@ import os import pickle -from typing import TYPE_CHECKING, Any, Dict, List, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from DashAI.back.converters.dataset_columns import ( + plan_new_column_names, rebuild_dataset_with_transformed_columns, ) from DashAI.back.preprocessing.column_ref import ConverterSequence, resolve_refs @@ -26,11 +27,21 @@ class SessionPreprocessor: of an earlier step. Fitting always uses only the "train" entry of whatever split dict is passed in, so no step ever sees validation or test rows during fit. + + Supervised converters (e.g. feature selectors) are fit with the + `target_columns` of the train split as `y`. The target is never part of + a step's scope, so no converter ever transforms it. """ - def __init__(self, sequence: ConverterSequence, component_registry: Any): + def __init__( + self, + sequence: ConverterSequence, + component_registry: Any, + target_columns: Optional[List[str]] = None, + ): self.sequence = sequence self.component_registry = component_registry + self.target_columns = list(target_columns or []) self.fitted_converters: List[Any] = [] self.resolved_columns: Dict[int, List[str]] = {} self.resolved_slots: Dict[int, Dict[str, List[str]]] = {} @@ -66,6 +77,60 @@ def _classify_by_type( slots.setdefault(type_name, []).append(name) return slots + def _fit(self, converter: Any, train: "DashAIDataset", train_scope) -> Any: + """Fit a step's converter, passing the target to supervised ones.""" + # A preprocessor pickled before target_columns existed has no such + # attribute; it is only unpickled to transform, never to fit again. + target_columns = getattr(self, "target_columns", None) + if type(converter).SUPERVISED and target_columns: + return converter.fit(train_scope, train.select_columns(target_columns)) + return converter.fit(train_scope) + + def _record_group( + self, + index: int, + converter: Any, + train: "DashAIDataset", + scope_names: List[str], + output_names: List[str], + ) -> None: + """Record which real columns a step produced, and their slots. + + A converter that only rewrites its scope columns in place (e.g. a + scaler: "age" in, scaled "age" out) has no other way to expose its + result, so the scope names ARE the group. A converter that adds new + columns (e.g. Bag-of-Words, which also keeps its scope column + verbatim) only exposes the new ones: a passthrough scope column is + not this step's output, and would still carry its pre-conversion + type (e.g. Text), which is never valid as a resolved input column. + + New columns are recorded under the final name they get in the + dataset: one whose name is already taken is renamed by + rebuild_dataset_with_transformed_columns (e.g. "derived_1"). Slots + are classified with the names the converter itself produced, since + that is what its get_output_type knows. + """ + new_columns = [name for name in output_names if name not in scope_names] + if not new_columns: + self.resolved_columns[index] = list(output_names) + self.resolved_slots[index] = self._classify_by_type( + converter, self.resolved_columns[index] + ) + return + + removed = {name for name in scope_names if name not in output_names} + final_names = plan_new_column_names( + [name for name in train.column_names if name not in removed], + new_columns, + ) + self.resolved_columns[index] = [final_names[name] for name in new_columns] + self.resolved_slots[index] = { + type_name: [final_names[name] for name in names] + for type_name, names in self._classify_by_type( + converter, new_columns + ).items() + } + @staticmethod def _transform_split(converter, dataset, scope_names, train_transformed): """Transform one split's scoped columns, without crashing on 0 rows. @@ -129,7 +194,7 @@ def fit_transform( converter = self._instantiate(step) train_scope = current["train"].select_columns(scope_names) - converter = converter.fit(train_scope) + converter = self._fit(converter, current["train"], train_scope) train_transformed = converter.transform(train_scope) transformed_by_split = {"train": train_transformed} @@ -140,27 +205,12 @@ def fit_transform( converter, dataset, scope_names, train_transformed ) - # A converter that only rewrites its scope columns in place (e.g. a - # scaler: "age" in, scaled "age" out) has no other way to expose - # its result, so the scope names ARE the group. But a converter - # like Bag-of-Words additionally keeps its scope column verbatim - # alongside brand-new derived columns (see BagOfWordsConverter. - # transform's docstring: "the source text column is preserved - # unchanged") — for those, the untouched scope column is a - # passthrough, not this step's own output, so it must not leak - # into the group a later step or the wizard's input selection can - # reference (it would still carry the pre-conversion dtype, e.g. - # Text, which is never valid as a resolved input column). - new_columns = [ - name - for name in train_transformed.column_names - if name not in scope_names - ] - self.resolved_columns[index] = ( - new_columns if new_columns else list(train_transformed.column_names) - ) - self.resolved_slots[index] = self._classify_by_type( - converter, self.resolved_columns[index] + self._record_group( + index, + converter, + current["train"], + scope_names, + list(train_transformed.column_names), ) new_current = {} diff --git a/DashAI/back/preprocessing/structure.py b/DashAI/back/preprocessing/structure.py new file mode 100644 index 000000000..1ae485697 --- /dev/null +++ b/DashAI/back/preprocessing/structure.py @@ -0,0 +1,389 @@ +"""Estimate the dataset structure along a session's converter chain. + +`infer_structure` walks a ConverterSequence without fitting anything: each +step's scope is resolved against the estimated dataset state at that point, +checked against the converter's metadata, and the converter's own +`infer_output_columns` delta is applied the same way SessionPreprocessor +splices real output back into the dataset. The Models-module wizard uses it +to show which columns exist after every step, and session creation uses it +to reject a chain that would reference a column that no longer exists. +""" + +from typing import Any, Dict, List, Set, Tuple + +from DashAI.back.converters.dataset_columns import plan_new_column_names +from DashAI.back.preprocessing.column_ref import ConverterStep +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + ColumnItem, + RowsNotSupportedError, + StateItem, + StepStructure, + StructureDelta, + StructureMessage, + StructureResult, + known_width, + type_fields, +) +from DashAI.back.splitters.splits_payload import schema_placeholder_defaults + + +class StructureError(Exception): + """A chain or ref the estimate rejects, with the message for the frontend.""" + + def __init__(self, code: str, **params: Any): + super().__init__(code) + self.message = StructureMessage(code=code, params=params) + + +def _identity(item: StateItem) -> Tuple: + """What a later delta or ref uses to recognise an item.""" + if isinstance(item, ColumnItem): + return ("column", item.name) + return ("block", item.step, item.slot) + + +def infer_structure( + dataset_types: Dict[str, Any], + candidates: List[str], + target: List[str], + steps: List[ConverterStep], + component_registry: Any, +) -> StructureResult: + """Estimate the dataset state after every step of a converter chain. + + Parameters + ---------- + dataset_types : dict + Every dataset column name, in dataset order, mapped to its DashAI + type. + candidates : list of str + The original columns the user may feed into the chain. The initial + state is these columns, minus the target. + target : list of str + The output columns. They never enter the state and no scope may + reference them. + steps : list of ConverterStep + The chain, in order. + component_registry : ComponentRegistry or mapping + Resolves a converter name to ``{"class": converter_class}``. + + Returns + ------- + StructureResult + The initial state, one entry per step (its status, the state after + it and what it produced) and the final state. After the first + invalid step the rest are "blocked" and the final state is the one + before it. + """ + target_set = set(target) + state: List[StateItem] = [] + for name in dataset_types: + if name in candidates and name not in target_set: + type_name, dtype = type_fields(dataset_types[name]) + state.append(ColumnItem(name=name, type=type_name, dtype=dtype)) + initial = list(state) + # The runtime renames a new column against every column of the dataset, + # not just the candidates, so name clashes are checked against all. + taken: Set[str] = set(dataset_types) + + results: List[StepStructure] = [] + blocked = False + for index, step in enumerate(steps): + if blocked: + results.append(StepStructure(status="blocked", state=state)) + continue + try: + state, added, warnings = _apply_step( + index, step, state, taken, target_set, component_registry + ) + except StructureError as e: + results.append(StepStructure(status="error", state=state, error=e.message)) + blocked = True + continue + results.append( + StepStructure(status="ok", state=state, added=added, warnings=warnings) + ) + + return StructureResult( + initial=initial, + steps=results, + final=state, + valid=not blocked, + ) + + +def _apply_step( + index: int, + step: ConverterStep, + state: List[StateItem], + taken: Set[str], + target: Set[str], + component_registry: Any, +) -> Tuple[List[StateItem], List[StateItem], List[StructureMessage]]: + """Estimate one step. Updates `taken` in place; raises StructureError.""" + try: + converter_class = component_registry[step.converter]["class"] + except KeyError as e: + raise StructureError("unknown_converter", converter=step.converter) from e + converter = _instantiate(converter_class, step.params) + + scope = resolve_state_refs(step.scope, state, target) + _check_scope(converter_class, scope) + + warnings: List[StructureMessage] = [] + try: + delta = converter.infer_output_columns(scope) + except RowsNotSupportedError as e: + raise StructureError("rows_not_supported", converter=step.converter) from e + except ValueError as e: + # A converter validating its own configuration against the scope + # (e.g. ColumnArithmetic without a constant for a single column). + raise StructureError("converter_rejected", detail=str(e)) from e + except Exception: + # A converter whose estimate is broken (e.g. an older plugin) falls + # back to the conservative default: its scope is consumed and its + # output is one block of unknown size and type. + delta = StructureDelta(added=[BlockItem()]) + warnings.append( + StructureMessage(code="inference_fallback", params={"step": index}) + ) + warnings.extend(delta.warnings) + warnings.extend(_check_bounds(converter_class, converter, scope)) + + new_state, added = _apply_delta(index, state, scope, delta, taken) + return new_state, added, warnings + + +def _instantiate(converter_class: Any, params: Dict[str, Any]) -> Any: + """Validate the user's params and build the converter with them. + + The params are validated the way the frontend form submits them, with + every field it seeds from the schema placeholders (as _validate_splits + does for splitters), but the converter is built from the params exactly + as given, as SessionPreprocessor builds it, so the estimate describes + the converter that will really run. + """ + try: + schema = getattr(converter_class, "SCHEMA", None) + if schema is not None: + schema.model_validate( + {**schema_placeholder_defaults(converter_class), **params} + ) + return converter_class(**params) + except Exception as e: + raise StructureError("invalid_params", detail=str(e)) from e + + +def resolve_state_refs( + refs: List[Any], state: List[StateItem], target: Set[str] +) -> List[StateItem]: + """Map ColumnRefs to the dataset state items they point at. + + Parameters + ---------- + refs : list of RawColumnRef | GroupColumnRef + The refs to resolve, e.g. a step's scope or the model inputs. + state : list of ColumnItem | BlockItem + The estimated dataset state to resolve them against. + target : set of str + The output columns, which no ref may point at. + + Returns + ------- + list of ColumnItem | BlockItem + The items, in ref order, without duplicates. + + Raises + ------ + StructureError + With code "target_in_scope" for a ref to the target, or + "missing_ref" for a ref to something that does not exist (anymore). + """ + scope: List[StateItem] = [] + seen: Set[Tuple] = set() + for ref in refs: + if ref.kind == "raw": + if ref.name in target: + raise StructureError("target_in_scope", column=ref.name) + matches = [ + item + for item in state + if isinstance(item, ColumnItem) + and item.origin is None + and item.name == ref.name + ] + label = ref.name + elif ref.name is not None: + matches = [ + item + for item in state + if isinstance(item, ColumnItem) + and item.origin == ref.step + and item.name == ref.name + ] + label = ref.name + elif ref.slot is not None: + matches = [ + item + for item in state + if isinstance(item, BlockItem) + and item.step == ref.step + and item.slot == ref.slot + ] + label = f"{ref.step}:{ref.slot}" + else: + # Whole group: everything the step produced that still exists. + matches = [ + item + for item in state + if (isinstance(item, ColumnItem) and item.origin == ref.step) + or (isinstance(item, BlockItem) and item.step == ref.step) + ] + label = str(ref.step) + if not matches: + raise StructureError("missing_ref", ref=label) + for item in matches: + if _identity(item) not in seen: + seen.add(_identity(item)) + scope.append(item) + return scope + + +def _check_scope(converter_class: Any, scope: List[StateItem]) -> None: + """Apply the converter's declared column restrictions to its scope. + + Mirrors the frontend's evaluateColumnEligibility: empty allow-lists mean + no restriction. An item of unknown type is let through, since only the + fit can tell. + """ + metadata = converter_class.get_metadata() + allowed_types = metadata.get("allowed_types") or [] + allowed_dtypes = metadata.get("allowed_dtypes") or [] + non_allowed_dtypes = metadata.get("non_allowed_dtypes") or [] + for item in scope: + if item.type is None: + continue + if ( + (allowed_types and item.type not in allowed_types) + or (allowed_dtypes and item.dtype not in allowed_dtypes) + or (item.dtype in non_allowed_dtypes) + ): + raise StructureError( + "type_not_allowed", + column=item.name if isinstance(item, ColumnItem) else item.label, + type=item.type, + allowed=allowed_types, + ) + + cardinality = metadata.get("input_cardinality") or {} + # A block of unknown size holds at least one column. + at_least = sum( + 1 if isinstance(item, ColumnItem) else (item.count or 1) for item in scope + ) + exactly = known_width(scope) + required = cardinality.get("exact") + minimum = required if required is not None else cardinality.get("min") + maximum = required if required is not None else cardinality.get("max") + too_few = minimum is not None and (exactly or at_least) < minimum + too_many = maximum is not None and exactly is not None and exactly > maximum + if too_few or too_many: + raise StructureError( + "cardinality", + min=minimum, + max=maximum, + selected=exactly if exactly is not None else at_least, + ) + + +def _check_bounds( + converter_class: Any, converter: Any, scope: List[StateItem] +) -> List[StructureMessage]: + """Check `n_components` against the scope width, for projections. + + Only converters flagged N_COMPONENTS_FEATURES_BOUNDED (the PCA family) + cannot output more components than input columns; kernel approximations + can. + """ + n_components = getattr(converter, "n_components", None) + if not getattr(converter_class, "N_COMPONENTS_FEATURES_BOUNDED", False): + return [] + if not isinstance(n_components, int) or isinstance(n_components, bool): + return [] + width = known_width(scope) + if width is None: + return [ + StructureMessage( + code="count_unknown_bound", params={"n_components": n_components} + ) + ] + if n_components > width: + raise StructureError( + "n_components_exceeds", n_components=n_components, columns=width + ) + return [] + + +def _apply_delta( + index: int, + state: List[StateItem], + scope: List[StateItem], + delta: StructureDelta, + taken: Set[str], +) -> Tuple[List[StateItem], List[StateItem]]: + """Apply a converter's delta to the state, as the runtime splices output. + + Surviving scope items stay in place (maybe retyped), consumed ones are + dropped, and new items are appended at the end with runtime names: a + new column whose name is taken is renamed like + rebuild_dataset_with_transformed_columns does. + """ + concrete = [item for item in delta.added if isinstance(item, ColumnItem)] + blocks = [item for item in delta.added if isinstance(item, BlockItem)] + if concrete and blocks: + raise ValueError( + f"Step {index} mixes concrete columns and blocks in one delta." + ) + + scope_ids = {_identity(item) for item in scope} + kept = {_identity(item): item for item in delta.kept} + new_state: List[StateItem] = [] + removed_names: List[str] = [] + for item in state: + item_id = _identity(item) + if item_id not in scope_ids: + new_state.append(item) + elif item_id in kept: + new_state.append(kept[item_id]) + elif isinstance(item, ColumnItem): + removed_names.append(item.name) + + # A step that outputs blocks may keep some consumed columns under their + # own names (a selector keeps a subset), so their names stay taken. + if not blocks: + taken.difference_update(removed_names) + + added: List[StateItem] = [] + if concrete: + final_names = plan_new_column_names(taken, [item.name for item in concrete]) + for item in concrete: + added.append( + item.model_copy( + update={"name": final_names[item.name], "origin": index} + ) + ) + taken.update(final_names.values()) + else: + # A single block is the step's whole group (a ref with no slot); + # several are told apart by type, as the runtime classifies them. + for block in blocks: + added.append( + block.model_copy( + update={ + "step": index, + "slot": block.type if len(blocks) > 1 else None, + } + ) + ) + + return new_state + added, added diff --git a/DashAI/back/preprocessing/structure_types.py b/DashAI/back/preprocessing/structure_types.py new file mode 100644 index 000000000..449b894a1 --- /dev/null +++ b/DashAI/back/preprocessing/structure_types.py @@ -0,0 +1,141 @@ +"""Types describing the estimated structure of a dataset during preprocessing. + +The Models-module wizard needs to know which columns (and of which type) +exist at every point of a session's converter chain before anything is +fit. Each converter describes what it does to its scope through +``BaseConverter.infer_output_columns``, which returns a StructureDelta over +these state items; ``infer_structure`` (structure.py) applies the deltas step +by step. + +A state item is either a concrete column, whose name is known ahead of any +fit, or a symbolic block: columns whose names (and sometimes count) only +exist once the converter is fit, e.g. one-hot columns, one per category. +""" + +from typing import Any, Dict, List, Literal, Optional, Tuple, Union + +from pydantic import BaseModel, Field +from typing_extensions import Annotated + + +class RowsNotSupportedError(Exception): + """Raised by a converter that changes the dataset's row count. + + Session preprocessing does not support these converters yet: resampling + must run on the training partition only and carry the target along, + which the session preprocessor does not do. + """ + + +class ColumnItem(BaseModel): + kind: Literal["column"] = "column" + name: str + # A DashAI type's display_name(), e.g. "Integer". None means unknown. + type: Optional[str] = None + dtype: Optional[str] = None + # None for an original dataset column (even after being replaced in + # place); otherwise the index of the step that created it. + origin: Optional[int] = None + + +class BlockItem(BaseModel): + kind: Literal["block"] = "block" + # Index of the step that produced the block, filled by infer_structure. + step: int = -1 + # The slot a GroupColumnRef uses to reference this block. It is fixed + # when the block is created, while `type` follows later in-place + # conversions of its columns, mirroring the runtime, where a step's + # slots are classified once, at that step. + slot: Optional[str] = None + label: str = "output" + type: Optional[str] = None + dtype: Optional[str] = None + # Number of columns in the block, or None when it is only known after + # fit (e.g. one-hot encoding: one column per category). + count: Optional[int] = None + + +StateItem = Annotated[Union[ColumnItem, BlockItem], Field(discriminator="kind")] + + +class StructureMessage(BaseModel): + # Suffix of the frontend i18n key the message is translated with. + code: str + params: Dict[str, Any] = Field(default_factory=dict) + + +class StructureDelta(BaseModel): + """What a converter does to the items in its scope. + + Scope items missing from `kept` are removed from the dataset state. A + kept item keeps its identity (a column's name, a block's step and slot) + but may carry a new type. `added` are the items the converter creates: + either all concrete columns or all blocks, never a mix. + """ + + kept: List[StateItem] = Field(default_factory=list) + added: List[StateItem] = Field(default_factory=list) + warnings: List[StructureMessage] = Field(default_factory=list) + + +class StepStructure(BaseModel): + status: Literal["ok", "error", "blocked"] + # Dataset state after this step. + state: List[StateItem] = Field(default_factory=list) + # Items this step produced. + added: List[StateItem] = Field(default_factory=list) + error: Optional[StructureMessage] = None + warnings: List[StructureMessage] = Field(default_factory=list) + + +class StructureResult(BaseModel): + initial: List[StateItem] + steps: List[StepStructure] + final: List[StateItem] + valid: bool + + +def known_width(items: List[StateItem]) -> Optional[int]: + """Return how many real columns the items stand for, if that is known. + + A concrete column counts as one; a block counts as its `count`. None + when any block's size is only known after fit. + """ + width = 0 + for item in items: + if isinstance(item, ColumnItem): + width += 1 + elif item.count is None: + return None + else: + width += item.count + return width + + +def type_fields(dashai_type: Any) -> Tuple[Optional[str], Optional[str]]: + """Return the (display name, dtype) pair the frontend shows for a type. + + Parameters + ---------- + dashai_type : DashAIDataType or None + The type to describe. + + Returns + ------- + tuple of (str or None, str or None) + The type's display_name() and its storage dtype (e.g. "int64"), + each None when unknown. + """ + if dashai_type is None: + return None, None + name = ( + dashai_type.display_name() + if hasattr(dashai_type, "display_name") + else type(dashai_type).__name__ + ) + dtype = None + if hasattr(dashai_type, "to_string"): + description = dashai_type.to_string() + if isinstance(description, dict): + dtype = description.get("dtype") + return name, dtype diff --git a/DashAI/front/src/api/modelSession.ts b/DashAI/front/src/api/modelSession.ts index 0c85e6f9e..d495b1d24 100644 --- a/DashAI/front/src/api/modelSession.ts +++ b/DashAI/front/src/api/modelSession.ts @@ -3,6 +3,7 @@ import type { IColumnRef, IConverterStep, IModelSession, + IStructureResult, } from "../types/modelSession"; const endpointURL = "/v1/model-session"; @@ -84,7 +85,9 @@ export const validateColumns = async ( inputColumns: string[], outputColumns: string[], inputRefs?: IColumnRef[], - converterOutputTypes?: Record, + // With preprocessing, the backend types every input ref from the chain's + // estimated structure. + preprocessing?: IConverterStep[], ): Promise => { const formData = { task_name: taskName, @@ -92,7 +95,11 @@ export const validateColumns = async ( inputs_columns: inputColumns, outputs_columns: outputColumns, input_refs: inputRefs, - converter_output_types: converterOutputTypes, + preprocessing: preprocessing?.map(({ converter, params, scope }) => ({ + converter, + params, + scope, + })), }; const response = await api.post( "/v1/model-session/validation", @@ -100,3 +107,32 @@ export const validateColumns = async ( ); return response.data; }; + +// Estimate the dataset state after every step of a preprocessing chain, +// without fitting anything (see the backend's infer_structure). +export const getPreprocessingStructure = async ({ + datasetId, + candidates, + outputColumns, + steps, +}: { + datasetId: number; + candidates: string[]; + outputColumns: string[]; + steps: IConverterStep[]; +}): Promise => { + const response = await api.post( + `${endpointURL}/preprocessing/structure`, + { + dataset_id: datasetId, + candidates, + output_columns: outputColumns, + steps: steps.map(({ converter, params, scope }) => ({ + converter, + params, + scope, + })), + }, + ); + return response.data; +}; diff --git a/DashAI/front/src/components/models/CreateSessionSteps.jsx b/DashAI/front/src/components/models/CreateSessionSteps.jsx index 89c138976..1e917a697 100644 --- a/DashAI/front/src/components/models/CreateSessionSteps.jsx +++ b/DashAI/front/src/components/models/CreateSessionSteps.jsx @@ -6,6 +6,7 @@ import { useFormik } from "formik"; import { useTourContext } from "../tour/TourProvider"; import SetNameAndDatasetStep from "./SetNameAndDatasetStep"; import PrepareDatasetStep from "./modelSession/PrepareDatasetStep"; +import BaseColumnsStep from "./modelSession/BaseColumnsStep"; import PreprocessingStep from "./modelSession/PreprocessingStep"; import SelectColumnsStep from "./modelSession/SelectColumnsStep"; import DatasetAutocomplete from "../notebooks/notebookCreation/DatasetAutocomplete"; @@ -26,6 +27,7 @@ import StepperNavigationFooter from "../shared/StepperNavigationFooter"; import { hasPartition } from "../../utils/splitsPayload"; const STEP_PREPARE_DATASET = "prepareDataset"; +const STEP_BASE_COLUMNS = "baseColumns"; const STEP_PREPROCESSING = "preprocessing"; const STEP_SELECT_COLUMNS = "selectColumns"; @@ -66,6 +68,9 @@ function CreateSessionSteps({ splits: {}, runs: [], applyPreprocessing: false, + // Original columns the preprocessing chain may work on (see + // BaseColumnsStep); only used when applyPreprocessing is on. + candidate_columns: [], preprocessing: [], input_column_refs: [], }); @@ -111,7 +116,12 @@ function CreateSessionSteps({ const steps = useMemo( () => [ STEP_PREPARE_DATASET, - ...(newExp.applyPreprocessing ? [STEP_PREPROCESSING] : []), + // With preprocessing, the output and candidate columns come first, so + // the chain is built knowing what it is for and never touches the + // output. + ...(newExp.applyPreprocessing + ? [STEP_BASE_COLUMNS, STEP_PREPROCESSING] + : []), STEP_SELECT_COLUMNS, ], [newExp.applyPreprocessing], @@ -128,6 +138,7 @@ function CreateSessionSteps({ dataset: newDataset, input_columns: [], output_columns: [], + candidate_columns: [], input_column_refs: [], splits: {}, })); @@ -257,6 +268,12 @@ function CreateSessionSteps({ const hasValidation = hasPartition(newExp.splits, "validation"); const hasTest = hasPartition(newExp.splits, "test"); + // Steps added while the switch was on and then abandoned by turning + // it off must not be sent: the session would still preprocess. + const preprocessing = newExp.applyPreprocessing + ? newExp.preprocessing + : []; + let effectiveName = sessionName; let response; try { @@ -271,7 +288,7 @@ function CreateSessionSteps({ hasTest ? allMetricNames : [], newExp.evaluation_strategy, JSON.stringify(newExp.splits), - newExp.preprocessing, + preprocessing, newExp.input_column_refs, ); } catch (createError) { @@ -289,7 +306,7 @@ function CreateSessionSteps({ hasTest ? allMetricNames : [], newExp.evaluation_strategy, JSON.stringify(newExp.splits), - newExp.preprocessing, + preprocessing, newExp.input_column_refs, ); } else { @@ -328,12 +345,14 @@ function CreateSessionSteps({ const stepTitle = { [STEP_PREPARE_DATASET]: t("models:label.prepareDataset"), + [STEP_BASE_COLUMNS]: t("models:label.baseColumnsTitle"), [STEP_PREPROCESSING]: t("models:label.preprocessingOptional"), [STEP_SELECT_COLUMNS]: t("models:label.selectColumnsTitle"), }[currentStep]; const stepSubtitle = { [STEP_PREPARE_DATASET]: t("models:label.selectDatasetAndPrepare"), + [STEP_BASE_COLUMNS]: t("models:label.baseColumnsDescription"), [STEP_PREPROCESSING]: t("models:label.preprocessingOptionalDescription"), [STEP_SELECT_COLUMNS]: t("models:label.selectColumnsDescription"), }[currentStep]; @@ -391,13 +410,30 @@ function CreateSessionSteps({ )} + {currentStep === STEP_BASE_COLUMNS && selectedDataset && ( + <> + {infoLoading ? ( + + + + ) : ( + + )} + + )} + {currentStep === STEP_PREPROCESSING && selectedDataset && ( )} diff --git a/DashAI/front/src/components/models/SessionInfoContent.jsx b/DashAI/front/src/components/models/SessionInfoContent.jsx index 8b79df8ae..a937b2333 100644 --- a/DashAI/front/src/components/models/SessionInfoContent.jsx +++ b/DashAI/front/src/components/models/SessionInfoContent.jsx @@ -7,8 +7,7 @@ import { getComponents } from "../../api/component"; import ParamInfoList, { ParamInfoBox } from "./ParamInfoBox"; import { buildStepDisplayNames, - buildColumnKeysAndTypes, - refToKey, + labelForRef, } from "./modelSession/sessionColumnRefs"; const SPLIT_TYPE_LABEL_KEYS = { @@ -56,14 +55,6 @@ export default function SessionInfoContent({ preprocessingSteps, convertersMeta, ); - // datasetTypes isn't needed here: a raw scope ref is shown as its own - // column name regardless, and a group scope ref's label only depends on - // the earlier step it points to (see buildColumnKeysAndTypes). - const { optionLabels: scopeOptionLabels } = buildColumnKeysAndTypes({ - datasetTypes: {}, - preprocessing: preprocessingSteps, - convertersMeta, - }); const getDatasetName = () => { if (!session.dataset_id || !datasets.length) return t("common:unknown"); @@ -199,10 +190,7 @@ export default function SessionInfoContent({ key={`${step.converter}-${index}`} label={stepDisplayNames[index]} value={(step.scope || []) - .map((ref) => { - const key = refToKey(ref); - return scopeOptionLabels[key] || key; - }) + .map((ref) => labelForRef(ref, stepDisplayNames)) .join(", ")} /> ))} diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx index 266abc4ce..533acecea 100644 --- a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.jsx @@ -1,6 +1,9 @@ import React, { useEffect, useMemo, useState } from "react"; import PropTypes from "prop-types"; import { + Accordion, + AccordionDetails, + AccordionSummary, Box, Paper, Typography, @@ -13,6 +16,7 @@ import { useMaterialReactTable, } from "material-react-table"; import DeleteIcon from "@mui/icons-material/Delete"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; import Transform from "@mui/icons-material/Transform"; import { useTheme } from "@mui/material/styles"; import { useTranslation } from "react-i18next"; @@ -23,11 +27,11 @@ import DeleteConfirmationModal from "../../threeSectionLayout/DeleteConfirmation import ItemsToDeleteList from "../../notebooks/converter/ItemsToDeleteList"; import { useExplorersAndConverters } from "../../notebooks/context/ExplorersAndConvertersContext"; import { - buildColumnKeysAndTypes, buildStepDisplayNames, - groupKey, - refToKey, + labelForRef, + stateToOptions, } from "./sessionColumnRefs"; +import { formatStructureMessage } from "./structureMessages"; function TypeChip({ type }) { if (!type) return null; @@ -49,14 +53,14 @@ function TypeChip({ type }) { TypeChip.propTypes = { type: PropTypes.string }; -function RefChip({ refKey, label, columnTypes }) { +function RefChip({ label, type }) { return ( {label} - + } /> @@ -64,23 +68,49 @@ function RefChip({ refKey, label, columnTypes }) { } RefChip.propTypes = { - refKey: PropTypes.string.isRequired, label: PropTypes.string.isRequired, - columnTypes: PropTypes.object.isRequired, + type: PropTypes.string, +}; + +/** + * One chip per item of an estimated dataset state: an original column shows + * its name, anything a step produced shows its label (a block also shows its + * column count, "N" when only known after fit). + */ +function StateChips({ items, stepDisplayNames }) { + const { allKeys, columnTypes, optionLabels } = stateToOptions( + items, + stepDisplayNames, + ); + return ( + + {allKeys.map((key) => ( + + ))} + + ); +} + +StateChips.propTypes = { + items: PropTypes.array.isRequired, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, }; /** * Mirrors the notebook's own ConverterParametersTable (same columns, same * MaterialReactTable setup): a compact two-column table with one row per - * converter attribute, values rendered as wrapped chips (each with a - * colored type badge) instead of plain text, since a session converter's - * scope/output are references to columns or groups, not literal strings. + * converter attribute. The scope is plain text, like Notebooks shows an + * applied converter's scope; the output is one chip per column or block the + * step produces, each with its type. */ function SessionConverterParametersTable({ step, - optionLabels, - columnTypes, - outputEntries, + added, + stepDisplayNames, t, localization, }) { @@ -91,34 +121,14 @@ function SessionConverterParametersTable({ const paramRows = [ { - // Plain comma-joined text, matching the notebook's own - // ConverterParametersTable (ConverterBox.jsx) — no chips here, since - // that's how Notebooks shows an already-applied converter's scope. key: t("datasets:label.scopeColumns"), value: step.scope - .map((ref) => { - const key = refToKey(ref); - return optionLabels[key] || key; - }) + .map((ref) => labelForRef(ref, stepDisplayNames)) .join(", "), }, { - // Usually one chip; more than one when this step's scope mixed - // column types (e.g. SimpleImputer preserving both a categorical and - // a numeric column), so each declared slot gets its own chip. key: t("datasets:label.converterOutput"), - value: ( - - {outputEntries.map(({ key, label }) => ( - - ))} - - ), + value: , }, ]; @@ -142,14 +152,8 @@ function SessionConverterParametersTable({ SessionConverterParametersTable.propTypes = { step: PropTypes.object.isRequired, - optionLabels: PropTypes.object.isRequired, - columnTypes: PropTypes.object.isRequired, - outputEntries: PropTypes.arrayOf( - PropTypes.shape({ - key: PropTypes.string.isRequired, - label: PropTypes.string.isRequired, - }), - ).isRequired, + added: PropTypes.array.isRequired, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, t: PropTypes.func.isRequired, localization: PropTypes.object.isRequired, }; @@ -157,12 +161,11 @@ SessionConverterParametersTable.propTypes = { /** * A single applied-converter card, styled after the notebook's own * ConverterBox (icon + real component display name + description + - * parameters table), but built against the session's preprocessing step - * shape (`{converter, scope, outputSlots}`) instead of the notebook's - * (`{parameters: {scope, target}}`), and with a "Salida" row showing the - * converter's declared output type(s) — usually one chip, more than one - * when the step's scope mixed column types — since that group doesn't - * exist as a real column until the session is created. + * parameters table), built against the session's preprocessing step shape + * (`{converter, params, scope}`). Its output and status come from the + * chain's estimated structure: a step that cannot work is outlined in red + * with the reason, a step after it is dimmed (it cannot be checked until + * the earlier one is fixed), and warnings are shown under the table. */ function SessionConverterCard({ step, @@ -170,54 +173,29 @@ function SessionConverterCard({ displayName, description, onDelete, - datasetTypes, - preprocessing, + stepStructure, stepDisplayNames, }) { const theme = useTheme(); const { t } = useTranslation(["datasets", "models", "common"]); const localization = useTableLocalization(); - - const { columnTypes } = buildColumnKeysAndTypes({ - datasetTypes, - preprocessing, - }); - // Every earlier step's group label(s) use its resolved display name (e.g. - // "Bag of Words: output"), matching this card's own output label(s) - // below — not the raw registry name buildColumnKeysAndTypes falls back - // to when it has no display-name lookup of its own. One label per - // declared slot, so a step whose scope mixed types (more than one slot) - // gets one distinguishable label per slot. - const optionLabels = {}; - preprocessing.forEach((s, i) => { - const name = stepDisplayNames[i]; - const slots = s.outputSlots?.length > 0 ? s.outputSlots : [{ slot: null }]; - // No "(slot)" suffix here: RefChip already shows the slot's type as its - // own colored badge right next to this label, so repeating it as text - // would be redundant (a step with several slots gets several chips - // with identical text but different badges, which is enough to tell - // them apart). - slots.forEach(({ slot }) => { - const key = groupKey(i, slot); - optionLabels[key] = `${name}: output`; - }); - }); - const ownSlots = - step.outputSlots?.length > 0 ? step.outputSlots : [{ slot: null }]; - const outputEntries = ownSlots.map(({ slot }) => { - const key = groupKey(index, slot); - return { key, label: optionLabels[key] || `${displayName}: output` }; - }); + const status = stepStructure?.status; return ( + {status === "error" && ( + + {formatStructureMessage(t, stepStructure.error)} + + )} + {status === "blocked" && ( + + {t("models:structure.blocked")} + + )} + {(stepStructure?.warnings || []).map((warning, i) => ( + + {formatStructureMessage(t, warning)} + + ))} ); @@ -274,26 +271,51 @@ SessionConverterCard.propTypes = { displayName: PropTypes.string.isRequired, description: PropTypes.string, onDelete: PropTypes.func.isRequired, - datasetTypes: PropTypes.object, - preprocessing: PropTypes.array, + stepStructure: PropTypes.object, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, +}; + +/** + * The estimated dataset state at the end of the chain: what the model + * inputs will be picked from in the next step. + */ +function DatasetStatePanel({ finalState, stepDisplayNames }) { + const { t } = useTranslation(["models"]); + return ( + + }> + + {t("models:structure.currentState")} + + + + + + + ); +} + +DatasetStatePanel.propTypes = { + finalState: PropTypes.array.isRequired, stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, }; /** * Cards for every converter already added to the session's preprocessing - * sequence, mirroring the notebook module's applied-tool cards. + * sequence, mirroring the notebook module's applied-tool cards, under a + * panel with the estimated dataset state after the whole chain. * - * Deleting a converter cascades: any later converter whose scope - * references its output group would be left pointing at a step that no - * longer exists, so every converter configured after it is removed too. - * Confirmed first via the same `DeleteConfirmationModal` + - * `ItemsToDeleteList` the notebook's own converter deletion uses, so the - * user sees exactly what else is about to go before confirming. + * Deleting a converter cascades: any later converter could reference what + * it produces and would be left pointing at a step that no longer exists, + * so every converter configured after it is removed too. Confirmed first + * via the same `DeleteConfirmationModal` + `ItemsToDeleteList` the + * notebook's own converter deletion uses, so the user sees exactly what + * else is about to go before confirming. */ export default function AppliedConvertersView({ newExp, setNewExp, - datasetTypes, + structure, }) { const { t } = useTranslation(["experiments", "models", "datasets", "common"]); const theme = useTheme(); @@ -307,7 +329,7 @@ export default function AppliedConvertersView({ // A converter is draggable from SessionConvertersRightBar's ToolList/ // ToolGrid for free (drag-start lives in the shared ToolListItem/ // ToolGridItem, not in those wrappers), so this view only needs to be a - // drop target — same "application/x-dashai-tool" payload and the same + // drop target: same "application/x-dashai-tool" payload and the same // setPendingDropTool hand-off Notebooks' own NotebookView.jsx uses, which // ToolList/ToolGrid already resolve through their normal click-to-add path. useEffect(() => { @@ -380,7 +402,7 @@ export default function AppliedConvertersView({ // Numbers duplicate converter types ("Simple Imputer" / "Simple Imputer // (2)") so two steps of the same type never render with identical names - // — see buildStepDisplayNames. + // (see buildStepDisplayNames). const stepDisplayNames = buildStepDisplayNames(steps, convertersMeta); const itemsToDelete = useMemo(() => { @@ -408,7 +430,7 @@ export default function AppliedConvertersView({ onDrop={handleDrop} sx={{ position: "relative", - // A percentage minHeight is unreliable here — this Box is a flex + // A percentage minHeight is unreliable here: this Box is a flex // item inside CreateSessionSteps' flex-column scroll container, and // percentage heights don't resolve dependably through it. flex: 1 // makes it actually fill the remaining vertical space (with content @@ -456,6 +478,13 @@ export default function AppliedConvertersView({ )} + {structure && ( + + )} + {steps.length === 0 ? ( {t("experiments:label.noConverterAdded")} @@ -473,8 +502,7 @@ export default function AppliedConvertersView({ meta?.description || meta?.metadata?.short_description } onDelete={(i) => setDeleteIndex(i)} - datasetTypes={datasetTypes} - preprocessing={steps} + stepStructure={structure?.steps?.[index]} stepDisplayNames={stepDisplayNames} /> ); @@ -503,5 +531,7 @@ export default function AppliedConvertersView({ AppliedConvertersView.propTypes = { newExp: PropTypes.object.isRequired, setNewExp: PropTypes.func.isRequired, - datasetTypes: PropTypes.object, + // The chain's estimated structure (see usePreprocessingStructure); null + // until the first estimate arrives. + structure: PropTypes.object, }; diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx index 6b8f360bf..71cd18d81 100644 --- a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx @@ -38,7 +38,33 @@ const AVAILABLE_CONVERTERS = [ }, ]; -const datasetTypes = { age: { type: "Integer" }, text: { type: "Text" } }; +const column = (name, type, origin) => ({ + kind: "column", + name, + type, + dtype: null, + origin, +}); + +const block = (step, slot, type, count) => ({ + kind: "block", + step, + slot, + label: "output", + type, + dtype: null, + count, +}); + +const okStep = (added) => ({ + status: "ok", + state: [], + added, + error: null, + warnings: [], +}); + +const structureOf = (steps) => ({ initial: [], steps, final: [], valid: true }); describe("AppliedConvertersView", () => { let setPendingDropTool; @@ -53,117 +79,176 @@ describe("AppliedConvertersView", () => { renderView({ newExp: { preprocessing: [] }, setNewExp: () => {}, - datasetTypes, }); expect(screen.getByText("No converter was added.")).toBeInTheDocument(); }); - it("shows a converter's real display name, scope and output type chip", async () => { + it("shows a converter's real display name, scope and estimated output", async () => { const newExp = { preprocessing: [ { converter: "Binarizer", params: { threshold: 0.5 }, scope: [{ kind: "raw", name: "age" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; + const structure = structureOf([okStep([column("bin_age", "Integer", 0)])]); - renderView({ newExp, setNewExp: () => {}, datasetTypes }); + renderView({ newExp, setNewExp: () => {}, structure }); expect(await screen.findByText("Binarizador")).toBeInTheDocument(); expect(screen.getByText("age")).toBeInTheDocument(); - expect(screen.getByText("Binarizador: output")).toBeInTheDocument(); + expect(screen.getAllByText("bin_age").length).toBeGreaterThan(0); expect(screen.getAllByText("Integer").length).toBeGreaterThan(0); }); - it("shows a chained converter's scope as the earlier converter's output group", async () => { + it("shows a chained converter's scope as the earlier converter's output", async () => { const newExp = { preprocessing: [ { converter: "BagOfWordsConverter", params: {}, scope: [{ kind: "raw", name: "text" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, { converter: "Binarizer", params: {}, scope: [{ kind: "group", step: 0 }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; + const structure = structureOf([ + okStep([block(0, null, "Integer", null)]), + okStep([block(1, null, "Integer", null)]), + ]); - renderView({ newExp, setNewExp: () => {}, datasetTypes }); + renderView({ newExp, setNewExp: () => {}, structure }); await screen.findByText("Bag of Words"); - // The second card's scope shows the first converter's output label. - expect(screen.getAllByText("Bag of Words: output").length).toBe(2); + // The second card's scope names the first converter's output. + expect(screen.getByText("Bag of Words: output")).toBeInTheDocument(); + // Each card's output block shows its size, N when unknown. + expect(screen.getAllByText("Bag of Words: output (N)").length).toBe(1); }); - it("shows one chip per declared slot when a step's scope mixed column types", async () => { + it("shows one chip per estimated block when a step's output mixes types", async () => { const newExp = { preprocessing: [ { - converter: "SimpleImputer", - params: { strategy: "most_frequent" }, + converter: "SelectKBest", + params: { k: 1 }, scope: [ { kind: "raw", name: "age" }, - { kind: "raw", name: "text" }, - ], - outputSlots: [ - { slot: "Integer", type: "Integer", dtype: "int64" }, - { slot: "Categorical", type: "Categorical", dtype: null }, + { kind: "raw", name: "score" }, ], }, ], }; - - renderView({ - newExp, - setNewExp: () => {}, - datasetTypes: { - ...datasetTypes, - // "SimpleImputer" isn't in AVAILABLE_CONVERTERS, so its display - // name falls back to the raw converter name — fine here, this - // test only cares about the output chips. - }, - }); - - // No "(slot)" suffix: each slot already gets its own colored type chip - // (asserted below), so both chips share the same label text. - expect(await screen.findAllByText("SimpleImputer: output")).toHaveLength(2); - expect(screen.getByText("Integer")).toBeInTheDocument(); - expect(screen.getByText("Categorical")).toBeInTheDocument(); + const structure = structureOf([ + okStep([ + block(0, "Integer", "Integer", null), + block(0, "Float", "Float", null), + ]), + ]); + + renderView({ newExp, setNewExp: () => {}, structure }); + + expect(await screen.findAllByText("SelectKBest: output (N)")).toHaveLength( + 2, + ); + expect(screen.getByText("Float")).toBeInTheDocument(); }); - it("numbers two converters of the same type so their cards and output labels don't collide", async () => { + it("numbers two converters of the same type so their cards don't collide", async () => { const newExp = { preprocessing: [ { converter: "Binarizer", params: { threshold: 0.5 }, scope: [{ kind: "raw", name: "age" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, { converter: "Binarizer", params: { threshold: 1 }, scope: [{ kind: "raw", name: "text" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; - renderView({ newExp, setNewExp: () => {}, datasetTypes }); + renderView({ newExp, setNewExp: () => {}, structure: null }); expect(await screen.findByText("Binarizador")).toBeInTheDocument(); expect(screen.getByText("Binarizador (2)")).toBeInTheDocument(); - expect(screen.getByText("Binarizador: output")).toBeInTheDocument(); - expect(screen.getByText("Binarizador (2): output")).toBeInTheDocument(); + }); + + it("flags a step that cannot work and dims the ones after it", async () => { + const newExp = { + preprocessing: [ + { + converter: "Binarizer", + params: {}, + scope: [{ kind: "raw", name: "age" }], + }, + { + converter: "Binarizer", + params: {}, + scope: [{ kind: "raw", name: "text" }], + }, + ], + }; + const structure = { + ...structureOf([]), + valid: false, + steps: [ + { + status: "error", + state: [], + added: [], + error: { code: "missing_ref", params: { ref: "age" } }, + warnings: [], + }, + { status: "blocked", state: [], added: [], error: null, warnings: [] }, + ], + }; + + renderView({ newExp, setNewExp: () => {}, structure }); + + expect( + await screen.findByText( + '"age" does not exist at this point of the chain: an earlier step may have consumed it.', + ), + ).toBeInTheDocument(); + expect(screen.getByTestId("session-converter-card-0")).toHaveAttribute( + "data-status", + "error", + ); + expect(screen.getByTestId("session-converter-card-1")).toHaveAttribute( + "data-status", + "blocked", + ); + expect( + screen.getByText("Fix the previous step to check this one."), + ).toBeInTheDocument(); + }); + + it("shows the dataset state after the whole chain", async () => { + const structure = { + ...structureOf([]), + final: [column("age", "Integer", null), block(0, null, "Float", 2)], + }; + + renderView({ + newExp: { preprocessing: [] }, + setNewExp: () => {}, + structure, + }); + + expect( + await screen.findByText("Dataset after preprocessing"), + ).toBeInTheDocument(); + expect(screen.getByText("age")).toBeInTheDocument(); }); it("cascades deletion to every converter configured after the deleted one", async () => { @@ -174,18 +259,16 @@ describe("AppliedConvertersView", () => { converter: "BagOfWordsConverter", params: {}, scope: [{ kind: "raw", name: "text" }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, { converter: "Binarizer", params: {}, scope: [{ kind: "group", step: 0 }], - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], }, ], }; - renderView({ newExp, setNewExp, datasetTypes }); + renderView({ newExp, setNewExp, structure: null }); await screen.findByText("Bag of Words"); const deleteButtons = screen.getAllByRole("button", { name: "Remove" }); @@ -211,7 +294,6 @@ describe("AppliedConvertersView", () => { renderView({ newExp: { preprocessing: [] }, setNewExp: () => {}, - datasetTypes, }); const dropZone = screen.getByText("No converter was added.").parentElement; @@ -225,7 +307,6 @@ describe("AppliedConvertersView", () => { renderView({ newExp: { preprocessing: [] }, setNewExp: () => {}, - datasetTypes, }); const dropZone = screen.getByText("No converter was added.").parentElement; diff --git a/DashAI/front/src/components/models/modelSession/BaseColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/BaseColumnsStep.jsx new file mode 100644 index 000000000..37b285aad --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/BaseColumnsStep.jsx @@ -0,0 +1,96 @@ +import React, { useEffect, useLayoutEffect, useState } from "react"; +import PropTypes from "prop-types"; +import { Grid } from "@mui/material"; +import { useTranslation } from "react-i18next"; +import DivideDatasetColumns from "./DivideDatasetColumns"; + +/** + * Step of the session wizard shown before preprocessing: pick the output + * column and the candidate columns, all original dataset columns. + * + * The candidates are the columns the preprocessing chain may work on; the + * model inputs are chosen from what the chain leaves, in the last step. The + * output is never part of the candidates, so no converter can transform it. + * Nothing is validated against the task here: a candidate may be of a type + * the task rejects (e.g. text) precisely because a converter will turn it + * into something it accepts. + */ +function BaseColumnsStep({ + newExp, + setNewExp, + setNextEnabled, + datasetInfo, + datasetTypes, +}) { + const { t } = useTranslation(["models", "common"]); + const rawColumnNames = datasetInfo.column_names || []; + + const [outputColumnNames, setOutputColumnNames] = useState( + newExp.output_columns || [], + ); + const [candidateSelection, setCandidateSelection] = useState( + newExp.candidate_columns || [], + ); + + // The output is never a candidate, even if it was picked as one before. + const candidateOptions = rawColumnNames.filter( + (name) => !outputColumnNames.includes(name), + ); + const candidates = candidateSelection.filter( + (name) => !outputColumnNames.includes(name), + ); + + useLayoutEffect(() => { + // Default the first time the dataset's columns load: the last column is + // the output and every other one is a candidate. + if (rawColumnNames.length === 0) return; + if (outputColumnNames.length === 0 && candidateSelection.length === 0) { + const output = rawColumnNames[rawColumnNames.length - 1]; + setOutputColumnNames([output]); + setCandidateSelection(rawColumnNames.filter((name) => name !== output)); + } + }, [rawColumnNames.join(",")]); + + const ready = candidates.length >= 1 && outputColumnNames.length >= 1; + + useEffect(() => { + setNewExp((prev) => ({ + ...prev, + candidate_columns: candidates, + output_columns: outputColumnNames, + })); + setNextEnabled(ready); + }, [candidates.join(","), outputColumnNames.join(",")]); + + return ( + + + + ); +} + +BaseColumnsStep.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + setNextEnabled: PropTypes.func.isRequired, + datasetInfo: PropTypes.object, + datasetTypes: PropTypes.object, +}; + +export default BaseColumnsStep; diff --git a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx index 1508d1dc5..52424b698 100644 --- a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx +++ b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx @@ -26,6 +26,8 @@ function DivideDatasetColumns({ outputError = false, outputHelperText = "", disabled = false, + inputLabel, + outputDisabled = false, }) { const { t } = useTranslation(["experiments", "common", "models"]); const theme = useTheme(); @@ -145,7 +147,7 @@ function DivideDatasetColumns({ )} sx={{ mb: 8 }} - disabled={disabled || allColumnNames.length === 0} + disabled={disabled || outputDisabled || allColumnNames.length === 0} /> ); @@ -209,6 +211,10 @@ DivideDatasetColumns.propTypes = { outputError: PropTypes.bool, outputHelperText: PropTypes.string, disabled: PropTypes.bool, + // Overrides the input selector's label (defaults to "Input columns"). + inputLabel: PropTypes.string, + // Shows the output as fixed, e.g. when it was chosen in an earlier step. + outputDisabled: PropTypes.bool, }; export default DivideDatasetColumns; diff --git a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx index d6520bdc0..c6093685f 100644 --- a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx +++ b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx @@ -3,16 +3,16 @@ import PropTypes from "prop-types"; import { Box } from "@mui/material"; import ScopeStepSessionConverter from "./ScopeStepSessionConverter"; import ParameterStepConverter from "../../notebooks/converterCreation/ParameterStepConverter"; -import { resolveDeclaredOutputSlots } from "./sessionColumnRefs"; /** - * "Add a converter" form for the session wizard's preprocessing step — - * mirrors the notebook's FormConverterSection (same two-step scope/params - * flow, same ParameterStepConverter for the second step), but a session + * "Add a converter" form for the session wizard's preprocessing step, + * mirroring the notebook's FormConverterSection (same two-step scope/params + * flow, same ParameterStepConverter for the second step). A session * converter is never fit immediately: saving here only appends * `{converter, params, scope}` to the session's local `preprocessing` list. - * The actual fit happens once, later, when the session is created (see the - * backend's PreprocessingJob) — never from this form. + * What the step produces is estimated by the backend afterwards (see + * usePreprocessingStructure), and the actual fit happens once, when the + * session is created (see the backend's PreprocessingJob). */ export default function FormSessionConverterSection({ step, @@ -21,26 +21,18 @@ export default function FormSessionConverterSection({ tool, newExp, setNewExp, - datasetTypes, + finalState, + stepDisplayNames, filePath, hideButtons = false, - convertersMeta, }) { const [scope, setScope] = useState([]); const handleSaveConverter = async (params) => { - const outputSlots = resolveDeclaredOutputSlots({ - tool, - params, - scope, - datasetTypes, - preprocessing: newExp.preprocessing, - }); const newStep = { converter: tool.name, params: params || {}, scope, - outputSlots, }; setNewExp({ ...newExp, @@ -63,12 +55,11 @@ export default function FormSessionConverterSection({ {step === 0 && ( 0 ? () => setStep((s) => s + 1) @@ -86,6 +77,7 @@ export default function FormSessionConverterSection({ handleSaveConverter={handleSaveConverter} setStep={setStep} hideButtons={hideButtons} + warnAboutLeakage={false} /> )} @@ -99,8 +91,10 @@ FormSessionConverterSection.propTypes = { tool: PropTypes.object.isRequired, newExp: PropTypes.object.isRequired, setNewExp: PropTypes.func.isRequired, - datasetTypes: PropTypes.object.isRequired, + // The estimated dataset state at the end of the chain, where the new + // converter will be appended. + finalState: PropTypes.array.isRequired, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, filePath: PropTypes.string, hideButtons: PropTypes.bool, - convertersMeta: PropTypes.object, }; diff --git a/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx b/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx index 41a6d2051..d9f05e5c9 100644 --- a/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx +++ b/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx @@ -1,33 +1,41 @@ import React, { useEffect } from "react"; import PropTypes from "prop-types"; +import { Alert } from "@mui/material"; +import { useTranslation } from "react-i18next"; import { useModels } from "../ModelsContext"; import SessionConvertersRightBar from "./SessionConvertersRightBar"; import AppliedConvertersView from "./AppliedConvertersView"; +import usePreprocessingStructure from "./usePreprocessingStructure"; /** - * Optional preprocessing step of the session wizard: pick converters to fit - * on training data only (see the backend's PreprocessingJob), and configure - * each one's scope. A converter's scope may include the not-yet-materialized - * output group of an earlier converter in the same list, which is how - * chaining works before any real fit exists. + * Optional preprocessing step of the session wizard: build a chain of + * converters, fit on training data only (see the backend's + * PreprocessingJob), over the candidate columns picked in BaseColumnsStep. + * + * The dataset structure along the chain is estimated by the backend + * (usePreprocessingStructure) on every change: a new converter can only be + * scoped on columns that exist at the end of the chain, each card shows + * what its step produces, and a step that cannot work is flagged and blocks + * moving on. * * Styled like the notebook module's own converter picker: the catalog * (search + category list/grid) lives in the right bar * (SessionConvertersRightBar), the already-added converters are shown as * cards in the main content area (AppliedConvertersView). */ -function PreprocessingStep({ - newExp, - setNewExp, - setNextEnabled, - dataset, - datasetTypes, -}) { +function PreprocessingStep({ newExp, setNewExp, setNextEnabled, dataset }) { + const { t } = useTranslation(["models"]); const { setSessionRightContent } = useModels(); + const { structure, loading, error } = usePreprocessingStructure({ + datasetId: dataset?.id, + candidates: newExp.candidate_columns, + outputColumns: newExp.output_columns, + steps: newExp.preprocessing, + }); useEffect(() => { - setNextEnabled(true); - }, [setNextEnabled]); + setNextEnabled(Boolean(structure?.valid) && !loading && !error); + }, [structure, loading, error, setNextEnabled]); useEffect(() => { setSessionRightContent( @@ -35,18 +43,25 @@ function PreprocessingStep({ newExp={newExp} setNewExp={setNewExp} dataset={dataset} - datasetTypes={datasetTypes} + structure={structure} />, ); return () => setSessionRightContent(null); - }, [newExp, setNewExp, dataset, datasetTypes]); + }, [newExp, setNewExp, dataset, structure]); return ( - + <> + {error && ( + + {t("models:structure.loadError")} + + )} + + ); } @@ -55,7 +70,6 @@ PreprocessingStep.propTypes = { setNewExp: PropTypes.func.isRequired, setNextEnabled: PropTypes.func.isRequired, dataset: PropTypes.object, - datasetTypes: PropTypes.object, }; export default PreprocessingStep; diff --git a/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx index e2363e960..989d12d3d 100644 --- a/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx +++ b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx @@ -5,32 +5,32 @@ import { useTheme } from "@mui/material/styles"; import { useTranslation } from "react-i18next"; import ColumnSelector from "../../notebooks/ColumnSelector"; import FormSchemaButtonGroup from "../../shared/FormSchemaButtonGroup"; -import { buildColumnKeysAndTypes, keyToRef } from "./sessionColumnRefs"; +import { keyToRef, stateToOptions } from "./sessionColumnRefs"; /** * Scope step for a converter being added to a session's preprocessing - * sequence: column selection only (there's no row-level scope — the + * sequence: column selection only. There's no row-level scope: the * train/test split doesn't exist yet at config time, and a session * converter never sees rows outside its own fold's training partition at - * fit time regardless of what's picked here). + * fit time regardless of what's picked here. * - * The scope offered isn't just the dataset's raw columns: it's every raw - * column plus one synthetic key per output group of every converter - * already configured *before* this one in the sequence (chaining). The - * shared `ColumnSelector` only knows a plain `{name: {type, dtype}}` map, - * so groups are represented as synthetic keys built by `sessionColumnRefs` - * and translated back into real ColumnRef scope entries once the user - * picks a selection. + * The scope offered is the estimated dataset state at the end of the chain, + * where this converter is appended (chaining): the candidate columns that + * are still there, plus every column or block earlier converters produced, + * and never a column an earlier step consumed or the output column. The + * shared `ColumnSelector` only knows a plain `{name: {type, dtype}}` map, so + * each state item is represented by the synthetic key of the ColumnRef + * pointing at it (see `sessionColumnRefs`) and translated back into that + * ColumnRef once the user picks a selection. */ export default function ScopeStepSessionConverter({ tool, - datasetTypes, - preprocessing, + finalState, + stepDisplayNames, filePath, scope, setScope, nextStep, - convertersMeta, }) { const theme = useTheme(); const { t } = useTranslation(["common", "datasets", "models"]); @@ -42,9 +42,8 @@ export default function ScopeStepSessionConverter({ const inputCardinality = tool?.metadata?.input_cardinality || {}; const { columnTypes: columnTypesForSelector, optionLabels } = useMemo( - () => - buildColumnKeysAndTypes({ datasetTypes, preprocessing, convertersMeta }), - [datasetTypes, preprocessing, convertersMeta], + () => stateToOptions(finalState, stepDisplayNames), + [finalState, stepDisplayNames], ); const handleSelectionChange = (selected) => { @@ -102,11 +101,10 @@ export default function ScopeStepSessionConverter({ ScopeStepSessionConverter.propTypes = { tool: PropTypes.object.isRequired, - datasetTypes: PropTypes.object.isRequired, - preprocessing: PropTypes.array, + finalState: PropTypes.array.isRequired, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, filePath: PropTypes.string, scope: PropTypes.array.isRequired, setScope: PropTypes.func.isRequired, nextStep: PropTypes.func.isRequired, - convertersMeta: PropTypes.object, }; diff --git a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx index af54e0bfb..846514d26 100644 --- a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx +++ b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx @@ -19,16 +19,22 @@ import { isGroupKey, refToKey, keyToRef, - groupKey, - buildColumnKeysAndTypes, + buildStepDisplayNames, + stateToOptions, } from "./sessionColumnRefs"; +import usePreprocessingStructure from "./usePreprocessingStructure"; /** - * Step of the session wizard: pick input/output columns. A column can be a - * raw dataset column, or (when preprocessing is configured) the whole output - * group of one of the session's converter steps, represented here as a - * synthetic key (e.g. "__group__0") so the underlying Autocomplete only - * ever deals with plain strings — the same shape a raw column name has. + * Last step of the session wizard: pick the model's input columns (and, + * without preprocessing, its output column). + * + * With preprocessing, the inputs are picked from the estimated dataset + * state after the whole chain (see usePreprocessingStructure): surviving + * candidate columns plus what the converters produced, each represented by + * the synthetic key of the ColumnRef pointing at it, so the underlying + * Autocomplete only ever deals with plain strings. The output was already + * chosen in BaseColumnsStep and is shown fixed. Every item of the final + * state is selected by default. */ function SelectColumnsStep({ newExp, @@ -45,18 +51,29 @@ function SelectColumnsStep({ const [convertersMeta, setConvertersMeta] = useState({}); const rawColumnNames = datasetInfo.column_names || []; + const withPreprocessing = Boolean(newExp.applyPreprocessing); + const steps = newExp.preprocessing || []; - const { - allKeys: inputOptionNames, - columnTypes: columnTypesForSelector, - optionLabels, - } = buildColumnKeysAndTypes({ - datasetTypes, - preprocessing: newExp.preprocessing, - convertersMeta, + const { structure } = usePreprocessingStructure({ + datasetId: withPreprocessing ? dataset.id : undefined, + candidates: newExp.candidate_columns, + outputColumns: newExp.output_columns, + steps, }); - const [inputSelection, setInputSelection] = useState(() => + const stepDisplayNames = buildStepDisplayNames(steps, convertersMeta); + const finalOptions = stateToOptions(structure?.final, stepDisplayNames); + const inputOptionNames = withPreprocessing + ? finalOptions.allKeys + : rawColumnNames; + // The output keeps its dataset type; an input takes its type after the + // chain (e.g. a column cast in place by TypeCast). + const columnTypesForSelector = withPreprocessing + ? { ...datasetTypes, ...finalOptions.columnTypes } + : datasetTypes; + const optionLabels = withPreprocessing ? finalOptions.optionLabels : {}; + + const [inputSelectionState, setInputSelection] = useState(() => newExp.input_column_refs && newExp.input_column_refs.length > 0 ? newExp.input_column_refs.map(refToKey) : newExp.input_columns || [], @@ -65,41 +82,40 @@ function SelectColumnsStep({ newExp.output_columns || [], ); + // Only what can still be picked: going back and changing the chain can + // consume a column picked here before, and turning preprocessing off + // leaves only original columns. + const optionsKnown = withPreprocessing + ? Boolean(structure) + : rawColumnNames.length > 0; + const inputSelection = optionsKnown + ? inputSelectionState.filter((key) => inputOptionNames.includes(key)) + : inputSelectionState; + const inputColumnRefs = inputSelection.map(keyToRef); const rawInputNames = inputSelection.filter((key) => !isGroupKey(key)); const columnsReady = - inputSelection.length >= 1 && outputColumnNames.length >= 1; + inputSelection.length >= 1 && + outputColumnNames.length >= 1 && + (!withPreprocessing || Boolean(structure)); const [columnsAreValid, setColumnsAreValid] = useState(false); const [validationPending, setValidationPending] = useState(true); useLayoutEffect(() => { - // Auto-select sensible defaults the first time the dataset's columns load. - if (rawColumnNames.length === 0) return; + // Auto-select sensible defaults the first time the dataset's columns + // load. With preprocessing, the output comes from BaseColumnsStep and + // the inputs default once the estimated structure arrives (below). + if (rawColumnNames.length === 0 || withPreprocessing) return; if ( - inputSelection.length === 0 && + inputSelectionState.length === 0 && (!newExp.input_columns || newExp.input_columns.length === 0) ) { - const steps = newExp.preprocessing || []; - if (steps.length > 0) { - // With preprocessing configured, default to just the last step's - // output group(s) — the point of building a chain is usually to - // end up using its final result, not the raw columns it started - // from. Every raw column and every other group stays available to - // pick instead; this is only the starting default. - const lastIndex = steps.length - 1; - const slots = - steps[lastIndex].outputSlots?.length > 0 - ? steps[lastIndex].outputSlots - : [{ slot: null }]; - setInputSelection(slots.map(({ slot }) => groupKey(lastIndex, slot))); - } else { - setInputSelection( - rawColumnNames.length > 1 - ? rawColumnNames.slice(0, -1) - : [rawColumnNames[0]], - ); - } + setInputSelection( + rawColumnNames.length > 1 + ? rawColumnNames.slice(0, -1) + : [rawColumnNames[0]], + ); } if ( outputColumnNames.length === 0 && @@ -109,6 +125,15 @@ function SelectColumnsStep({ } }, [rawColumnNames.join(",")]); + useLayoutEffect(() => { + // With preprocessing, default to everything the chain leaves: the model + // is usually meant to use the chain's whole result. + if (!withPreprocessing || !structure) return; + if (inputSelection.length === 0) { + setInputSelection(finalOptions.allKeys); + } + }, [structure]); + const getTaskRequirements = async () => { try { const taskComponents = await getComponentsRequest({ @@ -146,29 +171,15 @@ function SelectColumnsStep({ setColumnsAreValid(false); return; } - const hasGroupRef = inputSelection.some(isGroupKey); - // Keyed "{step}" for a step with one declared type (unslotted, the - // common case) or "{step}:{slot}" for one of several — matches the - // key scheme validate_columns looks declared_type up by backend-side - // (see model_sessions.py), which mirrors GroupColumnRef.slot. - const converterOutputTypes = {}; - (newExp.preprocessing || []).forEach((step, index) => { - const slots = - step.outputSlots?.length > 0 - ? step.outputSlots - : [{ slot: null, type: step.outputType || null }]; - slots.forEach(({ slot, type }) => { - const key = slot == null ? String(index) : `${index}:${slot}`; - converterOutputTypes[key] = type || null; - }); - }); + // With preprocessing, the backend types every input from the chain's + // estimated structure, the same one this step's options come from. const validation = await validateColumnsRequest( newExp.task_name, dataset.id, - hasGroupRef ? rawInputNames : inputSelection, + rawInputNames, outputColumnNames, - hasGroupRef ? inputColumnRefs : undefined, - hasGroupRef ? converterOutputTypes : undefined, + withPreprocessing ? inputColumnRefs : undefined, + withPreprocessing ? steps : undefined, ); setColumnsAreValid(validation.dataset_status === "valid"); } catch (error) { @@ -216,9 +227,9 @@ function SelectColumnsStep({ }, []); useEffect(() => { - // Only needed to disambiguate two steps of the same converter type in - // the option labels below (e.g. "Simple Imputer" / "Simple Imputer - // (2)") — see buildStepDisplayNames. + // Only needed to name the steps in the option labels, disambiguating two + // of the same converter type (e.g. "Simple Imputer" / "Simple Imputer + // (2)"), see buildStepDisplayNames. let cancelled = false; getComponentsRequest({ selectTypes: ["Converter"] }) .then((data) => { @@ -375,6 +386,7 @@ function SelectColumnsStep({ outputColumnNames.length === 0 ? t("common:required") : "" } disabled={rawColumnNames.length === 0} + outputDisabled={withPreprocessing} /> diff --git a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx index 3a71c5c6c..aab8cdf77 100644 --- a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx +++ b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx @@ -16,21 +16,27 @@ import ToolGrid from "../../notebooks/tool/ToolGrid"; import { getComponents } from "../../../api/component"; import { evaluateColumnEligibility } from "../../../utils/columnEligibility"; import FormSessionConverterSection from "./FormSessionConverterSection"; -import { buildColumnKeysAndTypes } from "./sessionColumnRefs"; +import { buildStepDisplayNames, stateToOptions } from "./sessionColumnRefs"; /** * Converters-only sidebar for the session wizard's preprocessing step, * styled like the notebook module's own converters picker (same SearchBar, * ToolList/ToolGrid, category accordion, list/grid toggle). Unlike the * notebook flow, picking a converter here never fits anything or POSTs - * anywhere — see FormSessionConverterSection, which just appends the - * configured step to newExp.preprocessing. + * anywhere: FormSessionConverterSection just appends the configured step to + * newExp.preprocessing. + * + * A new converter always lands at the end of the chain, so it is offered + * (and scoped) against the estimated final dataset state: the surviving + * candidate columns plus everything earlier steps produced, never the + * output column. Converters that change the number of rows are hidden, + * since sessions do not support them yet. */ export default function SessionConvertersRightBar({ newExp, setNewExp, dataset, - datasetTypes, + structure, }) { const theme = useTheme(); const { t } = useTranslation(["models", "datasets", "common"]); @@ -56,20 +62,22 @@ export default function SessionConvertersRightBar({ }; }, [t]); - // A new converter can be scoped on any raw dataset column OR the output - // group (any declared slot) of any converter already configured — it - // always lands at the end of the sequence, so every existing step is - // "before" it and fair game to chain off. Gating this on raw columns - // alone wrongly blocked, e.g., PCA on a text-only dataset (no raw - // Integer/Float columns) even after adding Bag of Words, whose Integer - // output group PCA could legitimately scope on. + // Type-level friendly names (e.g. "Simple Imputer" for "SimpleImputer"), + // used to label earlier steps' output blocks, numbered the same way + // AppliedConvertersView numbers repeated converter types. + const convertersMeta = useMemo( + () => Object.fromEntries(converters.map((c) => [c.name, c])), + [converters], + ); + const stepDisplayNames = useMemo( + () => buildStepDisplayNames(newExp.preprocessing, convertersMeta), + [newExp.preprocessing, convertersMeta], + ); + const finalState = structure?.final; + const { columnTypes: allColumnTypes } = useMemo( - () => - buildColumnKeysAndTypes({ - datasetTypes, - preprocessing: newExp.preprocessing, - }), - [datasetTypes, newExp.preprocessing], + () => stateToOptions(finalState, stepDisplayNames), + [finalState, stepDisplayNames], ); const datasetColumns = useMemo( @@ -120,21 +128,14 @@ export default function SessionConvertersRightBar({ return { disabled, tooltip, validColumns }; }; - // Type-level friendly names (e.g. "Simple Imputer" for "SimpleImputer"), - // reused when scoping a new converter over an earlier one's output group - // — see FormSessionConverterSection -> ScopeStepSessionConverter, which - // disambiguates repeated types the same way AppliedConvertersView does. - const convertersMeta = useMemo( - () => Object.fromEntries(converters.map((c) => [c.name, c])), - [converters], - ); - const validatedConverters = useMemo( () => - converters.map((converter) => { - const validation = validateConverter(converter); - return { ...converter, ...validation }; - }), + converters + .filter((converter) => converter.metadata?.column_operation !== "rows") + .map((converter) => { + const validation = validateConverter(converter); + return { ...converter, ...validation }; + }), [converters, datasetColumns, t], ); @@ -149,8 +150,8 @@ export default function SessionConvertersRightBar({ }, [searchQuery, validatedConverters]); // Stable identity across re-renders (e.g. every keystroke in the search - // box) so ConfigureToolModal never remounts FormSessionConverterSection - // — and loses in-progress scope/parameter input — while it's open. + // box) so ConfigureToolModal never remounts FormSessionConverterSection, + // and loses in-progress scope/parameter input, while it's open. const SessionFormSection = useMemo(() => { function Wrapped(sectionProps) { return ( @@ -158,14 +159,14 @@ export default function SessionConvertersRightBar({ {...sectionProps} newExp={newExp} setNewExp={setNewExp} - datasetTypes={datasetTypes} + finalState={finalState || []} + stepDisplayNames={stepDisplayNames} filePath={dataset?.file_path} - convertersMeta={convertersMeta} /> ); } return Wrapped; - }, [newExp, setNewExp, datasetTypes, dataset, convertersMeta]); + }, [newExp, setNewExp, finalState, stepDisplayNames, dataset]); return ( slot == null ? `${GROUP_KEY_PREFIX}${step}` : `${GROUP_KEY_PREFIX}${step}${SLOT_SEPARATOR}${slot}`; +// A generated column whose name is known before fit (e.g. DateFeatures' +// "date_month"), mirroring the backend's GroupColumnRef.name. +export const namedGroupKey = (step, name) => + `${GROUP_KEY_PREFIX}${step}${NAME_SEPARATOR}${name}`; + export const isGroupKey = (key) => typeof key === "string" && key.startsWith(GROUP_KEY_PREFIX); -export const stepFromGroupKey = (key) => { - const withoutPrefix = key.slice(GROUP_KEY_PREFIX.length); - const separatorIndex = withoutPrefix.indexOf(SLOT_SEPARATOR); - const stepPart = - separatorIndex === -1 - ? withoutPrefix - : withoutPrefix.slice(0, separatorIndex); - return Number(stepPart); +const parseGroupKey = (key) => { + const match = GROUP_KEY_PATTERN.exec(key); + return { + step: Number(match[1]), + slot: match[2] ?? null, + name: match[3] ?? null, + }; }; -export const slotFromGroupKey = (key) => { - const withoutPrefix = key.slice(GROUP_KEY_PREFIX.length); - const separatorIndex = withoutPrefix.indexOf(SLOT_SEPARATOR); - return separatorIndex === -1 - ? null - : withoutPrefix.slice(separatorIndex + SLOT_SEPARATOR.length); -}; +export const stepFromGroupKey = (key) => parseGroupKey(key).step; + +export const slotFromGroupKey = (key) => parseGroupKey(key).slot; /** ColumnRef -> synthetic key */ -export const refToKey = (ref) => - ref.kind === "raw" ? ref.name : groupKey(ref.step, ref.slot ?? null); +export const refToKey = (ref) => { + if (ref.kind === "raw") return ref.name; + if (ref.name != null) return namedGroupKey(ref.step, ref.name); + return groupKey(ref.step, ref.slot ?? null); +}; /** synthetic key -> ColumnRef */ export const keyToRef = (key) => { if (!isGroupKey(key)) return { kind: "raw", name: key }; - const step = stepFromGroupKey(key); - const slot = slotFromGroupKey(key); + const { step, slot, name } = parseGroupKey(key); + if (name != null) return { kind: "group", step, name }; return slot == null ? { kind: "group", step } : { kind: "group", step, slot }; }; -// A step's declared output, normalized to a list of slots — even a -// homogeneous step (the common case) is one "slot" with slot: null, so -// every caller iterates the same shape regardless of how many there are. -// Falls back defensively for a step that predates outputSlots. -function stepOutputSlots(step) { - if (Array.isArray(step?.outputSlots) && step.outputSlots.length > 0) { - return step.outputSlots; +/** + * The ColumnRef that points at an item of the estimated dataset structure + * (see the backend's infer_structure): an original column by name, a + * generated column by its step and name, a block by its step and slot (a + * lone block has no slot: it is its step's whole group). + */ +export const itemToRef = (item) => { + if (item.kind === "column") { + return item.origin == null + ? { kind: "raw", name: item.name } + : { kind: "group", step: item.origin, name: item.name }; } - return [ - { - slot: null, - type: step?.outputType ?? null, - dtype: step?.outputDtype ?? null, - }, - ]; + return item.slot == null + ? { kind: "group", step: item.step } + : { kind: "group", step: item.step, slot: item.slot }; +}; + +const blockLabel = (block, stepName) => { + const base = + block.label && block.label !== "output" + ? `${stepName}: ${block.label}` + : `${stepName}: output`; + // "N" when the column count is only known after fit. + return `${base} (${block.count ?? "N"})`; +}; + +/** + * Selector options for the items of an estimated dataset state: one key per + * item (the synthetic key of the ColumnRef pointing at it), its type, and a + * label for every item that is not an original column. Plugs into the same + * `{allKeys, columnTypes, optionLabels}` shape ColumnSelector and + * DivideDatasetColumns already take. + * + * @param {Array} state items from a StructureResult state + * @param {string[]} stepDisplayNames one name per step (see + * buildStepDisplayNames) + */ +export function stateToOptions(state, stepDisplayNames = []) { + const allKeys = []; + const columnTypes = {}; + const optionLabels = {}; + (state || []).forEach((item) => { + const key = refToKey(itemToRef(item)); + allKeys.push(key); + columnTypes[key] = { type: item.type ?? null, dtype: item.dtype ?? null }; + if (item.kind === "block") { + optionLabels[key] = blockLabel(item, stepDisplayNames[item.step] ?? ""); + } else if (item.origin != null) { + optionLabels[key] = item.name; + } + }); + return { allKeys, columnTypes, optionLabels }; +} + +/** + * A label for a ColumnRef without the estimated structure, for sessions + * that are already created (e.g. the session info panel). + */ +export function labelForRef(ref, stepDisplayNames = []) { + if (ref.kind === "raw") return ref.name; + if (ref.name != null) return ref.name; + const stepName = stepDisplayNames[ref.step] ?? `${ref.step}`; + return ref.slot == null + ? `${stepName}: output` + : `${stepName}: output (${ref.slot})`; } /** * One display name per step, disambiguated when the sequence has more than * one step of the same converter type (same registry name, or same - * fallback string when `convertersMeta` hasn't loaded yet) — the first + * fallback string when `convertersMeta` hasn't loaded yet): the first * occurrence keeps the bare name, later ones get " (2)", " (3)", etc., by - * order of appearance. Computed over the FULL `steps` array regardless of - * any later truncation (e.g. buildColumnKeysAndTypes's `uptoStep`), so a - * step's numbering never shifts depending on which view is asking. + * order of appearance. Computed over the full `steps` array, so a step's + * numbering never shifts depending on which view is asking. */ export function buildStepDisplayNames(steps, convertersMeta = {}) { const baseNames = (steps || []).map( @@ -103,143 +162,11 @@ export function buildStepDisplayNames(steps, convertersMeta = {}) { }); } -/** - * Every column key a session's preprocessing sequence can be scoped over, - * up to (and not including) `uptoStep`: every raw dataset column, plus one - * group key per declared slot of every converter step before it (usually - * one key per step; more than one only when that step's scope mixed - * column types). Passing no `uptoStep` includes every configured step - * (used once a sequence is final and being displayed, e.g. in - * SelectColumnsStep, where every step is already "before" the - * column-selection step that comes after all of them). - * - * `convertersMeta` (registry name -> component, e.g. from getComponents) - * is optional: when supplied, option labels use each step's disambiguated - * display name (see buildStepDisplayNames) instead of its raw registry - * name. - */ -export function buildColumnKeysAndTypes({ - datasetTypes, - preprocessing, - uptoStep, - convertersMeta = {}, -}) { - const steps = preprocessing || []; - const limit = uptoStep === undefined ? steps.length : uptoStep; - const displayNames = buildStepDisplayNames(steps, convertersMeta); - - const columnTypes = { ...datasetTypes }; - const optionLabels = {}; - const allKeys = Object.keys(datasetTypes || {}); - - for (let index = 0; index < limit; index += 1) { - const step = steps[index]; - const name = displayNames[index]; - stepOutputSlots(step).forEach(({ slot, type, dtype }) => { - const key = groupKey(index, slot); - columnTypes[key] = { type: type || null, dtype: dtype || null }; - optionLabels[key] = slot - ? `${name}: output (${slot})` - : `${name}: output`; - allKeys.push(key); - }); - } - - return { allKeys, columnTypes, optionLabels }; -} - -// SimpleImputer only preserves its input column's type for the -// "most_frequent"/"constant" strategies (no arithmetic performed); "mean"/ -// "median" always produce something numeric, but which exact numeric type -// depends on a statistic computed from real data, so there's nothing to -// resolve ahead of a fit — tool.metadata.output_type ("Float") already -// covers that case correctly, since every allowed_types list in this -// codebase that includes Integer also includes Float (they're always -// interchangeable for compatibility checks). -function simpleImputerPreservesType(toolName, params) { - if (toolName !== "SimpleImputer") return false; - const strategy = params?.strategy; - return strategy === "most_frequent" || strategy === "constant"; -} - -/** - * The output slot(s) to record for a converter step once its scope is - * known — always an array of {slot, type, dtype}, even when there's just - * one (slot: null, meaning "the whole group", exactly like before slots - * existed). - * - * Most converters' declared output_type/output_dtype (metadata computed - * backend-side from a bare, never-fitted instance) is the best available - * answer regardless of scope, and stays a single slot. But a converter - * that only keeps or drops whole columns unchanged (backend - * PRESERVES_INPUT_TYPE, e.g. feature selection, VarianceThreshold, or - * SimpleImputer with "most_frequent"/"constant") has an output type - * that's actually already knowable — it's just whatever the chosen - * scope's own column type already is. When that scope mixes distinct - * types, this splits into one slot per distinct type instead of falling - * back to a single guess, matching exactly what SessionPreprocessor. - * _classify_by_type computes for real once the session is created. - */ -export function resolveDeclaredOutputSlots({ - tool, - params, - scope, - datasetTypes, - preprocessing, -}) { - const fallback = [ - { - slot: null, - type: tool?.metadata?.output_type || null, - dtype: tool?.metadata?.output_dtype || null, - }, - ]; - - const preservesInputType = - Boolean(tool?.metadata?.preserves_input_type) || - simpleImputerPreservesType(tool?.name, params); - - if (!preservesInputType || !scope?.length) return fallback; - - const { columnTypes } = buildColumnKeysAndTypes({ - datasetTypes, - preprocessing, - }); - const scopeEntries = scope.map((ref) => columnTypes[refToKey(ref)]); - if (scopeEntries.some((entry) => !entry?.type)) return fallback; - - const distinctTypes = [...new Set(scopeEntries.map((entry) => entry.type))]; - - const dtypeFor = (type) => { - const dtypes = [ - ...new Set( - scopeEntries - .filter((entry) => entry.type === type) - .map((entry) => entry.dtype) - .filter(Boolean), - ), - ]; - return dtypes.length === 1 ? dtypes[0] : null; - }; - - if (distinctTypes.length === 1) { - return [ - { slot: null, type: distinctTypes[0], dtype: dtypeFor(distinctTypes[0]) }, - ]; - } - - return distinctTypes.map((type) => ({ - slot: type, - type, - dtype: dtypeFor(type), - })); -} - /** * Every RAW dataset column name needed to compute a list of ColumnRef, * walking group refs back to their step's own scope recursively (a group * ref's step may itself reference an earlier group, chained arbitrarily - * deep). This is what a caller must actually supply values for — e.g. + * deep). This is what a caller must actually supply values for, e.g. * manual prediction, where the backend only ever accepts real dataset * columns as input (see BaseTask.process_manual_input), never a * converter's resolved output name like "pca_1": it runs the raw values diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js index 43aea6c70..ec3f3077c 100644 --- a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -5,10 +5,11 @@ import { slotFromGroupKey, refToKey, keyToRef, - buildColumnKeysAndTypes, buildStepDisplayNames, - resolveDeclaredOutputSlots, rawColumnsNeededFor, + itemToRef, + stateToOptions, + labelForRef, } from "./sessionColumnRefs"; describe("sessionColumnRefs", () => { @@ -40,247 +41,6 @@ describe("sessionColumnRefs", () => { expect(slotFromGroupKey(groupKey(2))).toBeNull(); }); - it("builds keys/types/labels for raw columns plus every prior step", () => { - const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ - datasetTypes: { age: { type: "Integer" }, text: { type: "Text" } }, - preprocessing: [ - { - converter: "BagOfWordsConverter", - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], - }, - { - converter: "Binarizer", - outputSlots: [{ slot: null, type: "Integer", dtype: null }], - }, - ], - uptoStep: 1, - }); - - expect(allKeys).toEqual(["age", "text", "__group__0"]); - expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); - expect(optionLabels.__group__0).toBe("BagOfWordsConverter: output"); - // step 1 (Binarizer) is excluded: uptoStep=1 only includes steps before it - expect(allKeys).not.toContain(groupKey(1)); - }); - - it("includes every step when uptoStep is omitted", () => { - const { allKeys } = buildColumnKeysAndTypes({ - datasetTypes: { age: { type: "Integer" } }, - preprocessing: [ - { - converter: "BagOfWordsConverter", - outputSlots: [{ slot: null, type: "Integer" }], - }, - { - converter: "Binarizer", - outputSlots: [{ slot: null, type: "Integer" }], - }, - ], - }); - - expect(allKeys).toEqual(["age", "__group__0", "__group__1"]); - }); - - it("offers one key per declared slot for a step with a heterogeneous scope", () => { - const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ - datasetTypes: {}, - preprocessing: [ - { - converter: "SimpleImputer", - outputSlots: [ - { slot: "Integer", type: "Integer", dtype: "int64" }, - { slot: "Categorical", type: "Categorical", dtype: null }, - ], - }, - ], - }); - - const integerKey = groupKey(0, "Integer"); - const categoricalKey = groupKey(0, "Categorical"); - expect(allKeys).toEqual([integerKey, categoricalKey]); - expect(columnTypes[integerKey]).toEqual({ - type: "Integer", - dtype: "int64", - }); - expect(columnTypes[categoricalKey]).toEqual({ - type: "Categorical", - dtype: null, - }); - expect(optionLabels[integerKey]).toBe("SimpleImputer: output (Integer)"); - expect(optionLabels[categoricalKey]).toBe( - "SimpleImputer: output (Categorical)", - ); - }); - - it("disambiguates option labels for two steps of the same converter type", () => { - const { optionLabels } = buildColumnKeysAndTypes({ - datasetTypes: {}, - preprocessing: [ - { - converter: "SimpleImputer", - outputSlots: [{ slot: null, type: "Float" }], - }, - { - converter: "SimpleImputer", - outputSlots: [{ slot: null, type: "Float" }], - }, - ], - convertersMeta: { - SimpleImputer: { display_name: "Simple Imputer" }, - }, - }); - - expect(optionLabels[groupKey(0)]).toBe("Simple Imputer: output"); - expect(optionLabels[groupKey(1)]).toBe("Simple Imputer (2): output"); - }); - - it("falls back to a single unslotted key for a step predating outputSlots", () => { - const { allKeys, columnTypes } = buildColumnKeysAndTypes({ - datasetTypes: {}, - preprocessing: [ - { converter: "Binarizer", outputType: "Integer", outputDtype: "int64" }, - ], - }); - - expect(allKeys).toEqual(["__group__0"]); - expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); - }); - - describe("resolveDeclaredOutputSlots", () => { - const datasetTypes = { - age: { type: "Integer", dtype: "int64" }, - score: { type: "Float", dtype: "float64" }, - name: { type: "Text", dtype: "string" }, - }; - - it("falls back to the converter's declared metadata by default", () => { - const tool = { - name: "Binarizer", - metadata: { output_type: "Integer", output_dtype: "int64" }, - }; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [{ kind: "raw", name: "age" }], - datasetTypes, - preprocessing: [], - }); - expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); - }); - - it("uses the scope's real type when the converter preserves input type", () => { - const tool = { - name: "SelectKBest", - metadata: { output_type: "Float", preserves_input_type: true }, - }; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [{ kind: "raw", name: "age" }], - datasetTypes, - preprocessing: [], - }); - expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); - }); - - it("splits into one slot per distinct type when a preserving converter's scope mixes types", () => { - const tool = { - name: "VarianceThreshold", - metadata: { output_type: "Float", preserves_input_type: true }, - }; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [ - { kind: "raw", name: "age" }, - { kind: "raw", name: "name" }, - ], - datasetTypes, - preprocessing: [], - }); - expect(result).toEqual( - expect.arrayContaining([ - { slot: "Integer", type: "Integer", dtype: "int64" }, - { slot: "Text", type: "Text", dtype: "string" }, - ]), - ); - expect(result).toHaveLength(2); - }); - - it("treats SimpleImputer as type-preserving only for most_frequent/constant", () => { - const tool = { - name: "SimpleImputer", - metadata: { output_type: "Float", output_dtype: null }, - }; - const scope = [{ kind: "raw", name: "name" }]; - - const mostFrequent = resolveDeclaredOutputSlots({ - tool, - params: { strategy: "most_frequent" }, - scope, - datasetTypes, - preprocessing: [], - }); - expect(mostFrequent).toEqual([ - { slot: null, type: "Text", dtype: "string" }, - ]); - - const mean = resolveDeclaredOutputSlots({ - tool, - params: { strategy: "mean" }, - scope, - datasetTypes, - preprocessing: [], - }); - expect(mean).toEqual([{ slot: null, type: "Float", dtype: null }]); - }); - - it("resolves a preserving converter's scope through a chained group ref", () => { - const tool = { - name: "SelectKBest", - metadata: { output_type: "Float", preserves_input_type: true }, - }; - const preprocessing = [ - { - converter: "BagOfWordsConverter", - outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], - }, - ]; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [{ kind: "group", step: 0 }], - datasetTypes, - preprocessing, - }); - expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); - }); - - it("resolves a preserving converter's scope through one specific slot of a chained group ref", () => { - const tool = { - name: "SelectKBest", - metadata: { output_type: "Float", preserves_input_type: true }, - }; - const preprocessing = [ - { - converter: "SimpleImputer", - outputSlots: [ - { slot: "Integer", type: "Integer", dtype: "int64" }, - { slot: "Categorical", type: "Categorical", dtype: null }, - ], - }, - ]; - const result = resolveDeclaredOutputSlots({ - tool, - params: {}, - scope: [{ kind: "group", step: 0, slot: "Integer" }], - datasetTypes, - preprocessing, - }); - expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); - }); - }); - describe("buildStepDisplayNames", () => { it("leaves a converter type's name unchanged when it appears only once", () => { const steps = [ @@ -395,4 +155,125 @@ describe("sessionColumnRefs", () => { expect(rawColumnsNeededFor(refs, [])).toEqual(["age"]); }); }); + + describe("named group refs", () => { + it("round-trips a named group ColumnRef through a key", () => { + const ref = { kind: "group", step: 1, name: "date_month" }; + const key = refToKey(ref); + expect(isGroupKey(key)).toBe(true); + expect(stepFromGroupKey(key)).toBe(1); + expect(slotFromGroupKey(key)).toBeNull(); + expect(keyToRef(key)).toEqual(ref); + }); + + it("keeps a column name containing the separators intact", () => { + const ref = { kind: "group", step: 0, name: "a__slot__b" }; + expect(keyToRef(refToKey(ref))).toEqual(ref); + }); + }); + + describe("itemToRef", () => { + it("refs an original column by name", () => { + const item = { kind: "column", name: "age", origin: null }; + expect(itemToRef(item)).toEqual({ kind: "raw", name: "age" }); + }); + + it("refs a generated column by its step and name", () => { + const item = { kind: "column", name: "date_month", origin: 2 }; + expect(itemToRef(item)).toEqual({ + kind: "group", + step: 2, + name: "date_month", + }); + }); + + it("refs a lone block as its step's whole group", () => { + const item = { kind: "block", step: 0, slot: null }; + expect(itemToRef(item)).toEqual({ kind: "group", step: 0 }); + }); + + it("refs one of several blocks by its slot", () => { + const item = { kind: "block", step: 0, slot: "Float" }; + expect(itemToRef(item)).toEqual({ + kind: "group", + step: 0, + slot: "Float", + }); + }); + }); + + describe("stateToOptions", () => { + const state = [ + { + kind: "column", + name: "age", + type: "Integer", + dtype: "int64", + origin: null, + }, + { + kind: "column", + name: "date_month", + type: "Integer", + dtype: "int64", + origin: 1, + }, + { + kind: "block", + step: 0, + slot: null, + label: "output", + type: "Float", + dtype: "float64", + count: 2, + }, + { + kind: "block", + step: 2, + slot: null, + label: "ohe_*", + type: "Integer", + dtype: "int64", + count: null, + }, + ]; + const stepNames = ["PCA", "Date Features", "One Hot Encoder"]; + + it("keys every item as the ref it stands for, typed", () => { + const { allKeys, columnTypes } = stateToOptions(state, stepNames); + + expect(allKeys.map(keyToRef)).toEqual(state.map(itemToRef)); + expect(columnTypes.age).toEqual({ type: "Integer", dtype: "int64" }); + expect(columnTypes[allKeys[2]]).toEqual({ + type: "Float", + dtype: "float64", + }); + }); + + it("labels blocks with their step and size, N when unknown", () => { + const { allKeys, optionLabels } = stateToOptions(state, stepNames); + + expect(optionLabels[allKeys[0]]).toBeUndefined(); + expect(optionLabels[allKeys[1]]).toBe("date_month"); + expect(optionLabels[allKeys[2]]).toBe("PCA: output (2)"); + expect(optionLabels[allKeys[3]]).toBe("One Hot Encoder: ohe_* (N)"); + }); + }); + + describe("labelForRef", () => { + const stepNames = ["PCA", "Date Features"]; + + it("labels a ref without needing the estimated structure", () => { + expect(labelForRef({ kind: "raw", name: "age" }, stepNames)).toBe("age"); + expect( + labelForRef({ kind: "group", step: 1, name: "date_month" }, stepNames), + ).toBe("date_month"); + expect(labelForRef({ kind: "group", step: 0 }, stepNames)).toBe( + "PCA: output", + ); + expect( + labelForRef({ kind: "group", step: 0, slot: "Float" }, stepNames), + ).toBe("PCA: output (Float)"); + }); + }); }); diff --git a/DashAI/front/src/components/models/modelSession/structureMessages.js b/DashAI/front/src/components/models/modelSession/structureMessages.js new file mode 100644 index 000000000..d25fcf795 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/structureMessages.js @@ -0,0 +1,27 @@ +/** + * Translate a structure message from the backend's infer_structure + * (`{code, params}`) into user-facing text, from `models:structure.`. + * + * A few codes need their params shaped first: a cardinality error names + * whichever bound was broken, and a list of accepted types is joined. + * + * @param {Function} t the i18next translation function + * @param {{code: string, params?: object}} message + * @returns {string} + */ +export function formatStructureMessage(t, message) { + if (!message) return ""; + const params = { ...(message.params || {}) }; + let key = message.code; + if (key === "cardinality") { + const tooFew = params.min != null && params.selected < params.min; + key = tooFew ? "cardinalityMin" : "cardinalityMax"; + } + if (Array.isArray(params.allowed)) { + params.allowed = params.allowed.join(", "); + } + return t(`models:structure.${key}`, { + ...params, + interpolation: { escapeValue: false }, + }); +} diff --git a/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.js b/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.js new file mode 100644 index 000000000..1811fe832 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.js @@ -0,0 +1,78 @@ +import { useEffect, useRef, useState } from "react"; +import { getPreprocessingStructure } from "../../../api/modelSession"; + +const DEBOUNCE_MS = 300; + +/** + * The estimated dataset structure along a session's preprocessing chain, + * from the backend's infer_structure: the state after every step, their + * errors and warnings, and the final state. + * + * Refetched on every change to the inputs, debounced so quick edits send + * one request. Only the response to the latest inputs is kept: a slower + * response to an earlier edit never overwrites a newer one. + * + * @param {object} args + * @param {number} args.datasetId the session's dataset; nothing is fetched + * without one + * @param {string[]} args.candidates original columns fed into the chain + * @param {string[]} args.outputColumns the target columns + * @param {Array} args.steps the chain's `{converter, params, scope}` + * @returns {{structure: object|null, loading: boolean, error: Error|null}} + */ +export default function usePreprocessingStructure({ + datasetId, + candidates, + outputColumns, + steps, +}) { + const [structure, setStructure] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const latestRequest = useRef(0); + + // A stable value to depend on: the caller may rebuild these arrays on + // every render, and a step may carry extra UI-only fields. + const payloadKey = JSON.stringify({ + datasetId, + candidates: candidates || [], + outputColumns: outputColumns || [], + steps: (steps || []).map(({ converter, params, scope }) => ({ + converter, + params, + scope, + })), + }); + + useEffect(() => { + const payload = JSON.parse(payloadKey); + const requestId = latestRequest.current + 1; + latestRequest.current = requestId; + if (!payload.datasetId) { + setStructure(null); + setLoading(false); + return undefined; + } + setLoading(true); + const timer = setTimeout(async () => { + try { + const result = await getPreprocessingStructure(payload); + if (requestId === latestRequest.current) { + setStructure(result); + setError(null); + } + } catch (requestError) { + if (requestId === latestRequest.current) { + setError(requestError); + } + } finally { + if (requestId === latestRequest.current) { + setLoading(false); + } + } + }, DEBOUNCE_MS); + return () => clearTimeout(timer); + }, [payloadKey]); + + return { structure, loading, error }; +} diff --git a/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.test.js b/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.test.js new file mode 100644 index 000000000..4708233c6 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/usePreprocessingStructure.test.js @@ -0,0 +1,80 @@ +import { act, renderHook } from "@testing-library/react"; +import usePreprocessingStructure from "./usePreprocessingStructure"; +import { getPreprocessingStructure } from "../../../api/modelSession"; + +jest.mock("../../../api/modelSession", () => ({ + getPreprocessingStructure: jest.fn(), +})); + +const deferred = () => { + let resolve; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +}; + +const args = (steps) => ({ + datasetId: 1, + candidates: ["age"], + outputColumns: ["label"], + steps, +}); + +describe("usePreprocessingStructure", () => { + beforeEach(() => { + jest.useFakeTimers(); + getPreprocessingStructure.mockReset(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it("sends one request for quick successive edits", async () => { + getPreprocessingStructure.mockResolvedValue({ valid: true }); + const { rerender } = renderHook( + (props) => usePreprocessingStructure(props), + { + initialProps: args([]), + }, + ); + + rerender(args([{ converter: "PCA", params: {}, scope: [] }])); + await act(async () => { + jest.advanceTimersByTime(300); + }); + + expect(getPreprocessingStructure).toHaveBeenCalledTimes(1); + expect(getPreprocessingStructure.mock.calls[0][0].steps).toHaveLength(1); + }); + + it("never lets a slower, older response overwrite a newer one", async () => { + const older = deferred(); + const newer = deferred(); + getPreprocessingStructure + .mockReturnValueOnce(older.promise) + .mockReturnValueOnce(newer.promise); + const { result, rerender } = renderHook( + (props) => usePreprocessingStructure(props), + { initialProps: args([]) }, + ); + + await act(async () => { + jest.advanceTimersByTime(300); + }); + rerender(args([{ converter: "PCA", params: {}, scope: [] }])); + await act(async () => { + jest.advanceTimersByTime(300); + }); + await act(async () => { + newer.resolve({ id: "newer" }); + }); + await act(async () => { + older.resolve({ id: "older" }); + }); + + expect(result.current.structure).toEqual({ id: "newer" }); + expect(result.current.loading).toBe(false); + }); +}); diff --git a/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx b/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx index 5e37f2fbb..cf872bdd4 100644 --- a/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx +++ b/DashAI/front/src/components/notebooks/converterCreation/ParameterStepConverter.jsx @@ -13,6 +13,10 @@ export default function ParameterStepConverter({ handleSaveConverter, setStep, hideButtons = false, + // The notebook fits a converter on the whole dataset, so one that learns + // from the data can leak test information. Session preprocessing fits on + // the training split only, so it turns this warning off. + warnAboutLeakage = true, }) { const tourContext = useTourContext(); const { t } = useTranslation(["common", "datasets"]); @@ -72,7 +76,8 @@ export default function ParameterStepConverter({ nColumnsSelected > 0 && currentNComponents > nColumnsSelected; - const showLeakageWarning = tool?.metadata?.learns_from_data === true; + const showLeakageWarning = + warnAboutLeakage && tool?.metadata?.learns_from_data === true; return ( ; +} + +export interface IStepStructure { + status: "ok" | "error" | "blocked"; + state: IStateItem[]; + added: IStateItem[]; + error: IStructureMessage | null; + warnings: IStructureMessage[]; +} + +export interface IStructureResult { + initial: IStateItem[]; + steps: IStepStructure[]; + final: IStateItem[]; + valid: boolean; +} + export type IPreprocessingStatus = "ready" | "pending" | "failed"; export interface IModelSession { diff --git a/DashAI/front/src/utils/i18n/locales/de/models.json b/DashAI/front/src/utils/i18n/locales/de/models.json index 386f98882..86782b126 100644 --- a/DashAI/front/src/utils/i18n/locales/de/models.json +++ b/DashAI/front/src/utils/i18n/locales/de/models.json @@ -129,6 +129,9 @@ "innerLoopConfiguration": "Konfigurieren Sie die Datenaufteilung für die innere Schleife der verschachtelten Kreuzvalidierung", "innerSplitter": "Splitter für die innere Schleife", "inputColumns": "Eingabespalten", + "baseColumnsTitle": "Ausgabe- und Kandidatenspalten", + "baseColumnsDescription": "Wählen Sie die Ausgabespalte (die das Modell vorherzusagen lernt) und die Kandidatenspalten, die in das Modell einfließen können. Die Vorverarbeitung arbeitet nur mit den Kandidatenspalten, die Ausgabe wird nie transformiert.", + "candidateColumns": "Kandidatenspalten", "less": "Kleiner (Modell 1 < Modell 2)", "lines": "Linien", "liveMetrics": "Live-Metriken", @@ -346,5 +349,25 @@ "skippedUndownloadedRuns": "Durchläufe mit nicht heruntergeladenem Modell wurden übersprungen. Lade das Modell zuerst herunter.", "skippedUnavailableRuns": "Durchläufe, deren Modell noch einen Download oder Anmeldedaten benötigt, wurden übersprungen. Behebe dies zuerst.", "tabAvailableAfterFinish": "Verfügbar, sobald der Durchlauf das Training abgeschlossen hat." + }, + "structure": { + "currentState": "Datensatz nach der Vorverarbeitung", + "columnsCount": "{{count}} Spalten", + "unknownCount": "N Spalten, erst nach dem Training bekannt", + "blocked": "Korrigieren Sie den vorherigen Schritt, um diesen zu prüfen.", + "loadError": "Das Ergebnis der Vorverarbeitung konnte nicht geschätzt werden.", + "missing_ref": "\"{{ref}}\" existiert an dieser Stelle der Kette nicht: Ein früherer Schritt hat sie möglicherweise verbraucht.", + "target_in_scope": "\"{{column}}\" ist die Ausgabespalte, und die Ausgabe wird nie transformiert.", + "type_not_allowed": "\"{{column}}\" ist {{type}}, was dieser Converter nicht akzeptiert (akzeptiert: {{allowed}}).", + "cardinalityMin": "Dieser Converter benötigt mindestens {{min}} Spalten, ausgewählt sind {{selected}}.", + "cardinalityMax": "Dieser Converter akzeptiert höchstens {{max}} Spalten, ausgewählt sind {{selected}}.", + "rows_not_supported": "Converter, die die Zeilenanzahl ändern, sind in Sitzungen noch nicht verfügbar.", + "n_components_exceeds": "Es werden {{n_components}} Komponenten angefordert, aber nur {{columns}} Spalten sind ausgewählt.", + "unknown_converter": "Der Converter \"{{converter}}\" ist nicht verfügbar.", + "invalid_params": "Ungültige Parameter: {{detail}}", + "converter_rejected": "Der Converter lehnt diese Konfiguration ab: {{detail}}", + "inference_fallback": "Das Ergebnis dieses Converters lässt sich nicht vorhersehen; seine Ausgabe wird als ein Block unbekannter Größe angezeigt.", + "count_unknown_bound": "Es werden {{n_components}} Komponenten angefordert; ob genug Spalten vorhanden sind, zeigt sich erst beim Training.", + "cast_may_skip": "Eine Spalte, die nicht in {{new_type}} umgewandelt werden kann, behält ihren ursprünglichen Typ." } } diff --git a/DashAI/front/src/utils/i18n/locales/en/models.json b/DashAI/front/src/utils/i18n/locales/en/models.json index a3264b4c6..e7047744c 100644 --- a/DashAI/front/src/utils/i18n/locales/en/models.json +++ b/DashAI/front/src/utils/i18n/locales/en/models.json @@ -188,6 +188,9 @@ "hyperparameterOptimizerConfiguration": "Hyperparameter Optimizer Configuration", "hyperparameters": "Hyperparameters", "inputColumns": "Input columns", + "baseColumnsTitle": "Output and candidate columns", + "baseColumnsDescription": "Choose the output column (the one the model learns to predict) and the candidate columns that could feed the model. Preprocessing only works on the candidate columns, and the output is never transformed.", + "candidateColumns": "Candidate columns", "liveMetrics": "Live Metrics", "localExplainer": "Local Explainer", "localExplainers": "Local Explainers", @@ -346,5 +349,25 @@ "skippedUndownloadedRuns": "Skipped runs whose model is not downloaded. Download the model first.", "skippedUnavailableRuns": "Skipped runs whose model still needs a download or credentials. Resolve those first.", "tabAvailableAfterFinish": "Available once the run finishes training." + }, + "structure": { + "currentState": "Dataset after preprocessing", + "columnsCount": "{{count}} columns", + "unknownCount": "N columns, known once trained", + "blocked": "Fix the previous step to check this one.", + "loadError": "Could not estimate the preprocessing result.", + "missing_ref": "\"{{ref}}\" does not exist at this point of the chain: an earlier step may have consumed it.", + "target_in_scope": "\"{{column}}\" is the output column, and the output is never transformed.", + "type_not_allowed": "\"{{column}}\" is {{type}}, which this converter does not accept (accepts: {{allowed}}).", + "cardinalityMin": "This converter needs at least {{min}} columns, but {{selected}} are selected.", + "cardinalityMax": "This converter accepts at most {{max}} columns, but {{selected}} are selected.", + "rows_not_supported": "Converters that change the number of rows are not available in sessions yet.", + "n_components_exceeds": "It asks for {{n_components}} components, but only {{columns}} columns are selected.", + "unknown_converter": "The converter \"{{converter}}\" is not available.", + "invalid_params": "Invalid parameters: {{detail}}", + "converter_rejected": "The converter rejects this configuration: {{detail}}", + "inference_fallback": "The result of this converter cannot be anticipated; its output is shown as one block of unknown size.", + "count_unknown_bound": "Asks for {{n_components}} components; whether there are enough columns is only known once trained.", + "cast_may_skip": "A column that cannot be converted to {{new_type}} keeps its original type." } } diff --git a/DashAI/front/src/utils/i18n/locales/es/models.json b/DashAI/front/src/utils/i18n/locales/es/models.json index 3136aee1e..a27149260 100644 --- a/DashAI/front/src/utils/i18n/locales/es/models.json +++ b/DashAI/front/src/utils/i18n/locales/es/models.json @@ -192,6 +192,9 @@ "hyperparameterOptimizerConfiguration": "Configuración del Optimizador de Hiperparámetros", "hyperparameters": "Hiperparámetros", "inputColumns": "Columnas de entrada", + "baseColumnsTitle": "Columna de salida y candidatas", + "baseColumnsDescription": "Elige la columna de salida (la que el modelo aprende a predecir) y las columnas candidatas que podrían llegar al modelo. El preprocesamiento solo trabaja sobre las candidatas y la salida nunca se transforma.", + "candidateColumns": "Columnas candidatas", "liveMetrics": "Métricas en Vivo", "localExplainer": "Explicador Local", "localExplainers": "Explicadores Locales", @@ -355,5 +358,25 @@ "skippedUndownloadedRuns": "Se omitieron las ejecuciones cuyo modelo no está descargado. Descarga el modelo primero.", "skippedUnavailableRuns": "Se omitieron las ejecuciones cuyo modelo aún requiere descarga o credenciales. Resuélvelo primero.", "tabAvailableAfterFinish": "Estará disponible cuando el run termine de entrenar." + }, + "structure": { + "currentState": "Dataset después del preprocesamiento", + "columnsCount": "{{count}} columnas", + "unknownCount": "N columnas, se sabe al entrenar", + "blocked": "Corrige el paso anterior para revisar este.", + "loadError": "No se pudo estimar el resultado del preprocesamiento.", + "missing_ref": "\"{{ref}}\" no existe en este punto de la cadena: un paso anterior pudo haberla consumido.", + "target_in_scope": "\"{{column}}\" es la columna de salida, y la salida nunca se transforma.", + "type_not_allowed": "\"{{column}}\" es {{type}}, que este converter no acepta (acepta: {{allowed}}).", + "cardinalityMin": "Este converter necesita al menos {{min}} columnas, pero hay {{selected}} seleccionadas.", + "cardinalityMax": "Este converter acepta como máximo {{max}} columnas, pero hay {{selected}} seleccionadas.", + "rows_not_supported": "Los converters que cambian la cantidad de filas todavía no están disponibles en sesiones.", + "n_components_exceeds": "Pide {{n_components}} componentes, pero solo hay {{columns}} columnas seleccionadas.", + "unknown_converter": "El converter \"{{converter}}\" no está disponible.", + "invalid_params": "Parámetros inválidos: {{detail}}", + "converter_rejected": "El converter rechaza esta configuración: {{detail}}", + "inference_fallback": "No se puede anticipar el resultado de este converter; su salida se muestra como un bloque de tamaño desconocido.", + "count_unknown_bound": "Pide {{n_components}} componentes; si alcanzan las columnas se sabe recién al entrenar.", + "cast_may_skip": "Una columna que no se pueda convertir a {{new_type}} conserva su tipo original." } } diff --git a/DashAI/front/src/utils/i18n/locales/pt/models.json b/DashAI/front/src/utils/i18n/locales/pt/models.json index 39c218dbb..3d0a74311 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/models.json +++ b/DashAI/front/src/utils/i18n/locales/pt/models.json @@ -131,6 +131,9 @@ "innerLoopConfiguration": "Configure a partição de dados para o loop interno de validação cruzada aninhada", "innerSplitter": "Divisor para o loop interno", "inputColumns": "Colunas de entrada", + "baseColumnsTitle": "Coluna de saída e colunas candidatas", + "baseColumnsDescription": "Escolha a coluna de saída (a que o modelo aprende a prever) e as colunas candidatas que podem alimentar o modelo. O pré-processamento só atua sobre as candidatas e a saída nunca é transformada.", + "candidateColumns": "Colunas candidatas", "less": "Menor (Modelo 1 < Modelo 2)", "lines": "Linhas", "liveMetrics": "Métricas ao Vivo", @@ -355,5 +358,25 @@ "skippedUndownloadedRuns": "Execuções cujo modelo não está baixado foram ignoradas. Baixe o modelo primeiro.", "skippedUnavailableRuns": "Execuções cujo modelo ainda precisa de download ou credenciais foram ignoradas. Resolva isso primeiro.", "tabAvailableAfterFinish": "Estará disponível quando a execução terminar o treinamento." + }, + "structure": { + "currentState": "Dataset após o pré-processamento", + "columnsCount": "{{count}} colunas", + "unknownCount": "N colunas, conhecido após o treinamento", + "blocked": "Corrija a etapa anterior para verificar esta.", + "loadError": "Não foi possível estimar o resultado do pré-processamento.", + "missing_ref": "\"{{ref}}\" não existe neste ponto da cadeia: uma etapa anterior pode tê-la consumido.", + "target_in_scope": "\"{{column}}\" é a coluna de saída, e a saída nunca é transformada.", + "type_not_allowed": "\"{{column}}\" é {{type}}, que este conversor não aceita (aceita: {{allowed}}).", + "cardinalityMin": "Este conversor precisa de pelo menos {{min}} colunas, mas há {{selected}} selecionadas.", + "cardinalityMax": "Este conversor aceita no máximo {{max}} colunas, mas há {{selected}} selecionadas.", + "rows_not_supported": "Conversores que alteram o número de linhas ainda não estão disponíveis em sessões.", + "n_components_exceeds": "Pede {{n_components}} componentes, mas há apenas {{columns}} colunas selecionadas.", + "unknown_converter": "O conversor \"{{converter}}\" não está disponível.", + "invalid_params": "Parâmetros inválidos: {{detail}}", + "converter_rejected": "O conversor rejeita esta configuração: {{detail}}", + "inference_fallback": "Não é possível antecipar o resultado deste conversor; sua saída é mostrada como um bloco de tamanho desconhecido.", + "count_unknown_bound": "Pede {{n_components}} componentes; se há colunas suficientes só se sabe após o treinamento.", + "cast_may_skip": "Uma coluna que não puder ser convertida para {{new_type}} mantém seu tipo original." } } diff --git a/DashAI/front/src/utils/i18n/locales/zh/models.json b/DashAI/front/src/utils/i18n/locales/zh/models.json index 249cb1375..656d12f67 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/models.json +++ b/DashAI/front/src/utils/i18n/locales/zh/models.json @@ -129,6 +129,9 @@ "innerLoopConfiguration": "配置嵌套交叉验证内层循环的数据划分", "innerSplitter": "内层循环划分器", "inputColumns": "输入列", + "baseColumnsTitle": "输出列与候选列", + "baseColumnsDescription": "选择输出列(模型学习预测的列)以及可能输入模型的候选列。预处理只作用于候选列,输出列永远不会被转换。", + "candidateColumns": "候选列", "less": "更小(模型 1 < 模型 2)", "lines": "折线", "liveMetrics": "实时指标", @@ -345,5 +348,25 @@ "skippedUndownloadedRuns": "已跳过模型未下载的运行。请先下载模型。", "skippedUnavailableRuns": "已跳过模型仍需下载或验证的运行。请先解决这些问题。", "tabAvailableAfterFinish": "运行训练完成后即可使用。" + }, + "structure": { + "currentState": "预处理后的数据集", + "columnsCount": "{{count}} 列", + "unknownCount": "N 列,训练后才能确定", + "blocked": "请先修正上一步,再检查这一步。", + "loadError": "无法估计预处理结果。", + "missing_ref": "“{{ref}}”在链的这一位置不存在:之前的某一步可能已将其消耗。", + "target_in_scope": "“{{column}}”是输出列,输出永远不会被转换。", + "type_not_allowed": "“{{column}}”的类型是 {{type}},此转换器不接受(接受:{{allowed}})。", + "cardinalityMin": "此转换器至少需要 {{min}} 列,但只选择了 {{selected}} 列。", + "cardinalityMax": "此转换器最多接受 {{max}} 列,但选择了 {{selected}} 列。", + "rows_not_supported": "会改变行数的转换器暂时无法在会话中使用。", + "n_components_exceeds": "要求 {{n_components}} 个成分,但只选择了 {{columns}} 列。", + "unknown_converter": "转换器“{{converter}}”不可用。", + "invalid_params": "参数无效:{{detail}}", + "converter_rejected": "转换器拒绝此配置:{{detail}}", + "inference_fallback": "无法预先确定此转换器的结果;其输出显示为一个大小未知的块。", + "count_unknown_bound": "要求 {{n_components}} 个成分;列数是否足够要到训练时才能确定。", + "cast_may_skip": "无法转换为 {{new_type}} 的列会保留其原始类型。" } } diff --git a/tests/back/api/test_explainer_job_preprocessing.py b/tests/back/api/test_explainer_job_preprocessing.py index 6f7ce836b..28821a8ca 100644 --- a/tests/back/api/test_explainer_job_preprocessing.py +++ b/tests/back/api/test_explainer_job_preprocessing.py @@ -59,7 +59,7 @@ def _train_session_with_scaler(client: TestClient, dataset_id: int): "scope": [{"kind": "raw", "name": "SepalLengthCm"}], }, ], - "input_column_refs": [{"kind": "group", "step": 0}], + "input_column_refs": [{"kind": "raw", "name": "SepalLengthCm"}], }, ) assert session_response.status_code == 201, session_response.text diff --git a/tests/back/api/test_model_session_api.py b/tests/back/api/test_model_session_api.py index d7f018847..d9b3fdeae 100644 --- a/tests/back/api/test_model_session_api.py +++ b/tests/back/api/test_model_session_api.py @@ -127,7 +127,7 @@ def test_create_model_session_with_preprocessing_requires_input_column_refs( assert response.status_code == 422, response.text -def test_validate_columns_accepts_a_group_ref_matching_the_task_type( +def test_validate_columns_rejects_group_refs_without_preprocessing( client: TestClient, dataset_1: Dataset ) -> None: response = client.post( @@ -135,38 +135,149 @@ def test_validate_columns_accepts_a_group_ref_matching_the_task_type( json={ "task_name": "TabularClassificationTask", "dataset_id": dataset_1.id, - "inputs_columns": ["SepalWidthCm"], + "inputs_columns": [], "outputs_columns": ["Species"], - "input_refs": [ - {"kind": "raw", "name": "SepalWidthCm"}, - {"kind": "group", "step": 0}, - ], - "converter_output_types": {"0": "Integer"}, + "input_refs": [{"kind": "group", "step": 0}], }, ) assert response.status_code == 200, response.text - assert response.json()["dataset_status"] == "valid" + assert response.json()["dataset_status"] == "invalid" + + +def test_bulk_delete_model_sessions(client: TestClient, dataset_1: Dataset) -> None: + created_ids = [] + for name in ["bulk_delete_session_1", "bulk_delete_session_2"]: + response = client.post( + "/api/v1/model-session/", + json={**SESSION_PARAMS, "dataset_id": dataset_1.id, "name": name}, + ) + assert response.status_code == 201, response.text + created_ids.append(response.json()["id"]) + + # A non-existent id mixed in should be skipped rather than failing the batch. + response = client.request( + "DELETE", + "/api/v1/model-session/", + json={"ids": [*created_ids, 999999]}, + ) + assert response.status_code == 204, response.text + + for session_id in created_ids: + response = client.get(f"/api/v1/model-session/{session_id}") + assert response.status_code == 404, response.text -def test_validate_columns_rejects_a_group_ref_with_an_incompatible_type( +BINARIZE_SEPAL_LENGTH = { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], +} + + +def test_preprocessing_structure_returns_the_state_after_each_step( client: TestClient, dataset_1: Dataset ) -> None: response = client.post( - "/api/v1/model-session/validation", + "/api/v1/model-session/preprocessing/structure", json={ - "task_name": "TabularClassificationTask", "dataset_id": dataset_1.id, - "inputs_columns": [], - "outputs_columns": ["Species"], - "input_refs": [{"kind": "group", "step": 0}], - "converter_output_types": {"0": "Text"}, + "candidates": ["SepalLengthCm", "SepalWidthCm"], + "output_columns": ["Species"], + "steps": [BINARIZE_SEPAL_LENGTH], }, ) assert response.status_code == 200, response.text - assert response.json()["dataset_status"] == "invalid" + body = response.json() + assert body["valid"] is True + assert [item["name"] for item in body["initial"]] == [ + "SepalLengthCm", + "SepalWidthCm", + ] + (added,) = body["steps"][0]["added"] + assert (added["name"], added["type"], added["origin"]) == ( + "bin_SepalLengthCm", + "Integer", + 0, + ) + + +def test_preprocessing_structure_reports_a_ref_to_a_missing_column( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/preprocessing/structure", + json={ + "dataset_id": dataset_1.id, + "candidates": ["SepalLengthCm"], + "output_columns": ["Species"], + "steps": [ + { + "converter": "StandardScaler", + "params": {}, + "scope": [{"kind": "raw", "name": "PetalWidthCm"}], + } + ], + }, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["valid"] is False + assert body["steps"][0]["error"]["code"] == "missing_ref" + + +def test_create_model_session_rejects_an_invalid_chain( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "invalid-chain-session", + "input_columns": [], + "preprocessing": [ + { + "converter": "StandardScaler", + "params": {}, + "scope": [{"kind": "raw", "name": "Species"}], + } + ], + "input_column_refs": [{"kind": "raw", "name": "SepalWidthCm"}], + }, + ) + assert response.status_code == 422, response.text + structure = response.json()["detail"]["structure"] + assert structure["steps"][0]["error"]["code"] == "target_in_scope" -def test_validate_columns_accepts_a_slotted_group_ref_matching_the_task_type( +def test_create_model_session_rejects_an_input_the_chain_consumed( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "consumed-input-session", + "input_columns": [], + "preprocessing": [ + { + "converter": "PCA", + "params": {"n_components": 1}, + "scope": [ + {"kind": "raw", "name": "SepalLengthCm"}, + {"kind": "raw", "name": "SepalWidthCm"}, + ], + } + ], + "input_column_refs": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ) + assert response.status_code == 422, response.text + assert response.json()["detail"]["error"]["code"] == "missing_ref" + + +def test_validate_columns_with_preprocessing_accepts_a_generated_column( client: TestClient, dataset_1: Dataset ) -> None: response = client.post( @@ -174,17 +285,20 @@ def test_validate_columns_accepts_a_slotted_group_ref_matching_the_task_type( json={ "task_name": "TabularClassificationTask", "dataset_id": dataset_1.id, - "inputs_columns": [], + "inputs_columns": ["SepalWidthCm"], "outputs_columns": ["Species"], - "input_refs": [{"kind": "group", "step": 0, "slot": "Integer"}], - "converter_output_types": {"0:Integer": "Integer", "0:Categorical": "Text"}, + "input_refs": [ + {"kind": "raw", "name": "SepalWidthCm"}, + {"kind": "group", "step": 0, "name": "bin_SepalLengthCm"}, + ], + "preprocessing": [BINARIZE_SEPAL_LENGTH], }, ) assert response.status_code == 200, response.text assert response.json()["dataset_status"] == "valid" -def test_validate_columns_rejects_a_slotted_group_ref_with_an_incompatible_type( +def test_validate_columns_with_preprocessing_rejects_an_input_cast_to_text( client: TestClient, dataset_1: Dataset ) -> None: response = client.post( @@ -192,34 +306,17 @@ def test_validate_columns_rejects_a_slotted_group_ref_with_an_incompatible_type( json={ "task_name": "TabularClassificationTask", "dataset_id": dataset_1.id, - "inputs_columns": [], + "inputs_columns": ["SepalLengthCm"], "outputs_columns": ["Species"], - "input_refs": [{"kind": "group", "step": 0, "slot": "Categorical"}], - "converter_output_types": {"0:Integer": "Integer", "0:Categorical": "Text"}, + "input_refs": [{"kind": "raw", "name": "SepalLengthCm"}], + "preprocessing": [ + { + "converter": "TypeCast", + "params": {"new_type": "Text", "on_error": "raise"}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + } + ], }, ) assert response.status_code == 200, response.text assert response.json()["dataset_status"] == "invalid" - - -def test_bulk_delete_model_sessions(client: TestClient, dataset_1: Dataset) -> None: - created_ids = [] - for name in ["bulk_delete_session_1", "bulk_delete_session_2"]: - response = client.post( - "/api/v1/model-session/", - json={**SESSION_PARAMS, "dataset_id": dataset_1.id, "name": name}, - ) - assert response.status_code == 201, response.text - created_ids.append(response.json()["id"]) - - # A non-existent id mixed in should be skipped rather than failing the batch. - response = client.request( - "DELETE", - "/api/v1/model-session/", - json={"ids": [*created_ids, 999999]}, - ) - assert response.status_code == 204, response.text - - for session_id in created_ids: - response = client.get(f"/api/v1/model-session/{session_id}") - assert response.status_code == 404, response.text diff --git a/tests/back/api/test_run_preprocessing_guard.py b/tests/back/api/test_run_preprocessing_guard.py index 4ea339e22..d3d1e6154 100644 --- a/tests/back/api/test_run_preprocessing_guard.py +++ b/tests/back/api/test_run_preprocessing_guard.py @@ -103,14 +103,15 @@ def test_run_creation_surfaces_the_preprocessing_error_when_failed( ), "preprocessing": [ { - # Binarizer only accepts numeric columns; scoping it on the - # categorical target makes PreprocessingJob's fit fail. - "converter": "Binarizer", - "params": {"threshold": 3.0}, - "scope": [{"kind": "raw", "name": "Species"}], + # A chain the structure estimate accepts but the data + # breaks: SepalLengthCm holds decimals (e.g. 5.1), which + # cannot be cast to Integer, so PreprocessingJob fails. + "converter": "TypeCast", + "params": {"new_type": "Integer", "on_error": "raise"}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], }, ], - "input_column_refs": [{"kind": "group", "step": 0}], + "input_column_refs": [{"kind": "raw", "name": "SepalLengthCm"}], }, ) assert session_response.status_code == 201, session_response.text diff --git a/tests/back/converters/test_download_converters_structure.py b/tests/back/converters/test_download_converters_structure.py new file mode 100644 index 000000000..2d98e6d6f --- /dev/null +++ b/tests/back/converters/test_download_converters_structure.py @@ -0,0 +1,80 @@ +"""Structure estimates of converters that download pretrained models. + +They cannot join the estimate-vs-runtime test (fitting loads the model), but +building them is cheap because the model is only loaded on fit, so their +`infer_output_columns` overrides are checked directly. +""" + +import pytest + +from DashAI.back.converters.hugging_face.embedding import Embedding +from DashAI.back.converters.hugging_face.image_embedding import ( + ImageEmbeddingConverter, +) +from DashAI.back.converters.hugging_face.tokenizer import TokenizerConverter +from DashAI.back.converters.segmentation.sam3_segment_converter import ( + SAM3SegmentConverter, +) +from DashAI.back.preprocessing.structure_types import BlockItem, ColumnItem +from DashAI.back.splitters.splits_payload import schema_placeholder_defaults + + +def _build(cls, **params): + return cls(**{**schema_placeholder_defaults(cls), **params}) + + +@pytest.mark.parametrize( + ("cls", "type_name"), + [(Embedding, "Float"), (TokenizerConverter, "Integer")], +) +def test_text_encoders_replace_the_text_with_one_block(cls, type_name): + inputs = [ColumnItem(name="review", type="Text", dtype="string")] + + delta = _build(cls).infer_output_columns(inputs) + + assert delta.kept == [] + (block,) = delta.added + assert isinstance(block, BlockItem) + assert (block.type, block.count) == (type_name, None) + + +@pytest.mark.parametrize("keep_source", [True, False]) +def test_image_embedding_keeps_the_source_only_when_asked(keep_source): + inputs = [ColumnItem(name="photo", type="Image")] + converter = _build(ImageEmbeddingConverter, keep_source_column=keep_source) + + delta = converter.infer_output_columns(inputs) + + assert delta.kept == (inputs if keep_source else []) + (block,) = delta.added + assert (block.type, block.count) == ("Float", None) + + +@pytest.mark.parametrize( + ("keep_mask", "expected"), + [ + (False, ["segment_1", "seg_score_1", "segment_2", "seg_score_2"]), + ( + True, + [ + "segment_1", + "seg_score_1", + "segment_2", + "seg_score_2", + "mask_1", + "mask_2", + ], + ), + ], +) +def test_sam3_names_one_column_set_per_rank(keep_mask, expected): + inputs = [ColumnItem(name="photo", type="Image")] + converter = _build(SAM3SegmentConverter, max_masks=2, keep_binary_mask=keep_mask) + + delta = converter.infer_output_columns(inputs) + + assert delta.kept == inputs + assert [item.name for item in delta.added] == expected + types = {item.name: item.type for item in delta.added} + assert types["segment_1"] == "Image" + assert types["seg_score_1"] == "Float" diff --git a/tests/back/converters/test_infer_output_columns_defaults.py b/tests/back/converters/test_infer_output_columns_defaults.py new file mode 100644 index 000000000..28812f7fe --- /dev/null +++ b/tests/back/converters/test_infer_output_columns_defaults.py @@ -0,0 +1,137 @@ +import pyarrow as pa +import pytest + +from DashAI.back.converters.base_converter import BaseConverter +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + ColumnItem, + RowsNotSupportedError, + type_fields, +) +from DashAI.back.types.value_types import Float, Integer + + +class _FakeConverter(BaseConverter): + SCHEMA = None + metadata = {} + + def get_output_type(self, column_name=None): + return Float(arrow_type=pa.float64()) + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + +class _Replace(_FakeConverter): + COLUMN_OPERATION = "replace" + + +class _ReplacePreserving(_FakeConverter): + COLUMN_OPERATION = "replace" + PRESERVES_INPUT_TYPE = True + + +class _Add(_FakeConverter): + COLUMN_OPERATION = "add" + + +class _Expand(_FakeConverter): + COLUMN_OPERATION = "expand" + + +class _Select(_FakeConverter): + COLUMN_OPERATION = "select" + + +class _Rows(_FakeConverter): + COLUMN_OPERATION = "rows" + + +class _Undeclared(_FakeConverter): + pass + + +def _inputs(): + return [ + ColumnItem(name="age", type="Integer", dtype="int64"), + ColumnItem(name="height", type="Float", dtype="float64"), + ] + + +def test_replace_keeps_inputs_retyped_with_output_type(): + delta = _Replace().infer_output_columns(_inputs()) + + assert [item.name for item in delta.kept] == ["age", "height"] + assert {item.type for item in delta.kept} == {"Float"} + assert delta.added == [] + + +def test_replace_preserving_input_type_keeps_inputs_unchanged(): + inputs = _inputs() + delta = _ReplacePreserving().infer_output_columns(inputs) + + assert delta.kept == inputs + assert delta.added == [] + + +def test_replace_retypes_a_block_without_changing_its_identity(): + block = BlockItem(step=0, slot="Categorical", label="ohe_*", type="Categorical") + delta = _Replace().infer_output_columns([block]) + + (kept,) = delta.kept + assert (kept.step, kept.slot) == (0, "Categorical") + assert kept.type == "Float" + + +def test_add_keeps_inputs_and_adds_one_unknown_block(): + inputs = _inputs() + delta = _Add().infer_output_columns(inputs) + + assert delta.kept == inputs + (block,) = delta.added + assert isinstance(block, BlockItem) + assert block.type == "Float" + assert block.count is None + + +def test_expand_removes_inputs_and_adds_one_unknown_block(): + delta = _Expand().infer_output_columns(_inputs()) + + assert delta.kept == [] + (block,) = delta.added + assert block.type == "Float" + assert block.count is None + + +def test_select_adds_one_block_per_input_type_preserving_types(): + delta = _Select().infer_output_columns(_inputs()) + + assert delta.kept == [] + assert sorted(block.type for block in delta.added) == ["Float", "Integer"] + assert all(block.count is None for block in delta.added) + + +def test_rows_is_not_supported(): + with pytest.raises(RowsNotSupportedError): + _Rows().infer_output_columns(_inputs()) + + +def test_undeclared_operation_behaves_like_expand(): + delta = _Undeclared().infer_output_columns(_inputs()) + + assert delta.kept == [] + (block,) = delta.added + assert block.count is None + + +def test_metadata_exposes_column_operation(): + assert _Replace.get_metadata()["column_operation"] == "replace" + assert _Undeclared.get_metadata()["column_operation"] is None + + +def test_type_fields_reads_display_name_and_dtype(): + assert type_fields(Integer(arrow_type=pa.int32())) == ("Integer", "int32") + assert type_fields(None) == (None, None) diff --git a/tests/back/preprocessing/test_column_ref.py b/tests/back/preprocessing/test_column_ref.py index 18510c20a..60087bc76 100644 --- a/tests/back/preprocessing/test_column_ref.py +++ b/tests/back/preprocessing/test_column_ref.py @@ -125,3 +125,25 @@ def test_converter_sequence_model_dump_round_trips(): dumped = sequence.model_dump(mode="json") restored = ConverterSequence.model_validate(dumped) assert restored == sequence + + +def test_named_group_ref_resolves_to_that_one_column(): + refs = [GroupColumnRef(step=0, name="date_month")] + resolved = resolve_refs(refs, {0: ["date_year", "date_month"]}) + assert resolved == ["date_month"] + + +def test_named_group_ref_to_a_column_the_step_never_produced_raises_key_error(): + with pytest.raises(KeyError): + resolve_refs([GroupColumnRef(step=0, name="age")], {0: ["date_year"]}) + + +def test_group_ref_cannot_have_both_a_slot_and_a_name(): + with pytest.raises(ValueError, match="slot"): + GroupColumnRef(step=0, slot="Integer", name="date_year") + + +def test_parse_column_refs_round_trips_a_named_group_ref(): + raw = [{"kind": "group", "step": 1, "name": "date_month"}] + (ref,) = parse_column_refs(raw) + assert (ref.step, ref.name, ref.slot) == (1, "date_month", None) diff --git a/tests/back/preprocessing/test_session_preprocessor.py b/tests/back/preprocessing/test_session_preprocessor.py index e2bd22c6c..1cb3a4ee5 100644 --- a/tests/back/preprocessing/test_session_preprocessor.py +++ b/tests/back/preprocessing/test_session_preprocessor.py @@ -500,3 +500,75 @@ def __init__(self, classes): assert restored.transform_dataset(single).to_pandas()["age"].tolist() == [10] assert restored.component_registry is None + + +class _TargetRecordingConverter(BaseConverter): + """A supervised converter that fails without a target, like a selector.""" + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + SUPERVISED = True + seen_targets = [] + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + if y is None: + raise ValueError("This transformer requires y for fitting") + type(self).seen_targets.append(list(y.column_names)) + return self + + def transform(self, x, y=None): + return x + + +def test_supervised_step_is_fit_with_the_target_columns(): + registry = _FakeRegistry({"Selector": _TargetRecordingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Selector", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry, target_columns=["label"]) + schema = { + "age": {"type": "Integer", "dtype": "int64"}, + "label": {"type": "Integer", "dtype": "int64"}, + } + split = {"train": _dataset({"age": [1, 2, 3], "label": [0, 1, 0]}, schema)} + + preprocessor.fit_transform(split) + + assert _TargetRecordingConverter.seen_targets == [["label"]] + + +def test_a_renamed_new_column_is_recorded_under_its_final_name(): + # The additive converter appends "derived", which already exists in the + # dataset outside the step's scope, so the new column becomes + # "derived_1"; the step's group must point at that one, not at the + # untouched original. + registry = _FakeRegistry({"Additive": _FakeAdditiveConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Additive", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + schema = { + "age": {"type": "Integer", "dtype": "int64"}, + "derived": {"type": "Integer", "dtype": "int64"}, + } + split = {"train": _dataset({"age": [1, 2], "derived": [7, 7]}, schema)} + + transformed, resolved = preprocessor.fit_transform(split) + + assert resolved == {0: ["derived_1"]} + assert transformed["train"].to_pandas()["derived_1"].tolist() == [10, 20] + assert preprocessor.resolved_slots == {0: {"Integer": ["derived_1"]}} diff --git a/tests/back/preprocessing/test_structure.py b/tests/back/preprocessing/test_structure.py new file mode 100644 index 000000000..63e80c566 --- /dev/null +++ b/tests/back/preprocessing/test_structure.py @@ -0,0 +1,228 @@ +import pandas as pd + +from DashAI.back.converters.imbalanced_learn.smote_converter import SMOTEConverter +from DashAI.back.converters.scikit_learn.pca import PCA +from DashAI.back.converters.scikit_learn.select_k_best import SelectKBest +from DashAI.back.converters.scikit_learn.standard_scaler import StandardScaler +from DashAI.back.converters.simple_converters.date_features import ( + DateFeaturesConverter, +) +from DashAI.back.dataloaders.classes.dashai_dataset import ( + to_dashai_dataset, + transform_dataset_with_schema, +) +from DashAI.back.preprocessing.column_ref import ( + ConverterSequence, + ConverterStep, + GroupColumnRef, + RawColumnRef, +) +from DashAI.back.preprocessing.session_preprocessor import SessionPreprocessor +from DashAI.back.preprocessing.structure import infer_structure +from DashAI.back.preprocessing.structure_types import BlockItem, ColumnItem + +REGISTRY = { + cls.__name__: {"class": cls} + for cls in ( + PCA, + SelectKBest, + StandardScaler, + DateFeaturesConverter, + SMOTEConverter, + ) +} + +SCHEMA = { + "age": {"type": "Integer", "dtype": "int64"}, + "height": {"type": "Float", "dtype": "float64"}, + "city": {"type": "Categorical", "dtype": "string"}, + "date": {"type": "Date", "dtype": "%Y-%m-%d"}, + "date_year": {"type": "Integer", "dtype": "int64"}, + "label": {"type": "Categorical", "dtype": "string"}, +} + + +def _dataset(): + frame = pd.DataFrame( + { + "age": [30, 41, 25, 60], + "height": [1.7, 1.8, 1.6, 1.75], + "city": ["a", "b", "a", "c"], + "date": ["2024-01-05", "2024-02-10", "2024-03-15", "2024-04-20"], + "date_year": [1, 2, 3, 4], + "label": ["yes", "no", "yes", "no"], + } + ) + return transform_dataset_with_schema(to_dashai_dataset(frame), SCHEMA) + + +def _infer(steps, candidates=("age", "height", "city", "date"), target=("label",)): + return infer_structure( + dataset_types=_dataset().types, + candidates=list(candidates), + target=list(target), + steps=steps, + component_registry=REGISTRY, + ) + + +def _step(converter, scope, **params): + return ConverterStep(converter=converter, params=params, scope=scope) + + +def _names(state): + return [item.name for item in state if isinstance(item, ColumnItem)] + + +def test_initial_state_is_the_candidates_without_the_target(): + result = _infer([], candidates=["age", "label", "city"]) + + assert _names(result.initial) == ["age", "city"] + assert result.final == result.initial + assert result.valid + + +def test_a_consumed_column_cannot_be_used_by_a_later_step(): + result = _infer( + [ + _step( + "PCA", + [RawColumnRef(name="age"), RawColumnRef(name="height")], + n_components=2, + ), + _step("StandardScaler", [RawColumnRef(name="age")]), + ] + ) + + first, second = result.steps + assert first.status == "ok" + assert "age" not in _names(first.state) + (block,) = first.added + assert isinstance(block, BlockItem) + assert (block.step, block.slot, block.count) == (0, None, 2) + assert second.status == "error" + assert second.error.code == "missing_ref" + assert not result.valid + + +def test_the_target_cannot_be_in_a_scope(): + result = _infer([_step("StandardScaler", [RawColumnRef(name="label")])]) + + assert result.steps[0].error.code == "target_in_scope" + + +def test_steps_after_a_broken_one_are_blocked(): + result = _infer( + [ + _step("StandardScaler", [RawColumnRef(name="missing")]), + _step("StandardScaler", [RawColumnRef(name="age")]), + ] + ) + + assert [step.status for step in result.steps] == ["error", "blocked"] + assert _names(result.final) == ["age", "height", "city", "date"] + + +def test_an_unknown_converter_is_an_error(): + result = _infer([_step("Nope", [RawColumnRef(name="age")])]) + + assert result.steps[0].error.code == "unknown_converter" + + +def test_a_scope_type_the_converter_does_not_accept_is_an_error(): + result = _infer([_step("StandardScaler", [RawColumnRef(name="city")])]) + + assert result.steps[0].error.code == "type_not_allowed" + + +def test_row_changing_converters_are_not_supported(): + result = _infer([_step("SMOTEConverter", [RawColumnRef(name="age")])]) + + assert result.steps[0].error.code == "rows_not_supported" + + +def test_more_components_than_columns_is_an_error(): + result = _infer( + [ + _step( + "PCA", + [RawColumnRef(name="age"), RawColumnRef(name="height")], + n_components=5, + ) + ] + ) + + error = result.steps[0].error + assert error.code == "n_components_exceeds" + assert error.params == {"n_components": 5, "columns": 2} + + +def test_a_named_generated_column_can_be_used_by_a_later_step(): + result = _infer( + [ + _step("DateFeaturesConverter", [RawColumnRef(name="date")]), + _step("StandardScaler", [GroupColumnRef(step=0, name="date_month")]), + ] + ) + + assert result.valid + month = next(item for item in result.final if item.name == "date_month") + assert (month.type, month.origin) == ("Float", 0) + + +def test_a_selector_over_mixed_types_adds_one_slotted_block_per_type(): + result = _infer( + [ + _step( + "SelectKBest", + [RawColumnRef(name="age"), RawColumnRef(name="height")], + k=1, + ), + ] + ) + + blocks = result.steps[0].added + assert {(block.slot, block.type) for block in blocks} == { + ("Integer", "Integer"), + ("Float", "Float"), + } + + +def test_a_clashing_new_column_is_renamed_like_the_runtime_does(): + # "date_year" exists in the dataset but is not a candidate, so the one + # DateFeatures creates must become "date_year_1", as the runtime names it. + step = _step("DateFeaturesConverter", [RawColumnRef(name="date")]) + + estimated = _infer([step]) + + preprocessor = SessionPreprocessor(ConverterSequence(steps=[step]), REGISTRY) + _, resolved = preprocessor.fit_transform({"train": _dataset()}) + added = [item.name for item in estimated.steps[0].added] + assert added[0] == "date_year_1" + assert added == resolved[0] + + +class _BrokenEstimateConverter(StandardScaler): + """Stands in for a plugin whose structure estimate crashes.""" + + def get_output_type(self, column_name=None): + raise RuntimeError("no idea") + + +def test_a_crashing_estimate_falls_back_to_one_unknown_block(): + registry = {**REGISTRY, "Broken": {"class": _BrokenEstimateConverter}} + + result = infer_structure( + dataset_types=_dataset().types, + candidates=["age", "height"], + target=["label"], + steps=[_step("Broken", [RawColumnRef(name="age")])], + component_registry=registry, + ) + + step = result.steps[0] + assert step.status == "ok" + assert [w.code for w in step.warnings] == ["inference_fallback"] + (block,) = step.added + assert (block.type, block.count) == (None, None) + assert _names(step.state) == ["height"] diff --git a/tests/back/preprocessing/test_structure_matches_runtime.py b/tests/back/preprocessing/test_structure_matches_runtime.py new file mode 100644 index 000000000..dca9f28b8 --- /dev/null +++ b/tests/back/preprocessing/test_structure_matches_runtime.py @@ -0,0 +1,323 @@ +"""The structure a converter estimates must match what it really produces. + +For every registered converter, `infer_output_columns` (called on an unfitted +instance, with no data) is compared against a real fit + transform on a small +dataset, spliced back the way SessionPreprocessor does it. This is what +guarantees that a converter chain the session wizard accepts never fails at +training time because it references a column that does not exist. +""" + +import pandas as pd +import pytest + +from DashAI.back.converters.base_converter import BaseConverter +from DashAI.back.converters.dataset_columns import ( + rebuild_dataset_with_transformed_columns, +) +from DashAI.back.dataloaders.classes.dashai_dataset import ( + to_dashai_dataset, + transform_dataset_with_schema, +) +from DashAI.back.initial_components import get_initial_components +from DashAI.back.preprocessing.session_preprocessor import SessionPreprocessor +from DashAI.back.preprocessing.structure_types import ( + BlockItem, + ColumnItem, + RowsNotSupportedError, + type_fields, +) +from DashAI.back.splitters.splits_payload import schema_placeholder_defaults + +N_ROWS = 30 + +SCHEMA = { + "i_full": {"type": "Integer", "dtype": "int64"}, + "i2": {"type": "Integer", "dtype": "int64"}, + "i_null": {"type": "Integer", "dtype": "int64"}, + "f1": {"type": "Float", "dtype": "float64"}, + "f2": {"type": "Float", "dtype": "float64"}, + "f_null": {"type": "Float", "dtype": "float64"}, + "cat": {"type": "Categorical", "dtype": "string"}, + "text": {"type": "Text", "dtype": "string"}, + "code": {"type": "Text", "dtype": "string"}, + "date": {"type": "Date", "dtype": "%Y-%m-%d"}, + "target": {"type": "Categorical", "dtype": "string"}, +} + +# Converters that cannot run here: they download pretrained models. Their +# overrides get direct unit tests instead. +NEEDS_DOWNLOAD = { + "Embedding", + "TokenizerConverter", + "ImageEmbeddingConverter", + "SAM3SegmentConverter", +} + +# (converter name, params on top of the schema placeholders, scope columns) +CASES = [ + ("StandardScaler", {}, ["i_full", "f1"]), + ("MinMaxScaler", {}, ["i_full", "f1"]), + ("MaxAbsScaler", {}, ["i_full", "f1"]), + ("Normalizer", {}, ["i_full", "f1"]), + ("SimpleImputer", {"strategy": "mean"}, ["i_null", "f_null"]), + ("SimpleImputer", {"strategy": "most_frequent"}, ["cat", "i_null"]), + ( + "SimpleImputer", + {"strategy": "median", "add_indicator": True}, + ["i_null", "f_null"], + ), + ("KNNImputer", {}, ["i_null", "f_null"]), + ("MissingIndicator", {}, ["i_null", "f_null"]), + ("Binarizer", {}, ["i_full", "f1"]), + ("OrdinalEncoder", {}, ["cat"]), + ("OneHotEncoder", {}, ["cat"]), + ("LabelEncoder", {}, ["cat"]), + ("PCA", {"n_components": 2}, ["i_full", "f1", "f2"]), + ("IncrementalPCA", {"n_components": 2}, ["i_full", "f1", "f2"]), + ("TruncatedSVD", {"n_components": 2}, ["i_full", "f1", "f2"]), + ("FastICA", {"n_components": 2}, ["i_full", "f1", "f2"]), + ("Nystroem", {"n_components": 5}, ["i_full", "f1"]), + ("RBFSampler", {"n_components": 5}, ["i_full", "f1"]), + ("SkewedChi2Sampler", {"n_components": 5}, ["i_full", "i2"]), + ("AdditiveChi2Sampler", {}, ["i_full", "i2"]), + ("PolynomialFeatures", {"degree": 2}, ["i_full", "f1"]), + ("VarianceThreshold", {}, ["i_full", "f1"]), + ("SelectKBest", {"k": 1}, ["i_full", "f1"]), + ("SelectPercentile", {}, ["i_full", "f1"]), + ("SelectFdr", {}, ["i_full", "f1"]), + ("SelectFpr", {}, ["i_full", "f1"]), + ("SelectFwe", {}, ["i_full", "f1"]), + ("GenericUnivariateSelect", {}, ["i_full", "f1"]), + ("BagOfWordsConverter", {}, ["text"]), + ("TFIDFConverter", {}, ["text"]), + ( + "CharacterReplacer", + {"char_to_replace": "-", "replacement_char": ""}, + ["code"], + ), + ("ColumnArithmetic", {"operation": "add"}, ["i_full", "i2"]), + ("ColumnArithmetic", {"operation": "multiply"}, ["i_null", "i2"]), + ("ColumnArithmetic", {"operation": "divide"}, ["i_full", "f1"]), + ("ColumnConcat", {}, ["cat", "text"]), + ("NumericExpansion", {"operation": "square"}, ["i_null", "i_full"]), + ("NumericExpansion", {"operation": "log1p"}, ["f1"]), + ("DateFeaturesConverter", {}, ["date"]), + ("TypeCast", {"new_type": "Float"}, ["i_full"]), + ("ColumnRemover", {}, ["f1"]), +] + + +# Cases whose output size follows from params and input columns alone: the +# estimate must be exact (no block of unknown size), not just conservative. +EXACT = { + "SimpleImputer[strategy=mean]", + "SimpleImputer[strategy=most_frequent]", + "KNNImputer", + "MissingIndicator", + "Binarizer", + "OrdinalEncoder", + "LabelEncoder", + "PCA[n_components=2]", + "IncrementalPCA[n_components=2]", + "TruncatedSVD[n_components=2]", + "FastICA[n_components=2]", + "Nystroem[n_components=5]", + "RBFSampler[n_components=5]", + "SkewedChi2Sampler[n_components=5]", + "AdditiveChi2Sampler", + "PolynomialFeatures[degree=2]", + "StandardScaler", + "MinMaxScaler", + "MaxAbsScaler", + "Normalizer", + "CharacterReplacer[char_to_replace=-,replacement_char=]", + "ColumnArithmetic[operation=add]", + "ColumnArithmetic[operation=multiply]", + "ColumnArithmetic[operation=divide]", + "ColumnConcat", + "NumericExpansion[operation=square]", + "NumericExpansion[operation=log1p]", + "DateFeaturesConverter", + "TypeCast[new_type=Float]", + "ColumnRemover", +} + +# Cases whose output is only concrete columns: names must be known. +CONCRETE_ONLY = { + "ColumnArithmetic[operation=add]", + "ColumnArithmetic[operation=multiply]", + "ColumnArithmetic[operation=divide]", + "ColumnConcat", + "NumericExpansion[operation=square]", + "NumericExpansion[operation=log1p]", + "DateFeaturesConverter", + "ColumnRemover", +} + + +def _converter_classes(): + return { + cls.__name__: cls + for cls in get_initial_components() + if isinstance(cls, type) + and issubclass(cls, BaseConverter) + and cls.__module__.startswith("DashAI.back.converters") + } + + +def _is_rows(cls): + return cls.COLUMN_OPERATION == "rows" + + +def _dataset(): + words = ["red apple", "green pear", "blue plum", "red plum", "green apple"] + frame = pd.DataFrame( + { + "i_full": [(i * 7) % 11 + 1 for i in range(N_ROWS)], + "i2": [(i * 3) % 5 + 1 for i in range(N_ROWS)], + "i_null": pd.array( + [None if i % 6 == 0 else i % 9 for i in range(N_ROWS)], + dtype="Int64", + ), + "f1": [0.5 + (i * 13) % 17 / 3 for i in range(N_ROWS)], + "f2": [1.25 + (i * 5) % 7 / 2 for i in range(N_ROWS)], + "f_null": [ + None if i % 5 == 0 else 0.1 + (i % 4) * 1.3 for i in range(N_ROWS) + ], + "cat": [["a", "b", "c"][i % 3] for i in range(N_ROWS)], + "text": [words[i % len(words)] for i in range(N_ROWS)], + "code": [f"{i % 4}-{i % 7}" for i in range(N_ROWS)], + "date": [f"2024-{i % 12 + 1:02d}-{i % 28 + 1:02d}" for i in range(N_ROWS)], + "target": [["yes", "no"][(i * 7) % 11 % 2] for i in range(N_ROWS)], + } + ) + return transform_dataset_with_schema(to_dashai_dataset(frame), SCHEMA) + + +def _build(cls, params): + return cls(**{**schema_placeholder_defaults(cls), **params}) + + +def _scope_items(dataset, scope): + items = [] + for name in scope: + type_name, dtype = type_fields(dataset.types[name]) + items.append(ColumnItem(name=name, type=type_name, dtype=dtype)) + return items + + +def _case_id(case): + name, params, _ = case + extra = ",".join(f"{k}={v}" for k, v in params.items()) + return f"{name}[{extra}]" if extra else name + + +@pytest.mark.parametrize("case", CASES, ids=[_case_id(c) for c in CASES]) +def test_estimated_structure_matches_runtime(case): + name, params, scope = case + cls = _converter_classes()[name] + dataset = _dataset() + + delta = _build(cls, params).infer_output_columns(_scope_items(dataset, scope)) + if _case_id(case) in EXACT: + unknown = [ + item + for item in delta.added + if isinstance(item, BlockItem) and item.count is None + ] + assert unknown == [], "the estimate should know the output size" + if _case_id(case) in CONCRETE_ONLY: + assert not any(isinstance(i, BlockItem) for i in delta.added), "no blocks" + + converter = _build(cls, params) + x = dataset.select_columns(scope) + if cls.SUPERVISED: + converter.fit(x, dataset.select_columns(["target"])) + else: + converter.fit(x) + transformed = converter.transform(x) + rebuilt = rebuild_dataset_with_transformed_columns( + dataset, + transformed, + scope, + [dataset.column_names.index(column) for column in scope], + ) + + def real_type(column): + return type_fields(rebuilt.types[column])[0] + + real_out = list(transformed.column_names) + real_new = [c for c in real_out if c not in scope] + concrete = [item for item in delta.added if isinstance(item, ColumnItem)] + blocks = [item for item in delta.added if isinstance(item, BlockItem)] + assert not (concrete and blocks), "a delta never mixes columns and blocks" + + # Every scope column the estimate says survives really survives, with + # the estimated type. This is the property that keeps a later step from + # referencing a column that no longer exists. + kept_names = [item.name for item in delta.kept if isinstance(item, ColumnItem)] + for item in delta.kept: + assert item.name in real_out, f"{item.name} was estimated to survive" + assert real_type(item.name) == item.type, f"type of kept {item.name}" + + # A scope column the estimate drops may still survive only when a block + # stands for it (a selector keeps some scope columns, unknown which). + real_survivors = [c for c in scope if c in real_out and c not in kept_names] + if real_survivors: + assert blocks, f"{real_survivors} survive but nothing accounts for them" + + # New concrete columns: exact names and types. + if concrete: + assert [item.name for item in concrete] == real_new, "new column names" + for item in concrete: + assert real_type(item.name) == item.type, f"type of new {item.name}" + return + if not blocks: + assert real_new == [], f"unexpected new columns {real_new}" + return + + # Symbolic blocks stand for the new columns plus any unnamed survivors. + # With an unknown count a block may turn out empty (a selector keeping + # nothing of that type), so real types only need to be a subset; with a + # known count, the block has exactly that many columns. + block_columns = real_new + real_survivors + real_by_type = {} + for column in block_columns: + real_by_type.setdefault(real_type(column), []).append(column) + estimated_types = {block.type for block in blocks} + assert set(real_by_type) <= estimated_types, "block types" + for block in blocks: + if block.count is not None: + assert block.count == len(real_by_type.get(block.type, [])), "count" + + # The runtime classifies a step's group (its new columns, or all its + # output when it adds none) into slots by type; they must line up with + # the blocks. + group = real_new or real_out + runtime_slots = SessionPreprocessor._classify_by_type(converter, group) + assert set(runtime_slots) <= estimated_types, "runtime slots" + + +def test_every_converter_is_covered(): + covered = {name for name, _, _ in CASES} | NEEDS_DOWNLOAD + uncovered = { + name + for name, cls in _converter_classes().items() + if not _is_rows(cls) and name not in covered + } + assert uncovered == set(), "add a CASES entry for every new converter" + + +def test_rows_operation_matches_changes_row_count(): + for name, cls in _converter_classes().items(): + assert _is_rows(cls) == bool(cls.CHANGES_ROW_COUNT), name + + +def test_rows_converters_are_not_supported_in_sessions(): + for cls in _converter_classes().values(): + if not _is_rows(cls): + continue + with pytest.raises(RowsNotSupportedError): + BaseConverter.infer_output_columns( + cls.__new__(cls), _scope_items(_dataset(), ["i_full"]) + )