From 02d82607c430b73fd42d65a0d17f7cb1f7d12829 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 12:27:39 -0300 Subject: [PATCH 01/88] feat: add error handling and job name methods to pipeline nodes --- DashAI/back/pipeline/data_selector_node.py | 6 +++ DashAI/back/pipeline/exploration_node.py | 28 +++++++++++--- DashAI/back/pipeline/prediction_node.py | 15 +++++++- DashAI/back/pipeline/retrieve_model_node.py | 6 +++ DashAI/back/pipeline/train_node.py | 41 +++++++++++++++++++-- 5 files changed, 85 insertions(+), 11 deletions(-) diff --git a/DashAI/back/pipeline/data_selector_node.py b/DashAI/back/pipeline/data_selector_node.py index 0acc22a58..33f78b738 100644 --- a/DashAI/back/pipeline/data_selector_node.py +++ b/DashAI/back/pipeline/data_selector_node.py @@ -31,6 +31,12 @@ def __init__(self, **kwargs) -> None: def set_status_as_delivered(self) -> None: log.debug("DataSelector executed successfully.") + def set_status_as_error(self) -> None: + log.error("DataSelector encountered an error.") + + def get_job_name(self) -> str: + return f"DataSelector: {self.kwargs.get('name', 'Unknown')}" + async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: context["dataset_name"] = self.kwargs["name"] dataset_dir = pathlib.Path(self.kwargs["file_path"]) diff --git a/DashAI/back/pipeline/exploration_node.py b/DashAI/back/pipeline/exploration_node.py index 7e9423706..cc9627bc4 100644 --- a/DashAI/back/pipeline/exploration_node.py +++ b/DashAI/back/pipeline/exploration_node.py @@ -6,7 +6,6 @@ from kink import di from DashAI.back.config import DefaultSettings -from DashAI.back.dependencies.database.models import Explorer from DashAI.back.dependencies.registry.component_registry import ComponentRegistry from DashAI.back.exploration.base_explorer import BaseExplorer from DashAI.back.job.base_job import BaseJob, JobError @@ -44,6 +43,12 @@ def __init__( def set_status_as_delivered(self) -> None: log.debug("DataExploration executed successfully.") + def set_status_as_error(self) -> None: + log.error("DataExploration encountered an error.") + + def get_job_name(self) -> str: + return "DataExploration Node" + async def run( self, context: Dict[str, Any], @@ -70,8 +75,8 @@ async def run( exploration_type = option["exploration_type"] parameters = option.get("parameters", {}) columns = option.get("columns", []) - id = option.get("id", str(idx)) - name = option.get("name") + explorer_id = option.get("id", str(idx)) + name = option.get("name", "") explorer_component_class: Type[BaseExplorer] = component_registry( di @@ -83,11 +88,22 @@ async def run( loaded_dataset, columns ) - explorer_info = Explorer( + # Create a simple explorer info object for pipeline execution + # This is not a database object, just a container for explorer metadata + class ExplorerInfo: + def __init__(self, exploration_type, columns, parameters, id, name): + self.exploration_type = exploration_type + self.columns = columns + self.parameters = parameters + self.id = id + self.name = name + + explorer_info = ExplorerInfo( exploration_type=exploration_type, columns=columns, parameters=parameters, - id=id, + id=explorer_id, + name=name, ) result = explorer_instance.launch_exploration( @@ -101,7 +117,7 @@ async def run( result=result, ) - results[str(id)] = { + results[str(explorer_id)] = { "exploration_type": exploration_type, "path": str(save_path), "parameters": parameters, diff --git a/DashAI/back/pipeline/prediction_node.py b/DashAI/back/pipeline/prediction_node.py index d6e7de006..9534c10c4 100644 --- a/DashAI/back/pipeline/prediction_node.py +++ b/DashAI/back/pipeline/prediction_node.py @@ -31,6 +31,12 @@ def __init__(self, **kwargs) -> None: def set_status_as_delivered(self) -> None: log.debug("Prediction executed successfully.") + def set_status_as_error(self) -> None: + log.error("Prediction encountered an error.") + + def get_job_name(self) -> str: + return "Prediction Node" + async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: loaded_dataset = context["dataset"] model = context["model_class"] @@ -40,10 +46,17 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: try: prepared_dataset = loaded_dataset.select_columns(context["input_columns"]) y_pred_proba = np.array(trained_model.predict(prepared_dataset)) + + # Check if it's classification (2D array with probabilities) or regression (1D array) if isinstance(y_pred_proba[0], str): + # Text/categorical predictions y_pred = y_pred_proba - else: + elif y_pred_proba.ndim == 2: + # Classification: multi-class probabilities, get class with highest probability y_pred = np.argmax(y_pred_proba, axis=1) + else: + # Regression: predictions are already the final values + y_pred = y_pred_proba except ValueError as ve: log.error(f"Validation Error: {ve}") diff --git a/DashAI/back/pipeline/retrieve_model_node.py b/DashAI/back/pipeline/retrieve_model_node.py index fe3d04df4..a06169f43 100644 --- a/DashAI/back/pipeline/retrieve_model_node.py +++ b/DashAI/back/pipeline/retrieve_model_node.py @@ -38,6 +38,12 @@ def __init__( def set_status_as_delivered(self) -> None: log.debug("RetrieveModel executed successfully.") + def set_status_as_error(self) -> None: + log.error("RetrieveModel encountered an error.") + + def get_job_name(self) -> str: + return f"RetrieveModel: {self.model}" + async def run( self, context: Dict[str, Any], diff --git a/DashAI/back/pipeline/train_node.py b/DashAI/back/pipeline/train_node.py index e47bd1b2a..e961537e7 100644 --- a/DashAI/back/pipeline/train_node.py +++ b/DashAI/back/pipeline/train_node.py @@ -75,6 +75,12 @@ def __init__( def set_status_as_delivered(self) -> None: log.debug("Train executed successfully.") + def set_status_as_error(self) -> None: + log.error("Train encountered an error.") + + def get_job_name(self) -> str: + return f"Train: {self.model}" + async def run( self, context: Dict[str, Any], @@ -109,7 +115,9 @@ async def run( try: prepared_dataset = task_instance.prepare_for_task( - dataset, output_columns_names + dataset=dataset, + input_columns=input_columns_names, + output_columns=output_columns_names, ) n_labels = None if self.task in [ @@ -130,8 +138,8 @@ async def run( x, y = select_columns( prepared_dataset, input_columns_names, output_columns_names ) - x = split_dataset(x) - y = split_dataset(y) + x_splits = split_dataset(x) + y_splits = split_dataset(y) except Exception as e: log.exception(e) @@ -159,7 +167,32 @@ async def run( ) from e try: - model.fit(x["train"], y["train"]) + # Check if model uses .train() (text models) or .fit() (sklearn models) + if hasattr(model, "train") and callable(getattr(model, "train")): + # Text models expect Dataset objects, not numpy arrays + model.train( + x_splits["train"], + y_splits["train"], + x_splits.get("validation"), + y_splits.get("validation"), + ) + # Keep splits as DashAIDataset for evaluation + x = x_splits + y = y_splits + elif hasattr(model, "fit") and callable(getattr(model, "fit")): + # Sklearn models expect numpy arrays + # Convert DashAIDataset to numpy arrays + x = {} + y = {} + for split in ["train", "validation", "test"]: + if split in x_splits: + x[split] = x_splits[split].to_pandas().values + if split in y_splits: + y[split] = y_splits[split].to_pandas().values.ravel() + + model.fit(x["train"], y["train"]) + else: + raise AttributeError(f"Model {model_class} has neither 'train' nor 'fit' method") except Exception as e: log.exception(e) raise JobError( From 515f421af169dbf9aab431dd285be8a646b72fc6 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 12:31:05 -0300 Subject: [PATCH 02/88] fix: add missing 'self' parameter to run method in BaseJob class feat: enhance PipelineJob with logging and error handling in run method --- DashAI/back/job/base_job.py | 2 +- DashAI/back/job/pipeline_job.py | 144 ++++++++++++++++++++------------ 2 files changed, 90 insertions(+), 56 deletions(-) diff --git a/DashAI/back/job/base_job.py b/DashAI/back/job/base_job.py index f1fcaaf52..bf32f5869 100644 --- a/DashAI/back/job/base_job.py +++ b/DashAI/back/job/base_job.py @@ -36,7 +36,7 @@ def get_job_name(self) -> str: raise NotImplementedError @abstractmethod - def run() -> None: + def run(self) -> None: """Run the job.""" raise NotImplementedError diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index c26c755b1..dc0fe6c92 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -1,8 +1,9 @@ +import asyncio import logging from typing import Any, Dict, List from kink import di -from sqlalchemy.orm import Session +from sqlalchemy.orm import sessionmaker from DashAI.back.dependencies.database.models import Pipeline from DashAI.back.dependencies.registry import ComponentRegistry @@ -13,68 +14,101 @@ class PipelineJob(BaseJob): def set_status_as_delivered(self) -> None: - pass - - async def run( + """Pipeline job delivered. No persistent status to update.""" + log.debug("PipelineJob marked as delivered.") + + def set_status_as_error(self) -> None: + """Pipeline job errored. No persistent status to update.""" + log.debug("PipelineJob marked as error.") + + def get_job_name(self) -> str: + """Return a descriptive name for the pipeline job.""" + pipeline_id = self.kwargs.get("id") + if not pipeline_id: + return "Pipeline" + + try: + session_factory: sessionmaker = di["session_factory"] + with session_factory() as db: + pipeline: Pipeline | None = db.get(Pipeline, pipeline_id) + if pipeline and getattr(pipeline, "name", None): + return f"Pipeline: {pipeline.name}" + except Exception: + pass + + return f"Pipeline ({pipeline_id})" + + def run( self, component_registry: ComponentRegistry = lambda di: di["component_registry"], ) -> None: - db: Session = self.kwargs["db"] - id: int = self.kwargs.get("id", None) - pipeline: Pipeline = db.get(Pipeline, id) - steps: List[Dict[str, Any]] = self.kwargs.get("steps", []) or pipeline.steps + """Execute the pipeline synchronously, awaiting async node runs internally.""" + session_factory: sessionmaker = di["session_factory"] + pipeline_id: int | None = self.kwargs.get("id") - if not id: + if not pipeline_id: raise JobError("No id provided to execute the pipeline.") - if not steps: - raise JobError("No steps provided to execute the pipeline.") - - if not steps: - raise JobError("Pipeline has no steps to execute") - - log.info(f"Starting pipeline execution for pipeline {id}...") - context: Dict[str, Any] = {"pipeline_id": id} + with session_factory() as db: + pipeline: Pipeline | None = db.get(Pipeline, pipeline_id) + if pipeline is None: + raise JobError(f"Pipeline with id {pipeline_id} not found.") - for idx, step in enumerate(steps): - node_id = step.get("id") - node_type = step.get("type") - node_config = step.get("config", {}) - - log.debug( - f"Pipeline {id}: Executing step {idx + 1}/{len(steps)} - " - f"{node_type} ({node_id})" + steps: List[Dict[str, Any]] = self.kwargs.get("steps", []) or ( + pipeline.steps or [] ) - - try: - node_class = component_registry(di)[node_type]["class"] - except KeyError as e: - error_msg = f"Component type {node_type} not found in registry" - raise JobError( - f"Error in node {node_id} ({node_type}): {error_msg}" - ) from e - - try: - node_instance = node_class(**node_config) - except Exception as e: - error_msg = f"Error in node {node_id} ({node_type}): {str(e)}" - log.exception(error_msg) - raise JobError(error_msg) from e - - try: - output = await node_instance.run(context=context) - self._update_context(context, pipeline, node_type, node_id, output) - log.debug(f"Node {node_id} executed successfully.") - - except Exception as e: - error_msg = f"Error in node {node_id} ({node_type}): {str(e)}" - log.exception(error_msg) - raise JobError(error_msg) from e - - log.info(f"Pipeline {id} execution completed successfully.") - db.add(pipeline) - db.commit() - self.set_status_as_delivered() + if not steps: + raise JobError("No steps provided to execute the pipeline.") + + log.info(f"Starting pipeline execution for pipeline {pipeline_id}...") + context: Dict[str, Any] = {"pipeline_id": pipeline_id} + + async def _execute_steps() -> None: + for idx, step in enumerate(steps): + node_id = step.get("id") + node_type = step.get("type") + node_config = step.get("config", {}) + + log.debug( + f"Pipeline {pipeline_id}: Executing step {idx + 1}/{len(steps)} - " + f"{node_type} ({node_id})" + ) + + try: + node_class = component_registry(di)[node_type]["class"] + except KeyError as e: + error_msg = f"Component type {node_type} not found in registry" + raise JobError( + f"Error in node {node_id} ({node_type}): {error_msg}" + ) from e + + try: + node_instance = node_class(**node_config) + except Exception as e: + error_msg = f"Error in node {node_id} ({node_type}): {str(e)}" + log.exception(error_msg) + raise JobError(error_msg) from e + + try: + output = await node_instance.run(context=context) + self._update_context( + context, pipeline, node_type, node_id, output + ) + log.debug(f"Node {node_id} executed successfully.") + except Exception as e: + error_msg = ( + f"Error in node {node_id} ({node_type}): {str(e)}" + ) + log.exception(error_msg) + raise JobError(error_msg) from e + + # Run async nodes in a new event loop + asyncio.run(_execute_steps()) + + log.info(f"Pipeline {pipeline_id} execution completed successfully.") + db.add(pipeline) + db.commit() + self.set_status_as_delivered() def _update_context( self, From 72152f256e1492c1a827102cb9cf123c103e38b5 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 12:31:44 -0300 Subject: [PATCH 03/88] feat: enhance pipeline prediction summary to differentiate between classification and regression data types --- DashAI/back/api/api_v1/endpoints/pipelines.py | 58 ++++++++++++------- DashAI/back/metrics/regression_metric.py | 7 ++- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/pipelines.py b/DashAI/back/api/api_v1/endpoints/pipelines.py index bb2e364eb..228f3d2e1 100644 --- a/DashAI/back/api/api_v1/endpoints/pipelines.py +++ b/DashAI/back/api/api_v1/endpoints/pipelines.py @@ -123,28 +123,44 @@ class distribution (if applicable), and sample data. if isinstance(data[0], str): summary["data_type"] = "string" else: - summary["data_type"] = "numeric" - class_set = set(data) - classes = [str(item) for item in class_set] - summary["Unique_classes"] = len(classes) - class_distribution = [] - id = 1 - for class_name in classes: - try: - occurrences = data.count(int(class_name)) - except ValueError as e: - raise HTTPException( - status_code=400, detail=f"Invalid class value: {class_name}" - ) from e - distribution = { - "id": id, - "Class": class_name, - "Ocurrences": occurrences, - "Percentage": round(occurrences / len(data) * 100, 2), + # Check if data looks like classification (integers) or regression (floats) + is_classification = all(isinstance(x, int) or (isinstance(x, float) and x.is_integer()) for x in data) + unique_values = len(set(data)) + + # If there are many unique values relative to total, it's likely regression + if not is_classification or unique_values > len(data) * 0.5: + # Regression: provide statistics + summary["data_type"] = "regression" + import numpy as np + data_array = np.array(data) + summary["statistics"] = { + "mean": float(np.mean(data_array)), + "median": float(np.median(data_array)), + "std": float(np.std(data_array)), + "min": float(np.min(data_array)), + "max": float(np.max(data_array)), } - id += 1 - class_distribution.append(distribution) - summary["class_distribution"] = class_distribution + else: + # Classification: count class occurrences + summary["data_type"] = "classification" + class_set = set(data) + classes = [str(item) for item in class_set] + summary["Unique_classes"] = len(classes) + class_distribution = [] + id = 1 + for class_name in classes: + # Count occurrences of the actual value (int or float) + original_value = int(float(class_name)) if '.' not in class_name else float(class_name) + occurrences = data.count(original_value) + distribution = { + "id": id, + "Class": class_name, + "Ocurrences": occurrences, + "Percentage": round(occurrences / len(data) * 100, 2), + } + id += 1 + class_distribution.append(distribution) + summary["class_distribution"] = class_distribution sample_data = [ {"id": idx, "value": value} for idx, value in enumerate(data[:50], 1) diff --git a/DashAI/back/metrics/regression_metric.py b/DashAI/back/metrics/regression_metric.py index daa198d77..ce77cae80 100644 --- a/DashAI/back/metrics/regression_metric.py +++ b/DashAI/back/metrics/regression_metric.py @@ -30,7 +30,12 @@ def prepare_to_metric( tuple[np.ndarray, np.ndarray] Tuple containing true values and predicted values as numpy arrays. """ - true_values = np.array(y.to_pandas().to_numpy().flatten()) + #true_values = np.array(y.to_pandas().to_numpy().flatten()) + # Handle both DashAIDataset and numpy array inputs + if isinstance(y, np.ndarray): + true_values = np.array(y).flatten() + else: + true_values = np.array(y.to_pandas().to_numpy().flatten()) pred_values = np.array(y_pred).flatten() return true_values, pred_values From 0ee70cd3dff5ca7ac01027611fa9ae2d586d15a6 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 12:31:53 -0300 Subject: [PATCH 04/88] feat: add load_preview and infer_datatypes endpoints for backward compatibility --- DashAI/back/api/api_v1/endpoints/datasets.py | 253 +++++++++++++++++++ 1 file changed, 253 insertions(+) diff --git a/DashAI/back/api/api_v1/endpoints/datasets.py b/DashAI/back/api/api_v1/endpoints/datasets.py index bd3494342..ce872497c 100644 --- a/DashAI/back/api/api_v1/endpoints/datasets.py +++ b/DashAI/back/api/api_v1/endpoints/datasets.py @@ -1248,6 +1248,259 @@ async def preview_with_types( ) from e +@router.post("/load_preview") +@inject +async def load_preview( + file: UploadFile = File(...), + params: str = Form(None), + component_registry: Dict = Depends(lambda: di["component_registry"]), +): + """ + Alias for preview_with_types - Load preview of a dataset file. + This endpoint is for backward compatibility with frontend API calls. + + Parameters + ---------- + file : UploadFile + The file uploaded by the user. + params : str + JSON string with parameters for the dataloader. + component_registry : Dict + Registry of available dataloaders. + + Returns + ------- + Dict + A dictionary containing: + - sample: First 10 rows of the dataset + - schema: Column types from Arrow + - inferred_types: Detailed type inference (DashAI types) + - preview_row_count: Number of rows used for inference + """ + try: + parsed_params = json.loads(params) if params else {} + + with tempfile.NamedTemporaryFile( + delete=False, suffix=file.filename + ) as tmp_file: + content = await file.read() + tmp_file.write(content) + tmp_file_path = tmp_file.name + + try: + inference_rows = parsed_params.get("inference_rows", 1000) + if file.filename.endswith(".zip"): + extract_dir = tempfile.mkdtemp() + try: + with zipfile.ZipFile(tmp_file_path, "r") as zf: + zf.extractall(extract_dir) + + supported_map = { + ".csv": "CSVDataLoader", + ".json": "JSONDataLoader", + ".xlsx": "ExcelDataLoader", + ".xls": "ExcelDataLoader", + } + dataloader_name = None + matched_file = None + for root, _, files in os.walk(extract_dir): + for f in files: + ext = os.path.splitext(f)[1].lower() + if ext in supported_map: + dataloader_name = supported_map[ext] + matched_file = os.path.join(root, f) + break + if dataloader_name: + break + + if dataloader_name is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "ZIP file does not contain any supported dataset files." + "Supported inner files: .csv, .json, .xlsx, .xls" + ), + ) + + if dataloader_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=( + f"Dataloader {dataloader_name} not found in registry." + ), + ) + + dataloader = component_registry[dataloader_name]["class"]() + + if ( + dataloader_name == "CSVDataLoader" + and "separator" not in parsed_params + ): + parsed_params["separator"] = "," + if ( + dataloader_name == "JSONDataLoader" + and "data_key" not in parsed_params + ): + parsed_params["data_key"] = None + + loaded_dataset = dataloader.load_preview( + filepath_or_buffer=matched_file, + params=parsed_params, + n_rows=inference_rows, + ) + + finally: + with contextlib.suppress(Exception): + shutil.rmtree(extract_dir, ignore_errors=True) + + else: + if file.filename.endswith(".csv"): + dataloader_name = "CSVDataLoader" + if "separator" not in parsed_params: + parsed_params["separator"] = "," + + elif file.filename.endswith(".xlsx") or file.filename.endswith(".xls"): + dataloader_name = "ExcelDataLoader" + + elif file.filename.endswith(".json"): + dataloader_name = "JSONDataLoader" + if "data_key" not in parsed_params: + parsed_params["data_key"] = None + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file type. Only CSV, Excel and JSON files are " + "supported." + ), + ) + + if dataloader_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Dataloader {dataloader_name} not found in registry.", + ) + + dataloader = component_registry[dataloader_name]["class"]() + + loaded_dataset = dataloader.load_preview( + filepath_or_buffer=tmp_file_path, + params=parsed_params, + n_rows=inference_rows, + ) + + sample_df = loaded_dataset.head(100) + + table = pa.Table.from_pandas(loaded_dataset) + arrow_schema = arrow_to_dashai_schema(table) + + return { + "sample": sample_df.to_dict(orient="records"), + "schema": arrow_schema, + } + + finally: + if os.path.exists(tmp_file_path): + os.unlink(tmp_file_path) + + except Exception as e: + logger.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load preview: {str(e)}", + ) from e + + +@router.post("/infer_datatypes") +@inject +async def infer_datatypes( + file: UploadFile = File(...), + params: str = Form(None), + component_registry: Dict = Depends(lambda: di["component_registry"]), +): + """ + Infer data types for dataset columns. + This endpoint is for backward compatibility with frontend API calls. + + Parameters + ---------- + file : UploadFile + The file uploaded by the user. + params : str + JSON string with parameters for the dataloader. + component_registry : Dict + Registry of available dataloaders. + + Returns + ------- + Dict + A dictionary containing inferred types for each column. + """ + try: + parsed_params = json.loads(params) if params else {} + + with tempfile.NamedTemporaryFile( + delete=False, suffix=file.filename + ) as tmp_file: + content = await file.read() + tmp_file.write(content) + tmp_file_path = tmp_file.name + + try: + inference_rows = parsed_params.get("inference_rows", 1000) + methods = parsed_params.get("methods", ["DashAIPtype"]) + + if file.filename.endswith(".csv"): + dataloader_name = "CSVDataLoader" + if "separator" not in parsed_params: + parsed_params["separator"] = "," + + elif file.filename.endswith(".xlsx") or file.filename.endswith(".xls"): + dataloader_name = "ExcelDataLoader" + + elif file.filename.endswith(".json"): + dataloader_name = "JSONDataLoader" + if "data_key" not in parsed_params: + parsed_params["data_key"] = None + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Unsupported file type", + ) + + if dataloader_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Dataloader {dataloader_name} not found", + ) + + dataloader = component_registry[dataloader_name]["class"]() + + loaded_dataset = dataloader.load_preview( + filepath_or_buffer=tmp_file_path, + params=parsed_params, + n_rows=inference_rows, + ) + + inferred_types = {} + for method in methods: + method_types = infer_types(loaded_dataset, method=method) + inferred_types.update(method_types) + + return inferred_types + + finally: + if os.path.exists(tmp_file_path): + os.unlink(tmp_file_path) + + except Exception as e: + logger.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to infer datatypes: {str(e)}", + ) from e + + @router.post("/validate_type_changes") @inject async def validate_type_changes( From 9eda843c94288823ca7985331367d530445ae677 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 12:32:16 -0300 Subject: [PATCH 05/88] feat: add error handling for unexpected validation errors and improve output handling in ModelFactory --- DashAI/back/api/api_v1/endpoints/model_sessions.py | 5 +++++ DashAI/back/models/model_factory.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index b698ca892..f2ae97cdd 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -159,6 +159,11 @@ async def validate_columns( except (TypeError, ValueError) as e: validation_response["dataset_status"] = "invalid" validation_response["error"] = str(e) + except Exception as e: + log.exception(f"[VALIDATION] Unexpected error in task {params.task_name}") + validation_response["dataset_status"] = "invalid" + validation_response["error"] = f"Unexpected error: {str(e)}" + return validation_response diff --git a/DashAI/back/models/model_factory.py b/DashAI/back/models/model_factory.py index e8ee27e59..9c204c8d3 100644 --- a/DashAI/back/models/model_factory.py +++ b/DashAI/back/models/model_factory.py @@ -1,4 +1,5 @@ from kink import di +import numpy as np from DashAI.back.metrics.base_metric import BaseMetric from DashAI.back.metrics.classification_metric import ClassificationMetric @@ -217,10 +218,14 @@ def evaluate(self, x, y, metrics): results[split] = split_results continue predictions = self.model.predict(x[split]) - if hasattr(self.model, "prepare_output"): - transformed_y = self.model.prepare_output(y[split]) + # Handle both numpy arrays and DashAIDataset objects + if isinstance(y[split], np.ndarray): + transformed_y = y[split] else: - transformed_y = self.model.prepare_dataset(y[split]) + if hasattr(self.model, "prepare_output"): + transformed_y = self.model.prepare_output(y[split]) + else: + transformed_y = self.model.prepare_dataset(y[split]) for metric in metrics: if ( isinstance(metric, type) From ee77e7bc9ed5d0d54829165beaaa30c2c6832323 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 12:32:32 -0300 Subject: [PATCH 06/88] feat: enhance dataset upload process by creating dataset record before enqueuing job and add validation error handling in TrainNode --- .../src/components/datasets/DatasetModal.jsx | 11 +- .../explorations/ExplorationRunner.jsx | 2 +- .../Steps/ConfigureExplorersStep.jsx | 2 +- .../explorations/explorers/ExplorersTable.jsx | 11 +- .../components/pipelines/nodes/TrainNode.jsx | 59 +++++-- .../results/ResultsPredictionModal.jsx | 148 +++++++++++++++++- 6 files changed, 212 insertions(+), 21 deletions(-) diff --git a/DashAI/front/src/components/datasets/DatasetModal.jsx b/DashAI/front/src/components/datasets/DatasetModal.jsx index 06f931a1a..01b2b0e30 100644 --- a/DashAI/front/src/components/datasets/DatasetModal.jsx +++ b/DashAI/front/src/components/datasets/DatasetModal.jsx @@ -22,7 +22,7 @@ import ConfigureAndUploadDataset from "./ConfigureAndUploadDataset"; import { useSnackbar } from "notistack"; import { enqueueDatasetJob as enqueueDatasetRequest } from "../../api/job"; import DatasetPreviewStep from "./DatasetPreviewStep"; -import { loadPreview } from "../../api/datasets"; +import { loadPreview, createDataset } from "../../api/datasets"; const steps = [ { name: "selectDataloader", label: "Select a way to upload" }, @@ -67,13 +67,18 @@ function DatasetModal({ open, setOpen, updateDatasets }) { ? newDataset.file.name : newDataset.params.name; newDataset.params["dataloader"] = newDataset.dataloader; - await enqueueDatasetRequest(newDataset.file, name, newDataset.url, { + + // First, create the dataset record in the database + const data = await createDataset(name); + + // Then, enqueue the job with the dataset ID + await enqueueDatasetRequest(data.id, newDataset.file, newDataset.url, { ...newDataset.params, schema: columnsSpec, }); enqueueSnackbar("Dataset upload job started", { variant: "success" }); - updateDatasets(); + updateDatasets(data.id); handleCloseDialog(); } catch (error) { console.error(error); diff --git a/DashAI/front/src/components/explorations/ExplorationRunner.jsx b/DashAI/front/src/components/explorations/ExplorationRunner.jsx index 6eb3e8402..08a71d2b6 100644 --- a/DashAI/front/src/components/explorations/ExplorationRunner.jsx +++ b/DashAI/front/src/components/explorations/ExplorationRunner.jsx @@ -3,13 +3,13 @@ import PropTypes from "prop-types"; import { Box, ButtonGroup, Paper, Typography } from "@mui/material"; import { DataGrid, GridToolbar } from "@mui/x-data-grid"; +import LoadingButton from "@mui/lab/LoadingButton"; import { PlayArrow as PlayArrowIcon, Check as CheckIcon, } from "@mui/icons-material"; -import { LoadingButton } from "@mui/material"; import { useSnackbar } from "notistack"; import { getComponents } from "../../api/component"; diff --git a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx index 5de494f3e..d01f1ff52 100644 --- a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx +++ b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx @@ -19,7 +19,7 @@ import { useSnackbar } from "notistack"; import useSchema from "../../../hooks/useSchema"; import { useExplorationsContext } from "../context"; -import ExplorersTable from "../ExplorationsTable"; +import ExplorersTable from "../explorers/ExplorersTable"; import { getComponents } from "../../../api/component"; diff --git a/DashAI/front/src/components/explorations/explorers/ExplorersTable.jsx b/DashAI/front/src/components/explorations/explorers/ExplorersTable.jsx index b929197ee..0017e8df1 100644 --- a/DashAI/front/src/components/explorations/explorers/ExplorersTable.jsx +++ b/DashAI/front/src/components/explorations/explorers/ExplorersTable.jsx @@ -5,7 +5,8 @@ import { DataGrid, GridToolbar } from "@mui/x-data-grid"; import { Grid, Paper, Typography } from "@mui/material"; import DeleteItemModal from "../../custom/DeleteItemModal"; -import { EditParametersDialog, EditColumnsDialog } from ".."; +import EditParametersDialog from "./EditParametersDialog"; +import EditColumnsDialog from "./EditColumnsDialog"; import { useExplorationsContext } from "../context"; /** @@ -70,11 +71,13 @@ function ExplorersTable({ explorerTypes = [] }) { headerName: "Type", minWidth: 200, flex: 1, - valueGetter: (params) => { + valueGetter: (value, row) => { + const actualRow = value && value.row ? value.row : row; + if (!actualRow) return ""; const explorerType = explorerTypes.find( - (explorer) => explorer.type === params.row.exploration_type, + (explorer) => explorer.type === actualRow.exploration_type, ); - return explorerType.label; + return explorerType?.label || actualRow.exploration_type || ""; }, }, { diff --git a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx b/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx index ad7c6a22a..1fac5405c 100644 --- a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx @@ -48,6 +48,7 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { const [availableTasks, setAvailableTasks] = useState([]); const [availableModels, setAvailableModels] = useState([]); const [availableMetrics, setAvailableMetrics] = useState([]); + const [validationErrors, setValidationErrors] = useState({}); const { enqueueSnackbar } = useSnackbar(); const datasetNode = prevNodes?.find((node) => node?.file_path && node?.id); const datasetId = datasetNode?.id ?? null; @@ -140,15 +141,22 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { const handleSave = async () => { const maxValue = datasetInfo?.total_columns; - const parsedInput = inputColumns + const columnNames = datasetInfo?.column_names || []; + + const parsedInputIndices = inputColumns ? parseRangeToIndex(inputColumns, maxValue) : []; - const parsedOutput = outputColumns + const parsedOutputIndices = outputColumns ? parseRangeToIndex(outputColumns, maxValue) : []; + + // Convert indices to column names (1-based to 0-based and then get names) + const inputColumnNames = parsedInputIndices.map(idx => columnNames[idx - 1] || `Column_${idx}`); + const outputColumnNames = parsedOutputIndices.map(idx => columnNames[idx - 1] || `Column_${idx}`); + const payload = { - input_columns: parsedInput, - output_columns: parsedOutput, + input_columns: parsedInputIndices, + output_columns: parsedOutputIndices, splits: { train: splits.train, validation: splits.validation, @@ -221,6 +229,7 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { } const maxValue = datasetInfo?.total_columns; + const columnNames = datasetInfo?.column_names || []; let input = []; let output = []; let parseError = false; @@ -252,24 +261,42 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { return; } + // Convert indices to column names + const inputColumnNames = input.map(idx => columnNames[idx - 1] || `Column_${idx}`); + const outputColumnNames = output.map(idx => columnNames[idx - 1] || `Column_${idx}`); + const results = await Promise.all( availableTasks.map(async (taskObj) => { try { const validation = await validateColumnsRequest( taskObj.name, datasetId, - input, - output, + inputColumnNames, + outputColumnNames, ); - return validation.dataset_status === "valid" ? taskObj.name : null; + if (validation.dataset_status === "valid") { + return { name: taskObj.name, valid: true }; + } else { + return { name: taskObj.name, valid: false, error: validation.error }; + } } catch (error) { console.warn(`Error validating task ${taskObj.name}`, error); - return null; + return { name: taskObj.name, valid: false, error: "Request failed" }; } }), ); - const valid = results.filter(Boolean); + // Extract valid tasks and errors + const validTasksList = results.filter(r => r.valid).map(r => r.name); + const errorsMap = {}; + results.forEach(r => { + if (!r.valid) { + errorsMap[r.name] = r.error; + } + }); + + setValidationErrors(errorsMap); + const valid = validTasksList; setValidTasks(valid); }; @@ -371,6 +398,19 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { Task and Model: + {validTasks.length === 0 && Object.keys(validationErrors).length > 0 && ( + + + ⚠️ No compatible tasks available. Validation errors: + + {Object.entries(validationErrors).map(([taskName, error]) => ( + + {taskName}: {error} + + ))} + + )} + { setModelParams({}); }} margin="normal" + disabled={validTasks.length === 0} > {availableTasks.map((taskObj) => ( )} - {tab === 1 && summary.data_type !== "string" && ( - <>Prediction Summary Tab Placeholder + {tab === 1 && summary.data_type === "regression" && ( + + + Regression Statistics + + + + + + + Mean + + + {summary.statistics?.mean?.toFixed(2) ?? "-"} + + + + + + + + + Median + + + {summary.statistics?.median?.toFixed(2) ?? "-"} + + + + + + + + + Std Deviation + + + {summary.statistics?.std?.toFixed(2) ?? "-"} + + + + + + + + + Minimum + + + {summary.statistics?.min?.toFixed(2) ?? "-"} + + + + + + + + + Maximum + + + {summary.statistics?.max?.toFixed(2) ?? "-"} + + + + + + + + + Total Predictions + + + {summary.total_data_points ?? "-"} + + + + + + + )} + {tab === 1 && summary.data_type === "classification" && ( + + + Class Distribution + + + + + + Class + Occurrences + Percentage + + + + {summary.class_distribution?.map((row) => ( + + {row.Class} + {row.Ocurrences} + {row.Percentage}% + + ))} + +
+ + + )} + {tab === 2 && ( + + + Sample Predictions (First 50) + + + + + + Index + Predicted Value + + + + {summary.sample_data?.map((row) => ( + + {row.id} + + {typeof row.value === "number" + ? row.value.toFixed(2) + : row.value} + + + ))} + +
+ + )} - {tab === 2 && <>Prediction Sample Table Placeholder} From c4038f791e79f77b1c3462de31c4ed829e1d0b94 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 14:05:17 -0300 Subject: [PATCH 07/88] refactor: apply pre-commit formatting --- .../api/api_v1/endpoints/model_sessions.py | 2 +- DashAI/back/api/api_v1/endpoints/pipelines.py | 18 +++++++++++++----- DashAI/back/job/pipeline_job.py | 7 +++---- DashAI/back/metrics/regression_metric.py | 1 - DashAI/back/models/model_factory.py | 2 +- DashAI/back/pipeline/exploration_node.py | 9 ++++++--- DashAI/back/pipeline/prediction_node.py | 8 +++++--- DashAI/back/pipeline/train_node.py | 10 ++++++---- .../src/components/datasets/DatasetModal.jsx | 4 ++-- 9 files changed, 37 insertions(+), 24 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index f2ae97cdd..6a9477a36 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -163,7 +163,7 @@ async def validate_columns( log.exception(f"[VALIDATION] Unexpected error in task {params.task_name}") validation_response["dataset_status"] = "invalid" validation_response["error"] = f"Unexpected error: {str(e)}" - + return validation_response diff --git a/DashAI/back/api/api_v1/endpoints/pipelines.py b/DashAI/back/api/api_v1/endpoints/pipelines.py index 228f3d2e1..7e6a0f589 100644 --- a/DashAI/back/api/api_v1/endpoints/pipelines.py +++ b/DashAI/back/api/api_v1/endpoints/pipelines.py @@ -123,15 +123,19 @@ class distribution (if applicable), and sample data. if isinstance(data[0], str): summary["data_type"] = "string" else: - # Check if data looks like classification (integers) or regression (floats) - is_classification = all(isinstance(x, int) or (isinstance(x, float) and x.is_integer()) for x in data) + # Check if data looks like integers or floats + is_classification = all( + isinstance(x, int) or (isinstance(x, float) and x.is_integer()) + for x in data + ) unique_values = len(set(data)) - - # If there are many unique values relative to total, it's likely regression + + # If there are many unique values relative to total -> regression if not is_classification or unique_values > len(data) * 0.5: # Regression: provide statistics summary["data_type"] = "regression" import numpy as np + data_array = np.array(data) summary["statistics"] = { "mean": float(np.mean(data_array)), @@ -150,7 +154,11 @@ class distribution (if applicable), and sample data. id = 1 for class_name in classes: # Count occurrences of the actual value (int or float) - original_value = int(float(class_name)) if '.' not in class_name else float(class_name) + original_value = ( + int(float(class_name)) + if "." not in class_name + else float(class_name) + ) occurrences = data.count(original_value) distribution = { "id": id, diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index dc0fe6c92..27d7e89fa 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -70,7 +70,8 @@ async def _execute_steps() -> None: node_config = step.get("config", {}) log.debug( - f"Pipeline {pipeline_id}: Executing step {idx + 1}/{len(steps)} - " + f"Pipeline {pipeline_id}: " + f"Executing step {idx + 1}/{len(steps)} - " f"{node_type} ({node_id})" ) @@ -96,9 +97,7 @@ async def _execute_steps() -> None: ) log.debug(f"Node {node_id} executed successfully.") except Exception as e: - error_msg = ( - f"Error in node {node_id} ({node_type}): {str(e)}" - ) + error_msg = f"Error in node {node_id} ({node_type}): {str(e)}" log.exception(error_msg) raise JobError(error_msg) from e diff --git a/DashAI/back/metrics/regression_metric.py b/DashAI/back/metrics/regression_metric.py index ce77cae80..6102a5474 100644 --- a/DashAI/back/metrics/regression_metric.py +++ b/DashAI/back/metrics/regression_metric.py @@ -30,7 +30,6 @@ def prepare_to_metric( tuple[np.ndarray, np.ndarray] Tuple containing true values and predicted values as numpy arrays. """ - #true_values = np.array(y.to_pandas().to_numpy().flatten()) # Handle both DashAIDataset and numpy array inputs if isinstance(y, np.ndarray): true_values = np.array(y).flatten() diff --git a/DashAI/back/models/model_factory.py b/DashAI/back/models/model_factory.py index 9c204c8d3..3ba35da45 100644 --- a/DashAI/back/models/model_factory.py +++ b/DashAI/back/models/model_factory.py @@ -1,5 +1,5 @@ -from kink import di import numpy as np +from kink import di from DashAI.back.metrics.base_metric import BaseMetric from DashAI.back.metrics.classification_metric import ClassificationMetric diff --git a/DashAI/back/pipeline/exploration_node.py b/DashAI/back/pipeline/exploration_node.py index cc9627bc4..f712e1021 100644 --- a/DashAI/back/pipeline/exploration_node.py +++ b/DashAI/back/pipeline/exploration_node.py @@ -89,15 +89,18 @@ async def run( ) # Create a simple explorer info object for pipeline execution - # This is not a database object, just a container for explorer metadata + # This is not a database object, + # just a container for explorer metadata class ExplorerInfo: - def __init__(self, exploration_type, columns, parameters, id, name): + def __init__( + self, exploration_type, columns, parameters, id, name + ): self.exploration_type = exploration_type self.columns = columns self.parameters = parameters self.id = id self.name = name - + explorer_info = ExplorerInfo( exploration_type=exploration_type, columns=columns, diff --git a/DashAI/back/pipeline/prediction_node.py b/DashAI/back/pipeline/prediction_node.py index 9534c10c4..82a06223b 100644 --- a/DashAI/back/pipeline/prediction_node.py +++ b/DashAI/back/pipeline/prediction_node.py @@ -46,13 +46,15 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: try: prepared_dataset = loaded_dataset.select_columns(context["input_columns"]) y_pred_proba = np.array(trained_model.predict(prepared_dataset)) - - # Check if it's classification (2D array with probabilities) or regression (1D array) + + # Check if it's classification (2D array with probabilities) + # or regression (1D array) if isinstance(y_pred_proba[0], str): # Text/categorical predictions y_pred = y_pred_proba elif y_pred_proba.ndim == 2: - # Classification: multi-class probabilities, get class with highest probability + # Classification: multi-class probabilities, + # get class with highest probability y_pred = np.argmax(y_pred_proba, axis=1) else: # Regression: predictions are already the final values diff --git a/DashAI/back/pipeline/train_node.py b/DashAI/back/pipeline/train_node.py index e961537e7..44ad9ce73 100644 --- a/DashAI/back/pipeline/train_node.py +++ b/DashAI/back/pipeline/train_node.py @@ -168,7 +168,7 @@ async def run( try: # Check if model uses .train() (text models) or .fit() (sklearn models) - if hasattr(model, "train") and callable(getattr(model, "train")): + if hasattr(model, "train") and callable(model.train): # Text models expect Dataset objects, not numpy arrays model.train( x_splits["train"], @@ -179,7 +179,7 @@ async def run( # Keep splits as DashAIDataset for evaluation x = x_splits y = y_splits - elif hasattr(model, "fit") and callable(getattr(model, "fit")): + elif hasattr(model, "fit") and callable(model.fit): # Sklearn models expect numpy arrays # Convert DashAIDataset to numpy arrays x = {} @@ -189,10 +189,12 @@ async def run( x[split] = x_splits[split].to_pandas().values if split in y_splits: y[split] = y_splits[split].to_pandas().values.ravel() - + model.fit(x["train"], y["train"]) else: - raise AttributeError(f"Model {model_class} has neither 'train' nor 'fit' method") + raise AttributeError( + f"Model {model_class} has neither 'train' nor 'fit' method" + ) except Exception as e: log.exception(e) raise JobError( diff --git a/DashAI/front/src/components/datasets/DatasetModal.jsx b/DashAI/front/src/components/datasets/DatasetModal.jsx index 01b2b0e30..4ccdc2cc1 100644 --- a/DashAI/front/src/components/datasets/DatasetModal.jsx +++ b/DashAI/front/src/components/datasets/DatasetModal.jsx @@ -67,10 +67,10 @@ function DatasetModal({ open, setOpen, updateDatasets }) { ? newDataset.file.name : newDataset.params.name; newDataset.params["dataloader"] = newDataset.dataloader; - + // First, create the dataset record in the database const data = await createDataset(name); - + // Then, enqueue the job with the dataset ID await enqueueDatasetRequest(data.id, newDataset.file, newDataset.url, { ...newDataset.params, From a4b02d726053ab2e338746cd3d3e38c54226c1d0 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 14:21:21 -0300 Subject: [PATCH 08/88] refactor: remove unnecessary whitespace in TrainNode component --- DashAI/front/src/components/pipelines/nodes/TrainNode.jsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx b/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx index 1fac5405c..ec78cadec 100644 --- a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx @@ -142,18 +142,18 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { const handleSave = async () => { const maxValue = datasetInfo?.total_columns; const columnNames = datasetInfo?.column_names || []; - + const parsedInputIndices = inputColumns ? parseRangeToIndex(inputColumns, maxValue) : []; const parsedOutputIndices = outputColumns ? parseRangeToIndex(outputColumns, maxValue) : []; - + // Convert indices to column names (1-based to 0-based and then get names) const inputColumnNames = parsedInputIndices.map(idx => columnNames[idx - 1] || `Column_${idx}`); const outputColumnNames = parsedOutputIndices.map(idx => columnNames[idx - 1] || `Column_${idx}`); - + const payload = { input_columns: parsedInputIndices, output_columns: parsedOutputIndices, @@ -294,7 +294,7 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { errorsMap[r.name] = r.error; } }); - + setValidationErrors(errorsMap); const valid = validTasksList; setValidTasks(valid); From c571c61a410bd653d7dc4cc1cb4de2799e3d6423 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 14:28:42 -0300 Subject: [PATCH 09/88] refactor: apply prettier formatting --- .../components/pipelines/nodes/TrainNode.jsx | 62 +++++++++++++------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx b/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx index ec78cadec..2ca1c0b4e 100644 --- a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx @@ -151,8 +151,12 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { : []; // Convert indices to column names (1-based to 0-based and then get names) - const inputColumnNames = parsedInputIndices.map(idx => columnNames[idx - 1] || `Column_${idx}`); - const outputColumnNames = parsedOutputIndices.map(idx => columnNames[idx - 1] || `Column_${idx}`); + const inputColumnNames = parsedInputIndices.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + const outputColumnNames = parsedOutputIndices.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); const payload = { input_columns: parsedInputIndices, @@ -262,8 +266,12 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { } // Convert indices to column names - const inputColumnNames = input.map(idx => columnNames[idx - 1] || `Column_${idx}`); - const outputColumnNames = output.map(idx => columnNames[idx - 1] || `Column_${idx}`); + const inputColumnNames = input.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + const outputColumnNames = output.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); const results = await Promise.all( availableTasks.map(async (taskObj) => { @@ -277,19 +285,27 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { if (validation.dataset_status === "valid") { return { name: taskObj.name, valid: true }; } else { - return { name: taskObj.name, valid: false, error: validation.error }; + return { + name: taskObj.name, + valid: false, + error: validation.error, + }; } } catch (error) { console.warn(`Error validating task ${taskObj.name}`, error); - return { name: taskObj.name, valid: false, error: "Request failed" }; + return { + name: taskObj.name, + valid: false, + error: "Request failed", + }; } }), ); // Extract valid tasks and errors - const validTasksList = results.filter(r => r.valid).map(r => r.name); + const validTasksList = results.filter((r) => r.valid).map((r) => r.name); const errorsMap = {}; - results.forEach(r => { + results.forEach((r) => { if (!r.valid) { errorsMap[r.name] = r.error; } @@ -398,18 +414,26 @@ const Train = ({ open, onClose, onSave, savedConfig, prevNodes }) => { Task and Model: - {validTasks.length === 0 && Object.keys(validationErrors).length > 0 && ( - - - ⚠️ No compatible tasks available. Validation errors: - - {Object.entries(validationErrors).map(([taskName, error]) => ( - - {taskName}: {error} + {validTasks.length === 0 && + Object.keys(validationErrors).length > 0 && ( + + + ⚠️ No compatible tasks available. Validation errors: - ))} - - )} + {Object.entries(validationErrors).map(([taskName, error]) => ( + + {taskName}: {error} + + ))} + + )} Date: Mon, 16 Feb 2026 16:06:08 -0300 Subject: [PATCH 10/88] fix: update import from DPTFeatureExtractor to DPTImageProcessor in depth controlnet model --- .../hugging_face/stable_diffusion_v1_depth_controlnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DashAI/back/models/hugging_face/stable_diffusion_v1_depth_controlnet.py b/DashAI/back/models/hugging_face/stable_diffusion_v1_depth_controlnet.py index f411d05ae..e1b59ec82 100644 --- a/DashAI/back/models/hugging_face/stable_diffusion_v1_depth_controlnet.py +++ b/DashAI/back/models/hugging_face/stable_diffusion_v1_depth_controlnet.py @@ -8,7 +8,7 @@ StableDiffusionXLControlNetPipeline, ) from PIL import Image -from transformers import DPTFeatureExtractor, DPTForDepthEstimation +from transformers import DPTImageProcessor, DPTForDepthEstimation from DashAI.back.core.schema_fields import ( enum_field, @@ -51,7 +51,7 @@ def get_depth_map(image, device): depth_estimator = DPTForDepthEstimation.from_pretrained( "Intel/dpt-hybrid-midas" ).to(device) - feature_extractor = DPTFeatureExtractor.from_pretrained( + feature_extractor = DPTImageProcessor.from_pretrained( "Intel/dpt-hybrid-midas", device=device ) From 934089ca8737ea223dc9ecd0d5c37806d5fce4e6 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 16 Feb 2026 16:15:27 -0300 Subject: [PATCH 11/88] fix: reorder import statements for consistency in stable_diffusion_v1_depth_controlnet.py --- .../models/hugging_face/stable_diffusion_v1_depth_controlnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DashAI/back/models/hugging_face/stable_diffusion_v1_depth_controlnet.py b/DashAI/back/models/hugging_face/stable_diffusion_v1_depth_controlnet.py index e1b59ec82..5298e73cf 100644 --- a/DashAI/back/models/hugging_face/stable_diffusion_v1_depth_controlnet.py +++ b/DashAI/back/models/hugging_face/stable_diffusion_v1_depth_controlnet.py @@ -8,7 +8,7 @@ StableDiffusionXLControlNetPipeline, ) from PIL import Image -from transformers import DPTImageProcessor, DPTForDepthEstimation +from transformers import DPTForDepthEstimation, DPTImageProcessor from DashAI.back.core.schema_fields import ( enum_field, From 1fc93dfa4b9bcdfbdd827406b28d9f0071bc30ac Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 22 Feb 2026 19:22:55 -0300 Subject: [PATCH 12/88] feat: add logging parameters for training and validation in save() and load() methods in DistilBertTransformer --- DashAI/back/models/hugging_face/distilbert_transformer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DashAI/back/models/hugging_face/distilbert_transformer.py b/DashAI/back/models/hugging_face/distilbert_transformer.py index 1d5447b6c..dc4f03084 100644 --- a/DashAI/back/models/hugging_face/distilbert_transformer.py +++ b/DashAI/back/models/hugging_face/distilbert_transformer.py @@ -449,6 +449,10 @@ def save(self, filename: Union[str, Path]) -> None: "weight_decay": self.training_args_params.get("weight_decay"), "num_labels": self.num_labels, "fitted": self.fitted, + "log_train_every_n_epochs": self.log_train_every_n_epochs, + "log_train_every_n_steps": self.log_train_every_n_steps, + "log_validation_every_n_epochs": self.log_validation_every_n_epochs, + "log_validation_every_n_steps": self.log_validation_every_n_steps, } config.save_pretrained(filename) @@ -470,6 +474,10 @@ def load(cls, filename: Union[str, Path]) -> Any: learning_rate=custom_params.get("learning_rate"), device=custom_params.get("device"), weight_decay=custom_params.get("weight_decay"), + log_train_every_n_epochs=custom_params.get("log_train_every_n_epochs"), + log_train_every_n_steps=custom_params.get("log_train_every_n_steps"), + log_validation_every_n_epochs=custom_params.get("log_validation_every_n_epochs"), + log_validation_every_n_steps=custom_params.get("log_validation_every_n_steps"), ) loaded_model.fitted = custom_params.get("fitted") From e68dc0dd4c9ccb8cb9c3a46042f42c3a88a6da50 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 22 Feb 2026 19:28:28 -0300 Subject: [PATCH 13/88] fix: improve formatting of logging parameters in DistilBertTransformer for load() method --- DashAI/back/models/hugging_face/distilbert_transformer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/DashAI/back/models/hugging_face/distilbert_transformer.py b/DashAI/back/models/hugging_face/distilbert_transformer.py index dc4f03084..6c27aac2d 100644 --- a/DashAI/back/models/hugging_face/distilbert_transformer.py +++ b/DashAI/back/models/hugging_face/distilbert_transformer.py @@ -476,8 +476,12 @@ def load(cls, filename: Union[str, Path]) -> Any: weight_decay=custom_params.get("weight_decay"), log_train_every_n_epochs=custom_params.get("log_train_every_n_epochs"), log_train_every_n_steps=custom_params.get("log_train_every_n_steps"), - log_validation_every_n_epochs=custom_params.get("log_validation_every_n_epochs"), - log_validation_every_n_steps=custom_params.get("log_validation_every_n_steps"), + log_validation_every_n_epochs=custom_params.get( + "log_validation_every_n_epochs" + ), + log_validation_every_n_steps=custom_params.get( + "log_validation_every_n_steps" + ), ) loaded_model.fitted = custom_params.get("fitted") From 9a431c90b6b3053c09051f3456baad3633c2d914 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 22 Feb 2026 19:53:58 -0300 Subject: [PATCH 14/88] feat: add setuptools to requirements --- requirements-cpu.txt | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-cpu.txt b/requirements-cpu.txt index 8658a63fd..a9a8a3944 100644 --- a/requirements-cpu.txt +++ b/requirements-cpu.txt @@ -47,6 +47,7 @@ sentencepiece # Optimization / hyperparam tuning optuna +setuptools hyperopt # Excel / IO / Utils diff --git a/requirements.txt b/requirements.txt index 598a66848..c513fda81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ transformers<5.0.0 sacrebleu sentencepiece optuna +setuptools hyperopt openpyxl httpx From 6ff34284d3d959ba9ab266fc7a032dceb0fe0e97 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 22 Feb 2026 20:12:17 -0300 Subject: [PATCH 15/88] fix: ensure setuptools is reinstalled for hyperopt compatibility --- .github/workflows/build-test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 1bf78ec75..6c7f8f13f 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -54,6 +54,8 @@ jobs: run: python -m pip install --upgrade pip setuptools wheel - run: python -m pip install --upgrade --upgrade-strategy eager -r requirements.txt - run: python -m pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt + - name: Ensure setuptools (provides pkg_resources for hyperopt) + run: python -m pip install --force-reinstall setuptools - name: Prepare frontend build run: mkdir -p DashAI/front/build @@ -63,4 +65,4 @@ jobs: name: react-build path: DashAI/front/build - name: Test with pytest - run: pytest -v + run: pytest -v \ No newline at end of file From 96f28a0405760f177649170a92c01182dbd78ccb Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 22 Feb 2026 20:20:49 -0300 Subject: [PATCH 16/88] fix: pre-commit modification for build-test.yaml --- .github/workflows/build-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 6c7f8f13f..06f581f07 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -65,4 +65,4 @@ jobs: name: react-build path: DashAI/front/build - name: Test with pytest - run: pytest -v \ No newline at end of file + run: pytest -v From bbead4dae9b2be447a44e37f1cdfc8693a6b9a1f Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 22 Feb 2026 20:59:15 -0300 Subject: [PATCH 17/88] fix: remove setuptools from requirements files and change imports in hyperopt_optimizer.py --- DashAI/back/optimizers/hyperopt_optimizer.py | 6 ++++-- requirements-cpu.txt | 1 - requirements.txt | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DashAI/back/optimizers/hyperopt_optimizer.py b/DashAI/back/optimizers/hyperopt_optimizer.py index 075141adc..58417aa84 100644 --- a/DashAI/back/optimizers/hyperopt_optimizer.py +++ b/DashAI/back/optimizers/hyperopt_optimizer.py @@ -1,7 +1,5 @@ import importlib -from hyperopt import Trials, fmin, hp, rand, tpe # noqa: F401 - from DashAI.back.core.enums.metrics import LevelEnum, SplitEnum from DashAI.back.core.schema_fields import ( BaseSchema, @@ -77,6 +75,8 @@ def search_space(self, hyperparams_data): ------- search_space: Dict with the information for the search space . """ + from hyperopt import hp + search_space = {} for _, hyperparameter, values, dtype in hyperparams_data: @@ -113,6 +113,8 @@ def optimize(self, model, input_dataset, output_dataset, parameters, metric, tas self.parameters = parameters self.metric = metric["class"] + from hyperopt import Trials, fmin + param_mapping = {key: (obj, key) for obj, key, _, _ in self.parameters} search_space = self.search_space(self.parameters) diff --git a/requirements-cpu.txt b/requirements-cpu.txt index a9a8a3944..8658a63fd 100644 --- a/requirements-cpu.txt +++ b/requirements-cpu.txt @@ -47,7 +47,6 @@ sentencepiece # Optimization / hyperparam tuning optuna -setuptools hyperopt # Excel / IO / Utils diff --git a/requirements.txt b/requirements.txt index c513fda81..598a66848 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,6 @@ transformers<5.0.0 sacrebleu sentencepiece optuna -setuptools hyperopt openpyxl httpx From b5c8204bf5bc11d7312b4eba9f87b2b20cd2c95d Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 23 Feb 2026 20:22:42 -0300 Subject: [PATCH 18/88] feat: implement light and dark canvas modes across pipeline components --- .../src/components/pipelines/CustomNode.jsx | 55 ++++++++++++++----- .../components/pipelines/PipelineDesigner.jsx | 30 ++++++++-- .../components/pipelines/PipelineToolbar.jsx | 52 +++++++++++++++--- .../front/src/pages/pipelines/NewPipeline.jsx | 10 +++- 4 files changed, 120 insertions(+), 27 deletions(-) diff --git a/DashAI/front/src/components/pipelines/CustomNode.jsx b/DashAI/front/src/components/pipelines/CustomNode.jsx index d49542e0e..bd67aab36 100644 --- a/DashAI/front/src/components/pipelines/CustomNode.jsx +++ b/DashAI/front/src/components/pipelines/CustomNode.jsx @@ -21,19 +21,46 @@ const CustomNode = ({ data, isConnectable }) => { const theme = useTheme(); const [hovered, setHovered] = useState(false); + const isLightCanvas = data.canvasMode === "light"; + const IconComponent = iconMap[data.icon] || SettingsIcon; const isDisabled = data.errors?.some((err) => err.includes("already exists")) ?? false; - const borderColor = - data.notConfigured && !isDisabled + + // Light canvas mode: white nodes, yellow borders, black text + // Dark canvas mode: theme-aware colors + const borderColor = isLightCanvas + ? data.notConfigured && !isDisabled + ? "2px solid #f9a825" + : "1px solid #f9a825" + : data.notConfigured && !isDisabled ? `2px solid ${theme.palette.warning.main}` : `1px solid ${theme.palette.ui.borderLight}`; - const iconColor = isDisabled - ? theme.palette.text.secondary - : theme.palette.text.primary; - const bgColor = isDisabled - ? theme.palette.ui.panelMedium - : theme.palette.background.paper; + + const iconColor = isLightCanvas + ? isDisabled + ? "#999999" + : "#333333" + : isDisabled + ? theme.palette.text.secondary + : theme.palette.text.primary; + + const bgColor = isLightCanvas + ? isDisabled + ? "#f5f5f5" + : "#ffffff" + : isDisabled + ? theme.palette.ui.panelMedium + : theme.palette.background.paper; + + const textColor = isLightCanvas ? "#1a1a1a" : theme.palette.text.primary; + const secondaryTextColor = isLightCanvas + ? "#666666" + : theme.palette.text.secondary; + const handleColor = isLightCanvas ? "#333333" : theme.palette.text.primary; + const handleDisabledColor = isLightCanvas + ? "#cccccc" + : theme.palette.ui.border; const nodeContent = ( { }} > )} @@ -82,10 +109,10 @@ const CustomNode = ({ data, isConnectable }) => { position={Position.Left} style={{ background: isDisabled - ? theme.palette.ui.border + ? handleDisabledColor : data.hasError ? theme.palette.error.main - : theme.palette.text.primary, + : handleColor, width: 8, height: 8, borderRadius: "50%", @@ -102,8 +129,8 @@ const CustomNode = ({ data, isConnectable }) => { position={Position.Right} style={{ background: isDisabled - ? theme.palette.ui.border - : theme.palette.text.primary, + ? handleDisabledColor + : handleColor, width: 8, height: 8, borderRadius: "50%", @@ -123,7 +150,7 @@ const CustomNode = ({ data, isConnectable }) => { }} > {data.name || data.label} diff --git a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx index 37908a0d9..35735edba 100644 --- a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx +++ b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx @@ -1,5 +1,6 @@ -import React, { useCallback } from "react"; +import React, { useCallback, useEffect } from "react"; import { Box, Tooltip } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; import ReactFlow, { addEdge, Background, @@ -30,8 +31,27 @@ function PipelineDesigner({ nodeIdCounter, setNodeIdCounter, availableNodes, + canvasMode, }) { const { screenToFlowPosition } = useReactFlow(); + const theme = useTheme(); + + const isLightCanvas = canvasMode === "light"; + const canvasBg = isLightCanvas ? "#ffffff" : theme.palette.background.box; + const canvasBorder = isLightCanvas + ? "1px solid #e0e0e0" + : `1px solid ${theme.palette.ui.borderLight}`; + const gridColor = isLightCanvas ? "#e0e0e0" : theme.palette.ui.border; + + // Sync canvasMode into existing nodes so CustomNode can read it + useEffect(() => { + setNodes((nds) => + nds.map((n) => ({ + ...n, + data: { ...n.data, canvasMode }, + })), + ); + }, [canvasMode, setNodes]); const onConnect = (params) => { setEdges((eds) => @@ -81,6 +101,7 @@ function PipelineDesigner({ errors: nodeErrors, source: nodeInfo?.source || false, target: nodeInfo?.target || false, + canvasMode, onDelete: () => { setNodes((nds) => nds.filter((n) => n.id !== nodeId)); setNodeData((prev) => { @@ -101,6 +122,7 @@ function PipelineDesigner({ setNodeIdCounter((prev) => prev + 1); }, [ + canvasMode, dragging, setNodes, setNodeData, @@ -134,8 +156,8 @@ function PipelineDesigner({ width: "100%", height: "100%", borderRadius: 12, - background: "#f9f9f9", - border: "1px solid #ccc", + background: canvasBg, + border: canvasBorder, boxShadow: "0 2px 6px rgba(0,0,0,0.1)", }} > @@ -164,7 +186,7 @@ function PipelineDesigner({ )} - + ); diff --git a/DashAI/front/src/components/pipelines/PipelineToolbar.jsx b/DashAI/front/src/components/pipelines/PipelineToolbar.jsx index 2a3fe8d9a..c8813f479 100644 --- a/DashAI/front/src/components/pipelines/PipelineToolbar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineToolbar.jsx @@ -1,5 +1,8 @@ import React from "react"; -import { Box, TextField, Button } from "@mui/material"; +import { Box, TextField, Button, IconButton, Tooltip } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; +import LightModeIcon from "@mui/icons-material/LightMode"; +import DarkModeIcon from "@mui/icons-material/DarkMode"; function PipelineToolbar({ pipelineName, @@ -8,7 +11,15 @@ function PipelineToolbar({ nameError, nameErrorMessage, handlePipelineNameChange, + canvasMode, + setCanvasMode, }) { + const theme = useTheme(); + + const toggleCanvasMode = () => { + setCanvasMode((prev) => (prev === "dark" ? "light" : "dark")); + }; + return ( - + + + + {canvasMode === "dark" ? ( + + ) : ( + + )} + + + + ); } diff --git a/DashAI/front/src/pages/pipelines/NewPipeline.jsx b/DashAI/front/src/pages/pipelines/NewPipeline.jsx index e4cf65f38..bb8d95607 100644 --- a/DashAI/front/src/pages/pipelines/NewPipeline.jsx +++ b/DashAI/front/src/pages/pipelines/NewPipeline.jsx @@ -1,4 +1,4 @@ -import React, { useRef } from "react"; +import React, { useRef, useState } from "react"; import { Box, Typography, @@ -6,6 +6,7 @@ import { DialogTitle, IconButton, } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; import CloseIcon from "@mui/icons-material/Close"; import { ReactFlowProvider } from "reactflow"; import CustomLayout from "../../components/custom/CustomLayout"; @@ -26,6 +27,8 @@ function NewPipeline() { const { pipelineId } = useParams(); const navigate = useNavigate(); const flowWrapperRef = useRef(null); + const theme = useTheme(); + const [canvasMode, setCanvasMode] = useState("dark"); const { // State @@ -125,7 +128,7 @@ function NewPipeline() { nodeHelp={nodeHelp} /> - + From 6d84520e3b7edb0a4abfcab6d7dbd5587465687d Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 23 Feb 2026 20:38:56 -0300 Subject: [PATCH 19/88] fix: prettier pre-commit modification for NewPipeline.jsx and CustomNode.jsx --- DashAI/front/src/components/pipelines/CustomNode.jsx | 12 +++--------- DashAI/front/src/pages/pipelines/NewPipeline.jsx | 8 +++++++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/DashAI/front/src/components/pipelines/CustomNode.jsx b/DashAI/front/src/components/pipelines/CustomNode.jsx index bd67aab36..cec6e0ac4 100644 --- a/DashAI/front/src/components/pipelines/CustomNode.jsx +++ b/DashAI/front/src/components/pipelines/CustomNode.jsx @@ -97,9 +97,7 @@ const CustomNode = ({ data, isConnectable }) => { }, }} > - + )} @@ -128,9 +126,7 @@ const CustomNode = ({ data, isConnectable }) => { type="source" position={Position.Right} style={{ - background: isDisabled - ? handleDisabledColor - : handleColor, + background: isDisabled ? handleDisabledColor : handleColor, width: 8, height: 8, borderRadius: "50%", @@ -149,9 +145,7 @@ const CustomNode = ({ data, isConnectable }) => { alignItems: "center", }} > - + {data.name || data.label} diff --git a/DashAI/front/src/pages/pipelines/NewPipeline.jsx b/DashAI/front/src/pages/pipelines/NewPipeline.jsx index bb8d95607..7dc1531fd 100644 --- a/DashAI/front/src/pages/pipelines/NewPipeline.jsx +++ b/DashAI/front/src/pages/pipelines/NewPipeline.jsx @@ -128,7 +128,13 @@ function NewPipeline() { nodeHelp={nodeHelp} /> - + Date: Mon, 23 Feb 2026 23:27:06 -0300 Subject: [PATCH 20/88] feat: add polling for dataset processing and improve result fetching in DataSelectorNode and PipelineResults components --- .../pipelines/nodes/DataSelectorNode.jsx | 66 +++++++++++++++++-- .../components/pipelines/results/Results.jsx | 61 ++++++++++++++++- 2 files changed, 117 insertions(+), 10 deletions(-) diff --git a/DashAI/front/src/components/pipelines/nodes/DataSelectorNode.jsx b/DashAI/front/src/components/pipelines/nodes/DataSelectorNode.jsx index a5d15e594..bec52cd46 100644 --- a/DashAI/front/src/components/pipelines/nodes/DataSelectorNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/DataSelectorNode.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useRef } from "react"; import PropTypes from "prop-types"; import DatasetModal from "../../../components/datasets/DatasetModal"; import { validateNode } from "../../../api/pipeline"; @@ -15,22 +15,49 @@ function DataSelectorNode({ onClose, onSave, savedConfig }) { const [loading, setLoading] = useState(true); const { enqueueSnackbar } = useSnackbar(); const [validationStatus, setValidationStatus] = useState(""); + const pollingRef = useRef(null); const fetchDatasets = async () => { setLoading(true); const res = await getDatasets(); setDatasets(res); setLoading(false); + return res; }; useEffect(() => { fetchDatasets(); + return () => { + if (pollingRef.current) { + clearInterval(pollingRef.current); + } + }; }, []); const handleDatasetUpdate = async (newDatasetId) => { - await fetchDatasets(); + const latestDatasets = await fetchDatasets(); if (newDatasetId) { setDatasetId(newDatasetId); + + // Check if the dataset is still processing (file_path not yet set) + const newDs = latestDatasets.find((d) => d.id === newDatasetId); + if (!newDs || !newDs.file_path) { + // Poll every 2 s until the background job sets file_path + if (pollingRef.current) clearInterval(pollingRef.current); + pollingRef.current = setInterval(async () => { + try { + const res = await getDatasets(); + setDatasets(res); + const updated = res.find((d) => d.id === newDatasetId); + if (updated && updated.file_path) { + clearInterval(pollingRef.current); + pollingRef.current = null; + } + } catch (err) { + console.error("Error polling datasets:", err); + } + }, 2000); + } } }; @@ -38,9 +65,26 @@ function DataSelectorNode({ onClose, onSave, savedConfig }) { const handleCloseModal = () => setOpenModal(false); const handleSave = async () => { - const selected = datasets.find((d) => d.id === datasetId); + // Re-fetch to obtain the latest file_path (set by the background job) + let latestDatasets; + try { + latestDatasets = await getDatasets(); + setDatasets(latestDatasets); + } catch { + latestDatasets = datasets; + } + + const selected = latestDatasets.find((d) => d.id === datasetId); if (!selected) { - console.error("Selected dataset not found."); + enqueueSnackbar("Please select a dataset first.", { variant: "warning" }); + return; + } + + if (!selected.file_path) { + enqueueSnackbar( + "The dataset is still being processed. Please wait a moment and try again.", + { variant: "warning" }, + ); return; } @@ -57,7 +101,9 @@ function DataSelectorNode({ onClose, onSave, savedConfig }) { onClose(); } else { setValidationStatus("error"); - enqueueSnackbar("Validation failed", { variant: "error" }); + enqueueSnackbar(response.message || "Validation failed", { + variant: "error", + }); } } catch (e) { setValidationStatus("error"); @@ -119,10 +165,16 @@ function DataSelectorNode({ onClose, onSave, savedConfig }) { diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 5091a10fa..0dc639f33 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from "react"; +import React, { useEffect, useState, useRef, useCallback } from "react"; import { Box, Typography, @@ -9,6 +9,7 @@ import { Tab, Paper, Button, + CircularProgress, } from "@mui/material"; import { ExpandMore, ArrowBackIosNew } from "@mui/icons-material"; import { getPipelineById } from "../../../api/pipeline"; @@ -18,15 +19,52 @@ import PipelineResultsPrediction from "./ResultsPrediction"; import ResultsTabParameters from "../../../pages/results/components/ResultsTabParameters"; import ResultsExploration from "./ResultsExploration"; +const POLL_INTERVAL_MS = 3000; + function PipelineResults({ pipelineId, onClose }) { const [results, setResults] = useState(null); const [trainTab, setTrainTab] = useState(0); + const [polling, setPolling] = useState(false); + const pollingRef = useRef(null); + + /** Returns true when the pipeline has at least one result section populated. */ + const hasAnyResult = useCallback((data) => { + if (!data) return false; + const hasExpl = data.exploration && data.exploration !== "No exploration data"; + const hasTr = data.train && Object.keys(data.train).length > 0; + const hasPred = data.prediction && data.prediction !== "No prediction data"; + return hasExpl || hasTr || hasPred; + }, []); useEffect(() => { + let cancelled = false; + const fetchResults = async () => { try { const response = await getPipelineById(pipelineId); + if (cancelled) return; setResults(response); + + // If results are not yet available, start polling + if (!hasAnyResult(response)) { + setPolling(true); + // Clear any previous interval before setting a new one + if (pollingRef.current) clearInterval(pollingRef.current); + pollingRef.current = setInterval(async () => { + try { + const updated = await getPipelineById(pipelineId); + if (cancelled) return; + setResults(updated); + if (hasAnyResult(updated)) { + clearInterval(pollingRef.current); + pollingRef.current = null; + setPolling(false); + } + } catch (err) { + console.error("Error polling pipeline results:", err); + } + }, POLL_INTERVAL_MS); + } } catch (error) { console.error("Error fetching results:", error); } @@ -35,7 +73,15 @@ function PipelineResults({ pipelineId, onClose }) { if (pipelineId) { fetchResults(); } - }, [pipelineId]); + + return () => { + cancelled = true; + if (pollingRef.current) { + clearInterval(pollingRef.current); + pollingRef.current = null; + } + }; + }, [pipelineId, hasAnyResult]); const handleTabChange = (event, newValue) => { setTrainTab(newValue); @@ -66,7 +112,16 @@ function PipelineResults({ pipelineId, onClose }) { Volver )} - No results available + {polling ? ( + + + + Pipeline is running. Results will appear automatically… + + + ) : ( + No results available + )} ); } From b39c7c0d9a4755cd1c091b2545b8d6bbebae350f Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 23 Feb 2026 23:34:16 -0300 Subject: [PATCH 21/88] fix: prettier pre-commit modification for Results.jsx --- DashAI/front/src/components/pipelines/results/Results.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 0dc639f33..3bb669423 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -30,7 +30,8 @@ function PipelineResults({ pipelineId, onClose }) { /** Returns true when the pipeline has at least one result section populated. */ const hasAnyResult = useCallback((data) => { if (!data) return false; - const hasExpl = data.exploration && data.exploration !== "No exploration data"; + const hasExpl = + data.exploration && data.exploration !== "No exploration data"; const hasTr = data.train && Object.keys(data.train).length > 0; const hasPred = data.prediction && data.prediction !== "No prediction data"; return hasExpl || hasTr || hasPred; From 85e05640fdd906b27f7c04e11ecee9b7370a5da6 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 13:43:00 -0300 Subject: [PATCH 22/88] feat: Add new pipeline nodes and update configurations for frontend data processing --- .../src/components/pipelines/CustomNode.jsx | 6 + .../front/src/components/pipelines/index.jsx | 3 + .../src/components/pipelines/nodeRegistry.jsx | 6 + .../pipelines/nodes/MetricsEvalNode.jsx | 138 +++++++ .../pipelines/nodes/SplitDataNode.jsx | 347 +++++++++++++++++ .../pipelines/nodes/TaskAndModelNode.jsx | 359 ++++++++++++++++++ .../front/src/pages/pipelines/NewPipeline.jsx | 8 +- DashAI/front/src/types/pipeline.ts | 22 ++ 8 files changed, 888 insertions(+), 1 deletion(-) create mode 100644 DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx create mode 100644 DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx create mode 100644 DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx diff --git a/DashAI/front/src/components/pipelines/CustomNode.jsx b/DashAI/front/src/components/pipelines/CustomNode.jsx index cec6e0ac4..efe6c1914 100644 --- a/DashAI/front/src/components/pipelines/CustomNode.jsx +++ b/DashAI/front/src/components/pipelines/CustomNode.jsx @@ -5,6 +5,9 @@ import InsertChartIcon from "@mui/icons-material/InsertChart"; import SettingsIcon from "@mui/icons-material/Settings"; import EmojiObjectsIcon from "@mui/icons-material/EmojiObjects"; import ManageHistoryIcon from "@mui/icons-material/ManageHistory"; +import CallSplitIcon from "@mui/icons-material/CallSplit"; +import ModelTrainingIcon from "@mui/icons-material/ModelTraining"; +import AssessmentIcon from "@mui/icons-material/Assessment"; import CloseIcon from "@mui/icons-material/Close"; import { useTheme } from "@mui/material/styles"; import { useState } from "react"; @@ -15,6 +18,9 @@ const iconMap = { SettingsIcon: SettingsIcon, EmojiObjectsIcon: EmojiObjectsIcon, ManageHistoryIcon: ManageHistoryIcon, + CallSplitIcon: CallSplitIcon, + ModelTrainingIcon: ModelTrainingIcon, + AssessmentIcon: AssessmentIcon, }; const CustomNode = ({ data, isConnectable }) => { diff --git a/DashAI/front/src/components/pipelines/index.jsx b/DashAI/front/src/components/pipelines/index.jsx index 460c2490f..818688d71 100644 --- a/DashAI/front/src/components/pipelines/index.jsx +++ b/DashAI/front/src/components/pipelines/index.jsx @@ -12,6 +12,9 @@ export { default as DataSelectorNode } from "./nodes/DataSelectorNode"; export { default as DataExplorationNode } from "./nodes/DataExplorationNode"; export { default as TrainNode } from "./nodes/TrainNode"; export { default as RetrieveModelNode } from "./nodes/RetrieveModelNode"; +export { default as SplitDataNode } from "./nodes/SplitDataNode"; +export { default as TaskAndModelNode } from "./nodes/TaskAndModelNode"; +export { default as MetricsEvalNode } from "./nodes/MetricsEvalNode"; export { default as ParamsSettings } from "./ParamsSettings"; export { default as ExplorationModal } from "./nodes/ExplorationModal"; diff --git a/DashAI/front/src/components/pipelines/nodeRegistry.jsx b/DashAI/front/src/components/pipelines/nodeRegistry.jsx index b88982351..66cf6b7f6 100644 --- a/DashAI/front/src/components/pipelines/nodeRegistry.jsx +++ b/DashAI/front/src/components/pipelines/nodeRegistry.jsx @@ -3,6 +3,9 @@ import DataExplorationNode from "./nodes/DataExplorationNode"; import TrainNode from "./nodes/TrainNode"; import RetrieveModelNode from "./nodes/RetrieveModelNode"; import ConfigurableNode from "./nodes/ConfigurableNode"; +import SplitDataNode from "./nodes/SplitDataNode"; +import TaskAndModelNode from "./nodes/TaskAndModelNode"; +import MetricsEvalNode from "./nodes/MetricsEvalNode"; const nodeRegistry = { DataSelector: DataSelectorNode, @@ -10,6 +13,9 @@ const nodeRegistry = { Train: TrainNode, RetrieveModel: RetrieveModelNode, Configurable: ConfigurableNode, + SplitData: SplitDataNode, + TaskAndModel: TaskAndModelNode, + MetricsEval: MetricsEvalNode, }; export default nodeRegistry; diff --git a/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx b/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx new file mode 100644 index 000000000..3040d02e2 --- /dev/null +++ b/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx @@ -0,0 +1,138 @@ +import React, { useState, useEffect } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Button, + DialogContent, + Typography, + TextField, + MenuItem, + Grid, +} from "@mui/material"; +import { getComponents as getComponentsRequest } from "../../../api/component"; +import { validateNode } from "../../../api/pipeline"; +import { useSnackbar } from "notistack"; + +/** + * MetricsEvalNode – selects metrics to evaluate the trained model. + * + * Expects a TaskAndModel upstream node (via prevNodes) so it can filter + * compatible metrics by the selected task. + */ +const MetricsEvalNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { + const [metrics, setMetrics] = useState(savedConfig?.metrics || []); + const [availableMetrics, setAvailableMetrics] = useState([]); + const { enqueueSnackbar } = useSnackbar(); + + // Locate the task from an upstream TaskAndModel (or Train) node + const taskNode = prevNodes?.find((node) => node?.task); + const task = taskNode?.task || ""; + + // ---------- Fetch compatible metrics ---------- + useEffect(() => { + const fetchMetrics = async () => { + if (!task) return; + try { + const metricsResult = await getComponentsRequest({ + selectTypes: ["Metric"], + relatedComponent: task, + }); + setAvailableMetrics(metricsResult); + } catch (error) { + console.error("Error fetching metrics:", error); + } + }; + fetchMetrics(); + }, [task]); + + // Restore saved selection + useEffect(() => { + if (savedConfig?.metrics) { + setMetrics(savedConfig.metrics); + } + }, [savedConfig]); + + // ---------- Save ---------- + const handleSave = async () => { + const payload = { metrics }; + + try { + const response = await validateNode("MetricsEval", payload); + if (response.status === "ok") { + onSave(payload); + onClose(); + } else { + enqueueSnackbar(response.message || "Validation failed", { + variant: "error", + }); + } + } catch (e) { + enqueueSnackbar("Error validating node", { variant: "error" }); + console.error(e); + } + }; + + return ( + + + + + Metric Evaluation + + {!task && ( + + ⚠️ Connect a Task & Model node first so compatible metrics + can be listed. + + )} + + + + setMetrics(e.target.value)} + margin="normal" + disabled={!task} + slotProps={{ select: { multiple: true } }} + > + {availableMetrics.map((metric) => ( + + {metric.name} + + ))} + + + + + + + + + ); +}; + +MetricsEvalNode.propTypes = { + open: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + onSave: PropTypes.func.isRequired, + savedConfig: PropTypes.object, + prevNodes: PropTypes.arrayOf(PropTypes.object), +}; + +MetricsEvalNode.defaultProps = { + savedConfig: null, + prevNodes: [], +}; + +export default MetricsEvalNode; diff --git a/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx b/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx new file mode 100644 index 000000000..df6cd2c14 --- /dev/null +++ b/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx @@ -0,0 +1,347 @@ +import React, { useState, useEffect, useRef } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Button, + DialogContent, + Typography, + TextField, + Grid, + FormControlLabel, + Checkbox, +} from "@mui/material"; +import { getDatasetInfo as getDatasetInfoRequest } from "../../../api/datasets"; +import { validateNode } from "../../../api/pipeline"; +import { parseRangeToIndex } from "../../../utils/parseRange"; +import { useSnackbar } from "notistack"; + +/** + * SplitDataNode – configures input/output columns and train/val/test splits. + * + * Expects `prevNodes` to contain the DataSelector config (with id & file_path) + * so it can fetch column metadata from the dataset. + */ +const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { + // Column ranges + const [inputColumns, setInputColumns] = useState( + savedConfig?.input_columns || "", + ); + const [outputColumns, setOutputColumns] = useState( + savedConfig?.output_columns || "", + ); + + // Splits + const [splits, setSplits] = useState( + savedConfig?.splits || { + train: 0.6, + validation: 0.2, + test: 0.2, + shuffle: true, + stratify: false, + splitType: "random", + }, + ); + + // Dataset info + const datasetNode = prevNodes?.find((node) => node?.file_path && node?.id); + const datasetId = datasetNode?.id ?? null; + const [datasetInfo, setDatasetInfo] = useState({}); + const [infoLoading, setInfoLoading] = useState(true); + + // Validation + const [inputError, setInputError] = useState(false); + const [outputError, setOutputError] = useState(false); + const [inputErrorMessage, setInputErrorMessage] = useState(""); + const [outputErrorMessage, setOutputErrorMessage] = useState(""); + const [validationStatus, setValidationStatus] = useState(""); + + const { enqueueSnackbar } = useSnackbar(); + const hasWarnedRef = useRef(false); + + // ---------- Fetch dataset info ---------- + const getDatasetInfo = async () => { + setInfoLoading(true); + try { + const info = await getDatasetInfoRequest(datasetId); + setDatasetInfo(info); + } catch (error) { + enqueueSnackbar("Error while trying to obtain the dataset info.", { + variant: "error", + }); + console.error(error); + } finally { + setInfoLoading(false); + } + }; + + useEffect(() => { + if (!datasetId) { + setInfoLoading(false); + if (!hasWarnedRef.current) { + enqueueSnackbar("Missing dataset – connect a DataSelector node first.", { + variant: "warning", + }); + hasWarnedRef.current = true; + } + return; + } + getDatasetInfo(); + }, [datasetId]); + + // ---------- Re-apply saved config when dataset info arrives ---------- + useEffect(() => { + const total = datasetInfo?.total_columns; + setInputColumns( + savedConfig?.input_columns?.join(",") || (total ? `1-${total - 1}` : ""), + ); + setOutputColumns( + savedConfig?.output_columns?.join(",") || (total ? `${total}` : ""), + ); + setSplits( + savedConfig?.splits || { + train: 0.6, + validation: 0.2, + test: 0.2, + shuffle: true, + stratify: false, + splitType: "random", + }, + ); + }, [savedConfig, datasetInfo]); + + // ---------- Validate column ranges on change ---------- + useEffect(() => { + const maxValue = datasetInfo?.total_columns; + if (!maxValue) return; + + try { + if (inputColumns) parseRangeToIndex(inputColumns, maxValue); + setInputError(false); + setInputErrorMessage(""); + } catch (err) { + setInputError(true); + setInputErrorMessage(err.message); + } + + try { + if (outputColumns) parseRangeToIndex(outputColumns, maxValue); + setOutputError(false); + setOutputErrorMessage(""); + } catch (err) { + setOutputError(true); + setOutputErrorMessage(err.message); + } + }, [inputColumns, outputColumns, datasetInfo]); + + // ---------- Save ---------- + const handleSave = async () => { + const maxValue = datasetInfo?.total_columns; + const columnNames = datasetInfo?.column_names || []; + + const parsedInput = inputColumns + ? parseRangeToIndex(inputColumns, maxValue) + : []; + const parsedOutput = outputColumns + ? parseRangeToIndex(outputColumns, maxValue) + : []; + + const inputColumnNames = parsedInput.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + const outputColumnNames = parsedOutput.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + + const payload = { + input_columns: parsedInput, + output_columns: parsedOutput, + splits: { + train: splits.train, + validation: splits.validation, + test: splits.test, + shuffle: splits.shuffle, + stratify: splits.stratify, + splitType: splits.splitType, + }, + }; + + try { + const response = await validateNode("SplitData", payload); + if (response.status === "ok") { + setValidationStatus("ok"); + onSave(payload); + onClose(); + } else { + setValidationStatus("error"); + enqueueSnackbar(response.message || "Validation failed", { + variant: "error", + }); + } + } catch (e) { + setValidationStatus("error"); + enqueueSnackbar("Error validating node", { variant: "error" }); + console.error(e); + } + }; + + const splitsSum = + Number(splits.train) + Number(splits.validation) + Number(splits.test); + const splitsValid = Math.abs(splitsSum - 1) < 0.001; + + return ( + + + {/* ---- Column selection ---- */} + + + Column Selection + + + + + setInputColumns(e.target.value)} + margin="normal" + error={inputError} + helperText={ + inputError ? inputErrorMessage : 'e.g. 1-4 or 1,3,5' + } + /> + + + + setOutputColumns(e.target.value)} + margin="normal" + error={outputError} + helperText={ + outputError ? outputErrorMessage : 'e.g. 5 or 5-6' + } + /> + + + {/* ---- Splits ---- */} + + + Data Splits + + + + + + setSplits({ ...splits, train: parseFloat(e.target.value) }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + + + + + setSplits({ ...splits, validation: parseFloat(e.target.value) }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + + + + + setSplits({ ...splits, test: parseFloat(e.target.value) }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + + + {!splitsValid && ( + + + Splits must sum to 1.0 (current: {splitsSum.toFixed(2)}) + + + )} + + + + setSplits({ ...splits, shuffle: e.target.checked }) + } + /> + } + label="Shuffle" + /> + + setSplits({ ...splits, stratify: e.target.checked }) + } + /> + } + label="Stratify" + /> + + + + + + + + ); +}; + +SplitDataNode.propTypes = { + open: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + onSave: PropTypes.func.isRequired, + savedConfig: PropTypes.object, + prevNodes: PropTypes.arrayOf(PropTypes.object), +}; + +SplitDataNode.defaultProps = { + savedConfig: null, + prevNodes: [], +}; + +export default SplitDataNode; diff --git a/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx new file mode 100644 index 000000000..d3c6bb0fa --- /dev/null +++ b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx @@ -0,0 +1,359 @@ +import React, { useState, useEffect, useRef } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Button, + DialogContent, + Typography, + TextField, + MenuItem, + Grid, + IconButton, +} from "@mui/material"; +import SettingsIcon from "@mui/icons-material/Settings"; +import useSchema from "../../../hooks/useSchema"; +import ParamsSettings from "../ParamsSettings"; +import { getComponents as getComponentsRequest } from "../../../api/component"; +import { validateColumns as validateColumnsRequest } from "../../../api/modelSession"; +import { getDatasetInfo as getDatasetInfoRequest } from "../../../api/datasets"; +import { validateNode } from "../../../api/pipeline"; +import { parseRangeToIndex } from "../../../utils/parseRange"; +import { useSnackbar } from "notistack"; + +/** + * TaskAndModelNode – selects ML task & model, configures hyper-parameters. + * + * Receives from SplitData (via prevNodes): + * input_columns, output_columns, splits, id (dataset id), file_path + */ +const TaskAndModelNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { + const [task, setTask] = useState(savedConfig?.task || ""); + const [model, setModel] = useState(savedConfig?.model || ""); + const [openSettings, setOpenSettings] = useState(false); + const [modelParams, setModelParams] = useState(savedConfig?.parameters || {}); + const [initialized, setInitialized] = useState(false); + + const [availableTasks, setAvailableTasks] = useState([]); + const [availableModels, setAvailableModels] = useState([]); + const [validTasks, setValidTasks] = useState([]); + const [validationErrors, setValidationErrors] = useState({}); + + const [datasetInfo, setDatasetInfo] = useState({}); + const [infoLoading, setInfoLoading] = useState(true); + + const { enqueueSnackbar } = useSnackbar(); + const hasWarnedRef = useRef(false); + + // Try to locate the dataset info from a SplitData or DataSelector upstream + const splitDataNode = prevNodes?.find( + (node) => node?.input_columns || node?.splits, + ); + const datasetNode = prevNodes?.find((node) => node?.file_path && node?.id); + const datasetId = datasetNode?.id ?? null; + const inputColumnsRaw = splitDataNode?.input_columns; + const outputColumnsRaw = splitDataNode?.output_columns; + + // ---------- Fetch dataset info ---------- + useEffect(() => { + if (!datasetId) { + setInfoLoading(false); + if (!hasWarnedRef.current) { + enqueueSnackbar( + "Missing dataset – connect a SplitData node first.", + { variant: "warning" }, + ); + hasWarnedRef.current = true; + } + return; + } + const fetchInfo = async () => { + setInfoLoading(true); + try { + const info = await getDatasetInfoRequest(datasetId); + setDatasetInfo(info); + } catch (error) { + enqueueSnackbar("Error fetching dataset info.", { variant: "error" }); + console.error(error); + } finally { + setInfoLoading(false); + } + }; + fetchInfo(); + }, [datasetId]); + + // ---------- Fetch available tasks / models ---------- + useEffect(() => { + const fetchComponents = async () => { + try { + const tasks = await getComponentsRequest({ selectTypes: ["Task"] }); + const models = await getComponentsRequest({ + selectTypes: ["Model"], + relatedComponent: task, + }); + setAvailableTasks(tasks); + setAvailableModels(models); + } catch (error) { + console.error("Error fetching components:", error); + } + }; + fetchComponents(); + }, [task]); + + // ---------- Model schema ---------- + const { defaultValues, modelSchema, yupSchema, loading } = useSchema({ + modelName: model, + }); + + useEffect(() => { + if (savedConfig) { + setTask(savedConfig.task || ""); + setModel(savedConfig.model || ""); + setModelParams(savedConfig.parameters || {}); + } + }, [savedConfig]); + + useEffect(() => { + setInitialized(false); + }, [model]); + + useEffect(() => { + if (loading || initialized) return; + if ( + savedConfig?.model === model && + savedConfig?.parameters && + Object.keys(savedConfig.parameters).length > 0 + ) { + setModelParams(savedConfig.parameters); + } else if (defaultValues && Object.keys(defaultValues).length > 0) { + setModelParams(defaultValues); + } else { + setModelParams({}); + } + setInitialized(true); + }, [loading, model, defaultValues, savedConfig, initialized]); + + const handleChange = (newValues) => setModelParams(newValues); + + // ---------- Validate available tasks against columns ---------- + useEffect(() => { + const validateAvailableTasks = async () => { + if (!datasetId || !inputColumnsRaw || !outputColumnsRaw) { + setValidTasks([]); + return; + } + + const maxValue = datasetInfo?.total_columns; + const columnNames = datasetInfo?.column_names || []; + + // inputColumnsRaw may be an array of indices already (from SplitData) + let inputColumnNames; + let outputColumnNames; + if (Array.isArray(inputColumnsRaw)) { + inputColumnNames = inputColumnsRaw.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + outputColumnNames = (outputColumnsRaw || []).map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + } else { + try { + const parsedInput = parseRangeToIndex(String(inputColumnsRaw), maxValue); + const parsedOutput = parseRangeToIndex(String(outputColumnsRaw), maxValue); + inputColumnNames = parsedInput.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + outputColumnNames = parsedOutput.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + } catch { + setValidTasks([]); + return; + } + } + + const results = await Promise.all( + availableTasks.map(async (taskObj) => { + try { + const validation = await validateColumnsRequest( + taskObj.name, + datasetId, + inputColumnNames, + outputColumnNames, + ); + if (validation.dataset_status === "valid") { + return { name: taskObj.name, valid: true }; + } + return { name: taskObj.name, valid: false, error: validation.error }; + } catch (error) { + return { name: taskObj.name, valid: false, error: "Request failed" }; + } + }), + ); + + const validList = results.filter((r) => r.valid).map((r) => r.name); + const errorsMap = {}; + results.forEach((r) => { + if (!r.valid) errorsMap[r.name] = r.error; + }); + setValidationErrors(errorsMap); + setValidTasks(validList); + }; + + validateAvailableTasks(); + }, [availableTasks, inputColumnsRaw, outputColumnsRaw, datasetId, datasetInfo]); + + // ---------- Save ---------- + const handleSave = async () => { + const payload = { + task, + model, + parameters: modelParams, + }; + + try { + const response = await validateNode("TaskAndModel", payload); + if (response.status === "ok") { + onSave(payload); + onClose(); + } else { + enqueueSnackbar(response.message || "Validation failed", { + variant: "error", + }); + } + } catch (e) { + enqueueSnackbar("Error validating node", { variant: "error" }); + console.error(e); + } + }; + + return ( + <> + + + + + Task and Model + + + + {/* Warnings when no compatible tasks */} + {validTasks.length === 0 && + Object.keys(validationErrors).length > 0 && ( + + + ⚠️ No compatible tasks available. Validation errors: + + {Object.entries(validationErrors).map(([taskName, error]) => ( + + {taskName}: {error} + + ))} + + )} + + {/* Task selector */} + + { + setTask(e.target.value); + setModel(""); + setModelParams({}); + }} + margin="normal" + disabled={validTasks.length === 0} + > + {availableTasks.map((taskObj) => ( + + {taskObj.name} + + ))} + + + + {/* Model selector + settings */} + + + + setModel(e.target.value)} + margin="normal" + disabled={!task} + > + {availableModels.map((m) => ( + + {m.name} + + ))} + + + + setOpenSettings(true)} + disabled={!model} + aria-label="model settings" + sx={{ mt: "8px" }} + > + + + + + + + + + + + + + setOpenSettings(false)} + /> + + ); +}; + +TaskAndModelNode.propTypes = { + open: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + onSave: PropTypes.func.isRequired, + savedConfig: PropTypes.object, + prevNodes: PropTypes.arrayOf(PropTypes.object), +}; + +TaskAndModelNode.defaultProps = { + savedConfig: null, + prevNodes: [], +}; + +export default TaskAndModelNode; diff --git a/DashAI/front/src/pages/pipelines/NewPipeline.jsx b/DashAI/front/src/pages/pipelines/NewPipeline.jsx index 7dc1531fd..031a3cd0f 100644 --- a/DashAI/front/src/pages/pipelines/NewPipeline.jsx +++ b/DashAI/front/src/pages/pipelines/NewPipeline.jsx @@ -191,7 +191,13 @@ function NewPipeline() { ? "Select Train Parameters" : selectedNode?.type === "Exploration" ? "Exploration Configuration" - : `Configure ${selectedNode?.type || "Node"}`} + : selectedNode?.type === "SplitData" + ? "Configure Data Splits" + : selectedNode?.type === "TaskAndModel" + ? "Select Task & Model" + : selectedNode?.type === "MetricsEval" + ? "Select Metrics" + : `Configure ${selectedNode?.type || "Node"}`} | string; + train?: { + info?: string; + parameters?: Record; + metrics?: Record; + model_path?: string; + input_columns?: string[]; + task?: string; + }; + task_and_model?: { + task?: string; + model?: string; + parameters?: Record; + model_path?: string; + input_columns?: string[]; + }; + metrics_result?: { + metric_names?: string[]; + results?: Record; + }; + split_data?: Record; + prediction?: string; } From 300a7ff7a485bcb1196ac3e991fe613bc91d522b Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 13:44:36 -0300 Subject: [PATCH 23/88] feat: Enhance PipelineResults and ResultsGraphs components for improved metrics handling and display --- .../components/pipelines/results/Results.jsx | 39 +++- .../pipelines/results/ResultsGraphs.jsx | 206 ++++-------------- .../pipelines/results/ResultsMetrics.jsx | 12 +- 3 files changed, 81 insertions(+), 176 deletions(-) diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 3bb669423..2b6c70068 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -33,8 +33,11 @@ function PipelineResults({ pipelineId, onClose }) { const hasExpl = data.exploration && data.exploration !== "No exploration data"; const hasTr = data.train && Object.keys(data.train).length > 0; + const hasAtomized = + (data.task_and_model && Object.keys(data.task_and_model).length > 0) || + (data.metrics_result && Object.keys(data.metrics_result).length > 0); const hasPred = data.prediction && data.prediction !== "No prediction data"; - return hasExpl || hasTr || hasPred; + return hasExpl || hasTr || hasAtomized || hasPred; }, []); useEffect(() => { @@ -95,17 +98,29 @@ function PipelineResults({ pipelineId, onClose }) { const hasExploration = results.exploration && results.exploration !== "No exploration data"; const hasTrain = results.train && Object.keys(results.train).length > 0; + // Atomized nodes: task_and_model + metrics_result + const hasAtomizedTrain = + (results.task_and_model && + Object.keys(results.task_and_model).length > 0) || + (results.metrics_result && + Object.keys(results.metrics_result).length > 0); const hasPrediction = results.prediction && results.prediction !== "No prediction data"; - const paramData = { - parameters: - results.train && results.train.parameters - ? results.train.parameters - : null, - }; + // Derive unified values from whichever format is available + const trainModelName = hasTrain + ? results.train.info + : results.task_and_model?.model ?? "-"; + const trainParameters = hasTrain + ? results.train.parameters + : results.task_and_model?.parameters ?? null; + const trainMetrics = hasTrain + ? results.train.metrics + : results.metrics_result?.results ?? null; + + const paramData = { parameters: trainParameters }; - if (!hasExploration && !hasTrain && !hasPrediction) { + if (!hasExploration && !hasTrain && !hasAtomizedTrain && !hasPrediction) { return ( {onClose && ( @@ -148,7 +163,7 @@ function PipelineResults({ pipelineId, onClose }) { )} - {hasTrain && ( + {(hasTrain || hasAtomizedTrain) && ( }> Train @@ -171,7 +186,7 @@ function PipelineResults({ pipelineId, onClose }) { Model Name - {results.train.info ?? "-"} + {trainModelName} )} @@ -183,13 +198,13 @@ function PipelineResults({ pipelineId, onClose }) { {trainTab === 2 && ( )} {trainTab === 3 && ( - + )} diff --git a/DashAI/front/src/components/pipelines/results/ResultsGraphs.jsx b/DashAI/front/src/components/pipelines/results/ResultsGraphs.jsx index a1a13e6d4..987280f80 100644 --- a/DashAI/front/src/components/pipelines/results/ResultsGraphs.jsx +++ b/DashAI/front/src/components/pipelines/results/ResultsGraphs.jsx @@ -1,169 +1,57 @@ import PropTypes from "prop-types"; -import React, { useEffect, useState } from "react"; +import React, { useMemo } from "react"; import { Alert, AlertTitle } from "@mui/material"; -import { useSnackbar } from "notistack"; -import graphsMaking from "../../../pages/results/constants/graphsMaking"; -import layoutMaking from "../../../pages/results/constants/layoutMaking"; -import ResultsGraphsLayout from "../../../pages/results/components/ResultsGraphsLayout"; +import ResultsGraphs from "../../../pages/results/components/ResultsGraphs"; -function PipelineResultsGraphs({ metrics }) { - const { enqueueSnackbar } = useSnackbar(); - const [selectedChart, setSelectedChart] = useState("radar"); - const [selectedParameters, setSelectedParameters] = useState([]); - const [showCustomMetrics, setShowCustomMetrics] = useState(false); - const [selectedGeneralMetric, setSelectedGeneralMetric] = useState("test"); - const [concatenatedMetrics, setConcatenatedMetrics] = useState([]); - const [tabularMetrics, setTabularMetrics] = useState([]); - const [chartData, setChartData] = useState({}); - const [filteredDataProcess, setFilteredDataProcess] = useState([]); - - const handleChangeChart = (chartType) => { - setSelectedChart(chartType); - }; - - const handleToggleParameter = (parameter) => { - const updatedParameters = selectedParameters.includes(parameter) - ? selectedParameters.filter((param) => param !== parameter) - : [...selectedParameters, parameter]; - setSelectedParameters(updatedParameters); - }; - - const handleToggleMetrics = () => { - setShowCustomMetrics(!showCustomMetrics); - setSelectedParameters([]); - }; - - useEffect(() => { - const processData = () => { - try { - const phases = Object.keys(metrics); - const metricNames = Object.keys(metrics[phases[0]]); - const tabularMetrics = []; - - phases.forEach((phase) => { - metricNames.forEach((metric) => { - tabularMetrics.push(`${phase} ${metric}`); - }); - }); - - const concatenatedMetrics = Array.from( - new Set(phases.concat(metricNames)), - ); - setConcatenatedMetrics(concatenatedMetrics); - setTabularMetrics(tabularMetrics); - - let parameterIndex = []; - let generalParameters = []; - - if (showCustomMetrics) { - parameterIndex = selectedParameters.map((param) => - tabularMetrics.indexOf(param), - ); - } else { - const criteria = {}; - concatenatedMetrics.forEach((item) => { - criteria[item] = item; - }); - - tabularMetrics.forEach((metric, index) => { - Object.entries(criteria).forEach(([metricName, substring]) => { - if ( - selectedGeneralMetric === metricName && - metric.includes(substring) - ) { - parameterIndex.push(index); - generalParameters.push(metric); - } - }); - }); - } - - const syntheticRun = { - status: 3, - train_metrics: metrics.train, - validation_metrics: metrics.validation, - test_metrics: metrics.test, - }; - - const newFilteredData = [syntheticRun]; - setFilteredDataProcess(newFilteredData); - - const graphsToView = {}; - let pieCounter = 0; - - const numericValues = []; - ["train", "validation", "test"].forEach((split) => { - metricNames.forEach((metric) => { - numericValues.push(metrics[split][metric]); - }); - }); - - const relevantNumericValues = parameterIndex.map( - (index) => numericValues[index], - ); - - graphsMaking( - graphsToView, - syntheticRun, - relevantNumericValues, - showCustomMetrics, - selectedParameters, - generalParameters, - pieCounter, - ); - - const { generalLayout, pieLayout } = layoutMaking( - selectedChart, - graphsToView, - ); - - const graphsToViewKeys = Object.keys(graphsToView); - const radarValues = graphsToView[graphsToViewKeys[0]]; - const barValues = graphsToView[graphsToViewKeys[1]]; - const pieValues = graphsToView[graphsToViewKeys[2]]; +function normalizeSplitMetrics(splitMetrics) { + return splitMetrics && typeof splitMetrics === "object" ? splitMetrics : {}; +} - setChartData({ - generalLayout, - pieLayout, - radarValues, - barValues, - pieValues, - }); - } catch (err) { - enqueueSnackbar("Error processing metrics", { variant: "error" }); - console.error(err); - } - }; +function PipelineResultsGraphs({ metrics }) { + const trainMetrics = normalizeSplitMetrics(metrics?.train); + const validationMetrics = normalizeSplitMetrics(metrics?.validation); + const testMetrics = normalizeSplitMetrics(metrics?.test); + + const hasAnyMetrics = + Object.keys(trainMetrics).length > 0 || + Object.keys(validationMetrics).length > 0 || + Object.keys(testMetrics).length > 0; + + const syntheticRuns = useMemo( + () => [ + { + status: 3, + train_metrics: trainMetrics, + validation_metrics: validationMetrics, + test_metrics: testMetrics, + }, + ], + [trainMetrics, validationMetrics, testMetrics], + ); - processData(); - }, [metrics, selectedParameters, selectedChart]); + if (!hasAnyMetrics) { + return ( + + No metric data available + Please provide valid metrics to render the graphs. + + ); + } - return ( - <> - {filteredDataProcess.length === 0 ? ( - - No metric data available - Please provide valid metrics to render the graphs. - - ) : ( - - )} - - ); + return ; } +PipelineResultsGraphs.propTypes = { + metrics: PropTypes.shape({ + train: PropTypes.object, + validation: PropTypes.object, + test: PropTypes.object, + }), +}; + +PipelineResultsGraphs.defaultProps = { + metrics: {}, +}; + export default PipelineResultsGraphs; diff --git a/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx b/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx index b3c05923a..4a6fd2fd7 100644 --- a/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx +++ b/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx @@ -37,12 +37,14 @@ function PipelineResultsMetrics({ metricsData = {} }) { ) : ( Object.entries(currentMetrics).map(([metricName, value]) => ( - - - {metricName} + + + + {metricName} + - - + + {typeof value === "number" ? value.toFixed(4) : value} From 7e1ed55ca209e2b869faca501ba71d5bafa78036 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 13:48:33 -0300 Subject: [PATCH 24/88] feat: Atomize train node into split, task, and metrics tables in database schema --- ...mize_train_node_into_split_task_metrics.py | 107 ++++++++++++++++++ .../back/tasks/tabular_classification_task.py | 41 ++++++- 2 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py diff --git a/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py b/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py new file mode 100644 index 000000000..23f66c6cb --- /dev/null +++ b/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py @@ -0,0 +1,107 @@ +"""Atomize train node into split/task/metrics tables. + +Revision ID: c4f7a8b2d1e3 +Revises: 98791b9df4f0 +Create Date: 2026-03-25 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c4f7a8b2d1e3" +down_revision: Union[str, None] = "98791b9df4f0" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _table_exists(table_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return table_name in inspector.get_table_names() + + +def _column_exists(table_name: str, column_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return any( + column["name"] == column_name + for column in inspector.get_columns(table_name) + ) + + +def upgrade() -> None: + if _table_exists("pipeline"): + if not _column_exists("pipeline", "split_data"): + op.add_column("pipeline", sa.Column("split_data", sa.JSON(), nullable=True)) + if not _column_exists("pipeline", "task_and_model"): + op.add_column( + "pipeline", sa.Column("task_and_model", sa.JSON(), nullable=True) + ) + if not _column_exists("pipeline", "metrics_result"): + op.add_column( + "pipeline", sa.Column("metrics_result", sa.JSON(), nullable=True) + ) + + if not _table_exists("split_data_node"): + op.create_table( + "split_data_node", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.Column("input_columns", sa.JSON(), nullable=False), + sa.Column("output_columns", sa.JSON(), nullable=False), + sa.Column("splits", sa.JSON(), nullable=False), + sa.Column("created", sa.DateTime(), nullable=True), + sa.Column("last_modified", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + + if not _table_exists("task_and_model_node"): + op.create_table( + "task_and_model_node", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.Column("task_name", sa.String(), nullable=False), + sa.Column("model_name", sa.String(), nullable=False), + sa.Column("parameters", sa.JSON(), nullable=True), + sa.Column("model_path", sa.String(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=True), + sa.Column("last_modified", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + + if not _table_exists("metrics_node"): + op.create_table( + "metrics_node", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.Column("metric_names", sa.JSON(), nullable=False), + sa.Column("results", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=True), + sa.Column("last_modified", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + if _table_exists("metrics_node"): + op.drop_table("metrics_node") + + if _table_exists("task_and_model_node"): + op.drop_table("task_and_model_node") + + if _table_exists("split_data_node"): + op.drop_table("split_data_node") + + if _table_exists("pipeline"): + with op.batch_alter_table("pipeline", schema=None) as batch_op: + if _column_exists("pipeline", "metrics_result"): + batch_op.drop_column("metrics_result") + if _column_exists("pipeline", "task_and_model"): + batch_op.drop_column("task_and_model") + if _column_exists("pipeline", "split_data"): + batch_op.drop_column("split_data") diff --git a/DashAI/back/tasks/tabular_classification_task.py b/DashAI/back/tasks/tabular_classification_task.py index 600c4f1e3..4255e08ab 100644 --- a/DashAI/back/tasks/tabular_classification_task.py +++ b/DashAI/back/tasks/tabular_classification_task.py @@ -1,12 +1,15 @@ from typing import List, Union +import logging from datasets import DatasetDict from DashAI.back.core.utils import MultilingualString from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset from DashAI.back.tasks.classification_task import ClassificationTask from DashAI.back.types.categorical import Categorical -from DashAI.back.types.value_types import Float, Integer +from DashAI.back.types.value_types import Float, Integer, Text + +log = logging.getLogger(__name__) class TabularClassificationTask(ClassificationTask): @@ -35,7 +38,7 @@ class TabularClassificationTask(ClassificationTask): en="Tabular Classification", es="Clasificación Tabular" ) metadata: dict = { - "inputs_types": [Float, Integer, Categorical], + "inputs_types": [Float, Integer, Categorical, Text], "outputs_types": [Categorical], "inputs_cardinality": "n", "outputs_cardinality": 1, @@ -49,7 +52,8 @@ def prepare_for_task( ) -> DashAIDataset: """Convert the dataset to DashAIDataset and check the columns types - A copy of the dataset is created. + A copy of the dataset is created. Text columns are automatically + converted to Categorical type. Parameters ---------- @@ -61,7 +65,32 @@ def prepare_for_task( DashAIDataset Dataset with the new types """ - dashai_dataset = super().prepare_for_task( - dataset, input_columns, output_columns + # Convert to DashAIDataset if needed (this is what the parent does first) + if isinstance(dataset, DatasetDict): + dashai_dataset = DashAIDataset(dataset) + else: + dashai_dataset = dataset + + # Convert Text columns to Categorical BEFORE validation + for col in input_columns: + if col in dashai_dataset.types: + col_type = dashai_dataset.types[col] + if isinstance(col_type, Text): + # Get unique values using dataset API + unique_values = [ + value for value in dashai_dataset.unique(col) if value is not None + ] + log.info( + f"Converting Text column '{col}' to Categorical with " + f"{len(unique_values)} categories" + ) + # Create a Categorical type with the unique values + categorical = Categorical(values=unique_values) + # Update the type in the dataset + dashai_dataset.types[col] = categorical + + # Now call parent's prepare_for_task for validation + # This will validate that all input columns are now of allowed types + return super().prepare_for_task( + dashai_dataset, input_columns, output_columns ) - return dashai_dataset From 0c89ca18982b4264f7815ab36c83248a86e8bc29 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 13:50:01 -0300 Subject: [PATCH 25/88] feat: Add MetricsEval, SplitData, and TaskAndModel nodes --- DashAI/back/pipeline/metrics_node.py | 100 +++++++++ DashAI/back/pipeline/split_data_node.py | 102 +++++++++ DashAI/back/pipeline/task_and_model_node.py | 233 ++++++++++++++++++++ 3 files changed, 435 insertions(+) create mode 100644 DashAI/back/pipeline/metrics_node.py create mode 100644 DashAI/back/pipeline/split_data_node.py create mode 100644 DashAI/back/pipeline/task_and_model_node.py diff --git a/DashAI/back/pipeline/metrics_node.py b/DashAI/back/pipeline/metrics_node.py new file mode 100644 index 000000000..255b55d05 --- /dev/null +++ b/DashAI/back/pipeline/metrics_node.py @@ -0,0 +1,100 @@ +import logging +from typing import Any, Dict, List + +from kink import di + +from DashAI.back.dependencies.registry import ComponentRegistry +from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.metrics.base_metric import BaseMetric + +log = logging.getLogger(__name__) + + +class MetricsEval(BaseJob): + """Pipeline node that evaluates the trained model with selected metrics. + + It expects both the split data **and** the trained model factory to + already be available in the pipeline context (populated by the + *SplitData* and *TaskAndModel* nodes respectively). + + Parameters + ---------- + metrics : List[str] + Names of the metrics to compute (must be registered in the + component registry). + """ + + def __init__(self, metrics: List[str]) -> None: + super().__init__(kwargs={"metrics": metrics}) + self.metrics = metrics + + # -- BaseJob interface --------------------------------------------------- + + def set_status_as_delivered(self) -> None: + log.debug("MetricsEval executed successfully.") + + def set_status_as_error(self) -> None: + log.error("MetricsEval encountered an error.") + + def get_job_name(self) -> str: + return "MetricsEval" + + # -- Execution ----------------------------------------------------------- + + async def run( + self, + context: Dict[str, Any], + component_registry: ComponentRegistry = lambda di: di["component_registry"], + ) -> Dict[str, Any]: + """Compute metrics for the trained model. + + Reads from ``context``: + * ``trained_x``, ``trained_y`` – from *TaskAndModel* + * ``model_factory`` – ``ModelFactory`` instance from *TaskAndModel* + + Returns + ------- + dict + ``{"metrics_result": { ... }}`` with per-metric scores. + """ + factory = context.get("model_factory") + x = context.get("trained_x") + y = context.get("trained_y") + + if factory is None or x is None or y is None: + raise JobError( + "MetricsEval requires a trained model. " + "Make sure TaskAndModel runs before this node." + ) + + # Resolve metric classes from registry + all_metrics = { + component_dict["name"]: component_dict + for component_dict in component_registry(di).get_components_by_types( + select="Metric" + ) + } + + metric_classes: List[BaseMetric] = [] + for metric_name in self.metrics: + entry = all_metrics.get(metric_name) + if entry and entry.get("class"): + metric_classes.append(entry["class"]) + else: + log.warning(f"Metric '{metric_name}' not found in registry.") + + if not metric_classes: + raise JobError("No valid metrics found to evaluate.") + + try: + model_metrics = factory.evaluate(x, y, metric_classes) + except Exception as e: + log.exception(e) + raise JobError("Metrics calculation failed") from e + + return { + "metrics_result": { + "metric_names": self.metrics, + "results": model_metrics, + }, + } diff --git a/DashAI/back/pipeline/split_data_node.py b/DashAI/back/pipeline/split_data_node.py new file mode 100644 index 000000000..860f5cdc2 --- /dev/null +++ b/DashAI/back/pipeline/split_data_node.py @@ -0,0 +1,102 @@ +import logging +from typing import Any, Dict, List + +from DashAI.back.dataloaders.classes.dashai_dataset import ( + get_column_names_from_indexes, +) +from DashAI.back.job.base_job import BaseJob, JobError + +log = logging.getLogger(__name__) + + +class SplitData(BaseJob): + """Pipeline node that configures the data partition of the dataset. + + It resolves the input/output column names and stores the split ratios + in the pipeline context. The actual ``prepare_for_task`` and split + operations are deferred to the *TaskAndModel* node which knows the + chosen task. + + Parameters + ---------- + input_columns : List[int] + Column indices to use as input features. + output_columns : List[int] + Column indices to use as output targets. + splits : Dict[str, float] + Dictionary with ``train``, ``validation``, and ``test`` ratios + (must sum to 1). + """ + + def __init__( + self, + input_columns: List[int], + output_columns: List[int], + splits: Dict[str, float], + ) -> None: + super().__init__( + kwargs={ + "input_columns": input_columns, + "output_columns": output_columns, + "splits": splits, + }, + ) + self.input_columns = input_columns + self.output_columns = output_columns + self.splits = splits + + # -- BaseJob interface --------------------------------------------------- + + def set_status_as_delivered(self) -> None: + log.debug("SplitData executed successfully.") + + def set_status_as_error(self) -> None: + log.error("SplitData encountered an error.") + + def get_job_name(self) -> str: + return "SplitData" + + # -- Execution ----------------------------------------------------------- + + async def run( + self, + context: Dict[str, Any], + ) -> Dict[str, Any]: + """Resolve column names and store split config in *context*. + + The following keys are added to ``context``: + * ``input_columns`` – resolved column names + * ``output_columns`` – resolved column names + * ``input_columns_idx`` – original column indices + * ``output_columns_idx`` – original column indices + * ``split_ratios`` – the requested split ratios dict + """ + dataset = context["dataset"] + + try: + input_columns_names = get_column_names_from_indexes( + dataset, self.input_columns + ) + output_columns_names = get_column_names_from_indexes( + dataset, self.output_columns + ) + except Exception as e: + log.exception(e) + raise JobError( + "Cannot resolve column names from the dataset.", + ) from e + + # Populate context for downstream nodes + context["input_columns"] = input_columns_names + context["output_columns"] = output_columns_names + context["input_columns_idx"] = self.input_columns + context["output_columns_idx"] = self.output_columns + context["split_ratios"] = self.splits + + return { + "split_data": { + "input_columns": self.input_columns, + "output_columns": self.output_columns, + "splits": self.splits, + }, + } diff --git a/DashAI/back/pipeline/task_and_model_node.py b/DashAI/back/pipeline/task_and_model_node.py new file mode 100644 index 000000000..5e78e71b1 --- /dev/null +++ b/DashAI/back/pipeline/task_and_model_node.py @@ -0,0 +1,233 @@ +import logging +import os +import shutil +from typing import Any, Dict + +from kink import di + +from DashAI.back.config import DefaultSettings +from DashAI.back.dataloaders.classes.dashai_dataset import ( + prepare_for_model_session, + select_columns, + split_dataset, +) +from DashAI.back.dependencies.registry import ComponentRegistry +from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.models.base_model import BaseModel +from DashAI.back.models.model_factory import ModelFactory +from DashAI.back.tasks.base_task import BaseTask + +log = logging.getLogger(__name__) + + +class TaskAndModel(BaseJob): + """Pipeline node that selects the ML task and model, and trains it. + + It expects the split data to already be available in the pipeline + context (populated by the *SplitData* node). + + Parameters + ---------- + task : str + Name of the ML task (e.g. ``"TabularClassificationTask"``). + model : str + Name of the model to train. + parameters : Dict[str, Any] | None + Model-specific hyper-parameters. + """ + + def __init__( + self, + task: str, + model: str, + parameters: Dict[str, Any] = None, + ) -> None: + super().__init__( + kwargs={ + "task": task, + "model": model, + "parameters": parameters, + }, + ) + self.task = task + self.model = model + self.parameters = parameters + + # -- BaseJob interface --------------------------------------------------- + + def set_status_as_delivered(self) -> None: + log.debug("TaskAndModel executed successfully.") + + def set_status_as_error(self) -> None: + log.error("TaskAndModel encountered an error.") + + def get_job_name(self) -> str: + return f"TaskAndModel: {self.model}" + + # -- Execution ----------------------------------------------------------- + + async def run( + self, + context: Dict[str, Any], + component_registry: ComponentRegistry = lambda di: di["component_registry"], + ) -> Dict[str, Any]: + """Train a model and save it to disk. + + Reads from ``context``: + * ``input_columns``, ``output_columns`` – populated by *SplitData* + * ``split_ratios`` – populated by *SplitData* + * ``dataset`` – the raw dataset + * ``pipeline_id`` + + Writes to ``context``: + * ``model_class``, ``model_path``, ``model_name``, ``task_name`` + * ``trained_x``, ``trained_y`` – datasets used during training + (needed by the *Metrics* node) + * ``x_splits``, ``y_splits``, ``n_labels`` + """ + pipeline_id = context["pipeline_id"] + dataset = context["dataset"] + input_columns_names = context["input_columns"] + output_columns_names = context["output_columns"] + split_ratios = context["split_ratios"] + + context["task_name"] = self.task + context["model_name"] = self.model + + # --- Resolve task and prepare dataset --- + try: + task_class: BaseTask = component_registry(di)[self.task]["class"] + task_instance = task_class() + + prepared_dataset = task_instance.prepare_for_task( + dataset=dataset, + input_columns=input_columns_names, + output_columns=output_columns_names, + ) + + n_labels = None + if self.task in [ + "TextClassificationTask", + "TabularClassificationTask", + "ImageClassificationTask", + ]: + all_classes = prepared_dataset.unique(output_columns_names[0]) + n_labels = len(all_classes) + + prepared_dataset, splits = prepare_for_model_session( + dataset=prepared_dataset, + splits=split_ratios, + output_columns=output_columns_names, + ) + + x, y = select_columns( + prepared_dataset, input_columns_names, output_columns_names + ) + x_splits = split_dataset(x) + y_splits = split_dataset(y) + + context["x_splits"] = x_splits + context["y_splits"] = y_splits + context["splits"] = splits + context["n_labels"] = n_labels + except Exception as e: + log.exception(e) + raise JobError( + f"Cannot prepare / split dataset for task {self.task}", + ) from e + + # --- Resolve model class from registry --- + try: + model_class = component_registry(di)[self.model]["class"] + context["model_class"] = model_class + except Exception as e: + log.exception(e) + raise JobError( + f"Unable to find Model with name {self.model} in registry.", + ) from e + + # --- Instantiate model via factory --- + try: + factory = ModelFactory(model_class, self.parameters, n_labels=n_labels) + model: BaseModel = factory.model + except Exception as e: + log.exception(e) + raise JobError( + f"Unable to instantiate model {self.model}", + ) from e + + # --- Train --- + try: + if hasattr(model, "train") and callable(model.train): + model.train( + x_splits["train"], + y_splits["train"], + x_splits.get("validation"), + y_splits.get("validation"), + ) + x = x_splits + y = y_splits + elif hasattr(model, "fit") and callable(model.fit): + x = {} + y = {} + for split in ["train", "validation", "test"]: + if split in x_splits: + x[split] = x_splits[split].to_pandas().values + if split in y_splits: + y[split] = y_splits[split].to_pandas().values.ravel() + model.fit(x["train"], y["train"]) + else: + raise AttributeError( + f"Model {model_class} has neither 'train' nor 'fit' method" + ) + except Exception as e: + log.exception(e) + raise JobError("Model training failed") from e + + # --- Save trained model --- + try: + settings = DefaultSettings() + sqlite_local = os.path.expanduser(settings.LOCAL_PATH) + train_root = os.path.join(sqlite_local, "pipelines", "train") + os.makedirs(train_root, exist_ok=True) + + path = os.path.join(train_root, str(pipeline_id)) + + # Backward-compatibility: older runs may have saved a file in + # `train/`. This node now expects that location to be + # a directory, so replace legacy file with a directory. + if os.path.isfile(path): + log.warning( + "Replacing legacy model file path with directory: %s", path + ) + os.remove(path) + + os.makedirs(path, exist_ok=True) + model_path = os.path.join(path, "model") + + # Ensure clean overwrite when retraining same pipeline id. + if os.path.isdir(model_path): + shutil.rmtree(model_path) + elif os.path.isfile(model_path): + os.remove(model_path) + + model.save(model_path) + context["model_path"] = model_path + except Exception as e: + log.exception(e) + raise JobError("Model saving failed") from e + + # Keep references for the Metrics node + context["trained_x"] = x + context["trained_y"] = y + context["model_factory"] = factory + + return { + "task_and_model": { + "task": self.task, + "model": self.model, + "parameters": self.parameters, + "model_path": model_path, + "input_columns": context.get("input_columns"), + }, + } From b62d136400db07e6cc06ff52f09ead9933efecab Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 13:50:34 -0300 Subject: [PATCH 26/88] feat: Add split_data, task_and_model, and metrics_result fields to pipeline creation and update --- DashAI/back/api/api_v1/endpoints/pipelines.py | 6 ++++++ DashAI/back/initial_components.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/DashAI/back/api/api_v1/endpoints/pipelines.py b/DashAI/back/api/api_v1/endpoints/pipelines.py index f18c208bb..1d7a2eb7f 100644 --- a/DashAI/back/api/api_v1/endpoints/pipelines.py +++ b/DashAI/back/api/api_v1/endpoints/pipelines.py @@ -363,6 +363,9 @@ async def create_pipeline( exploration=None, train=None, prediction=None, + split_data=None, + task_and_model=None, + metrics_result=None, ) db.add(new_pipeline) db.commit() @@ -425,6 +428,9 @@ async def update_pipeline( pipeline.exploration = None pipeline.train = None pipeline.prediction = None + pipeline.split_data = None + pipeline.task_and_model = None + pipeline.metrics_result = None steps_dict = [ step.model_dump() if hasattr(step, "model_dump") else step diff --git a/DashAI/back/initial_components.py b/DashAI/back/initial_components.py index 3043fdd08..a59f2ff95 100644 --- a/DashAI/back/initial_components.py +++ b/DashAI/back/initial_components.py @@ -117,8 +117,11 @@ from DashAI.back.pipeline import ( DataExploration, DataSelector, + MetricsEval, Prediction, RetrieveModel, + SplitData, + TaskAndModel, Train, ) from DashAI.back.plugins.utils import get_available_plugins @@ -279,6 +282,9 @@ def get_initial_components(): Train, RetrieveModel, Prediction, + SplitData, + TaskAndModel, + MetricsEval, SMOTEConverter, SMOTEENNConverter, RandomUnderSamplerConverter, From 8e507ce9513f87298627014b44032794d10a45a9 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 13:51:31 -0300 Subject: [PATCH 27/88] feat: Add validators for SplitData, TaskAndModel, and MetricsEval nodes --- DashAI/back/pipeline/__init__.py | 14 +++-- DashAI/back/pipeline/validator/__init__.py | 13 ++++- DashAI/back/pipeline/validator/validator.py | 62 +++++++++++++++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/DashAI/back/pipeline/__init__.py b/DashAI/back/pipeline/__init__.py index c88c7ce57..a23aa7de6 100644 --- a/DashAI/back/pipeline/__init__.py +++ b/DashAI/back/pipeline/__init__.py @@ -1,13 +1,19 @@ from .data_selector_node import DataSelector from .exploration_node import DataExploration -from .train_node import Train -from .retrieve_model_node import RetrieveModel +from .metrics_node import MetricsEval from .prediction_node import Prediction +from .retrieve_model_node import RetrieveModel +from .split_data_node import SplitData +from .task_and_model_node import TaskAndModel +from .train_node import Train __all__ = [ "DataSelector", "DataExploration", - "Train", - "RetrieveModel", + "MetricsEval", "Prediction", + "RetrieveModel", + "SplitData", + "TaskAndModel", + "Train", ] diff --git a/DashAI/back/pipeline/validator/__init__.py b/DashAI/back/pipeline/validator/__init__.py index a3786066f..76785faa9 100644 --- a/DashAI/back/pipeline/validator/__init__.py +++ b/DashAI/back/pipeline/validator/__init__.py @@ -1,5 +1,13 @@ from .pipeline_validator import PipelineValidator -from .validator import DataSelectorValidator, DataExplorationValidator, TrainValidator, RetrieveModelValidator +from .validator import ( + DataExplorationValidator, + DataSelectorValidator, + MetricsEvalValidator, + RetrieveModelValidator, + SplitDataValidator, + TaskAndModelValidator, + TrainValidator, +) __all__ = [ "PipelineValidator", @@ -7,4 +15,7 @@ "DataExplorationValidator", "TrainValidator", "RetrieveModelValidator", + "SplitDataValidator", + "TaskAndModelValidator", + "MetricsEvalValidator", ] diff --git a/DashAI/back/pipeline/validator/validator.py b/DashAI/back/pipeline/validator/validator.py index 4db447355..8dec3de65 100644 --- a/DashAI/back/pipeline/validator/validator.py +++ b/DashAI/back/pipeline/validator/validator.py @@ -110,9 +110,71 @@ def validate(self) -> Dict[str, str]: return {"status": "ok"} +class SplitDataValidator(BaseNodeValidator): + """Validator for the SplitData node configuration.""" + + def validate(self) -> Dict[str, str]: + input_cols = self.data.get("input_columns") + output_cols = self.data.get("output_columns") + splits = self.data.get("splits", {}) + + if not input_cols or not output_cols: + return { + "status": "error", + "message": "Input and output columns are required", + } + + train, val, test = ( + splits.get("train", 0), + splits.get("validation", 0), + splits.get("test", 0), + ) + if round(train + val + test, 5) != 1.0: + return { + "status": "error", + "message": "Train, validation, and test splits must sum to 1", + } + + return {"status": "ok"} + + +class TaskAndModelValidator(BaseNodeValidator): + """Validator for the TaskAndModel node configuration.""" + + def validate(self) -> Dict[str, str]: + task = self.data.get("task") + model = self.data.get("model") + + if task is None: + return {"status": "error", "message": "Task is required"} + + if model is None: + return {"status": "error", "message": "Model is required"} + + return {"status": "ok"} + + +class MetricsEvalValidator(BaseNodeValidator): + """Validator for the MetricsEval node configuration.""" + + def validate(self) -> Dict[str, str]: + metrics = self.data.get("metrics") + + if not metrics: + return { + "status": "error", + "message": "At least one metric must be selected", + } + + return {"status": "ok"} + + VALIDATOR_MAP = { "DataSelector": DataSelectorValidator, "DataExploration": DataExplorationValidator, "Train": TrainValidator, "RetrieveModel": RetrieveModelValidator, + "SplitData": SplitDataValidator, + "TaskAndModel": TaskAndModelValidator, + "MetricsEval": MetricsEvalValidator, } From 2072588b8ce4a5c5f5b6948c46d79be7ea90bd8a Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 13:52:31 -0300 Subject: [PATCH 28/88] feat: Add SplitData, TaskAndModel, and MetricsEval nodes with corresponding database models and pipeline integration --- DashAI/back/dependencies/database/models.py | 91 +++++++++++++++++++ DashAI/back/job/pipeline_job.py | 9 ++ .../pipeline/validator/nodes_definitions.py | 55 ++++++++++- 3 files changed, 153 insertions(+), 2 deletions(-) diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index 6f9350a18..10e46a0ea 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -533,6 +533,97 @@ class Pipeline(Base): train: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) prediction: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + # New JSON columns for atomized Train sub-node results + split_data: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + task_and_model: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + metrics_result: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + + # Relationships to new node tables + split_data_nodes: Mapped[List["SplitDataNode"]] = relationship( + "SplitDataNode", cascade="all, delete-orphan", back_populates="pipeline" + ) + task_and_model_nodes: Mapped[List["TaskAndModelNode"]] = relationship( + "TaskAndModelNode", cascade="all, delete-orphan", back_populates="pipeline" + ) + metrics_nodes: Mapped[List["MetricsNode"]] = relationship( + "MetricsNode", cascade="all, delete-orphan", back_populates="pipeline" + ) + + +class SplitDataNode(Base): + __tablename__ = "split_data_node" + """ + Table to store configuration and results for the Split Data step + of a pipeline. Configures how the dataset is partitioned into + train / validation / test splits. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_id: Mapped[int] = mapped_column( + ForeignKey("pipeline.id", ondelete="CASCADE"), nullable=False + ) + input_columns: Mapped[List[int]] = mapped_column(JSON, nullable=False) + output_columns: Mapped[List[int]] = mapped_column(JSON, nullable=False) + splits: Mapped[Dict[str, float]] = mapped_column(JSON, nullable=False) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + + # Relationships + pipeline: Mapped["Pipeline"] = relationship( + "Pipeline", back_populates="split_data_nodes" + ) + + +class TaskAndModelNode(Base): + __tablename__ = "task_and_model_node" + """ + Table to store configuration and results for the Task & Model step + of a pipeline. Selects the ML task and model, stores training + parameters, and keeps a reference to the persisted model artifact. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_id: Mapped[int] = mapped_column( + ForeignKey("pipeline.id", ondelete="CASCADE"), nullable=False + ) + task_name: Mapped[str] = mapped_column(String, nullable=False) + model_name: Mapped[str] = mapped_column(String, nullable=False) + parameters: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + model_path: Mapped[str] = mapped_column(String, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + + # Relationships + pipeline: Mapped["Pipeline"] = relationship( + "Pipeline", back_populates="task_and_model_nodes" + ) + + +class MetricsNode(Base): + __tablename__ = "metrics_node" + """ + Table to store configuration and results for the Metrics step + of a pipeline. Holds the list of metric names to compute and, + after execution, the evaluation results. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_id: Mapped[int] = mapped_column( + ForeignKey("pipeline.id", ondelete="CASCADE"), nullable=False + ) + metric_names: Mapped[List[str]] = mapped_column(JSON, nullable=False) + results: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + + # Relationships + pipeline: Mapped["Pipeline"] = relationship( + "Pipeline", back_populates="metrics_nodes" + ) + class ConverterList(Base): __tablename__ = "converter_list" diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index 27d7e89fa..3b5f58f79 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -135,6 +135,15 @@ def _update_context( elif node_type == "Train": context["train"] = output.get("train") pipeline.train = context["train"] + elif node_type == "SplitData": + context["split_data"] = output.get("split_data") + pipeline.split_data = context["split_data"] + elif node_type == "TaskAndModel": + context["task_and_model"] = output.get("task_and_model") + pipeline.task_and_model = context["task_and_model"] + elif node_type == "MetricsEval": + context["metrics_result"] = output.get("metrics_result") + pipeline.metrics_result = context["metrics_result"] elif node_type == "RetrieveModel": context["retrieve"] = output.get("retrieve") elif node_type == "Prediction": diff --git a/DashAI/back/pipeline/validator/nodes_definitions.py b/DashAI/back/pipeline/validator/nodes_definitions.py index 8ea712159..8d427ff7b 100644 --- a/DashAI/back/pipeline/validator/nodes_definitions.py +++ b/DashAI/back/pipeline/validator/nodes_definitions.py @@ -27,7 +27,7 @@ class NodeDefinition(BaseModel): source=True, target=False, predecessors=[], - successors=["DataExploration", "Train"], + successors=["DataExploration", "Train", "SplitData"], description=( "Loads a dataset stored in the system to be used in the pipeline." ), @@ -50,6 +50,57 @@ class NodeDefinition(BaseModel): input="Dataset object", configType="custom", ), + NodeDefinition( + type="SplitData", + name="Split Data", + icon="CallSplitIcon", + requiresConfiguration=True, + source=True, + target=True, + predecessors=["DataSelector"], + successors=["TaskAndModel"], + description=( + "Configures the partition of the dataset into " + "train / validation / test splits." + ), + input="Dataset object", + output="Split dataset subsets", + configType="custom", + ), + NodeDefinition( + type="TaskAndModel", + name="Task & Model", + icon="ModelTrainingIcon", + requiresConfiguration=True, + source=True, + target=True, + predecessors=["SplitData"], + successors=["MetricsEval", "Prediction"], + description=( + "Selects the ML task and model, configures hyper-parameters, " + "and trains the model on the split data." + ), + input="Split dataset subsets", + output="Trained model object", + configType="custom", + ), + NodeDefinition( + type="MetricsEval", + name="Metrics", + icon="AssessmentIcon", + requiresConfiguration=True, + source=True, + target=True, + predecessors=["TaskAndModel"], + successors=["Prediction"], + description=( + "Evaluates the trained model using the selected metrics " + "on each data split." + ), + input="Trained model and split data", + output="Evaluation results", + configType="custom", + ), NodeDefinition( type="Train", name="Train Model", @@ -74,7 +125,7 @@ class NodeDefinition(BaseModel): requiresConfiguration=False, source=False, target=True, - predecessors=["Train", "RetrieveModel"], + predecessors=["Train", "RetrieveModel", "TaskAndModel", "MetricsEval"], successors=[], description="Generates predictions using a trained model.", input="Trained model and dataset object", From fbfbb68843dd2b3e8c41c19a65408d9532211146 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 14:57:00 -0300 Subject: [PATCH 29/88] fix: Clean up formatting and improve readability in various node components and results handling --- .../pipeline/validator/nodes_definitions.py | 2 +- .../back/tasks/tabular_classification_task.py | 10 ++--- .../pipelines/nodes/MetricsEvalNode.jsx | 4 +- .../pipelines/nodes/SplitDataNode.jsx | 17 ++++--- .../pipelines/nodes/TaskAndModelNode.jsx | 45 ++++++++++++++----- .../components/pipelines/results/Results.jsx | 13 +++--- 6 files changed, 56 insertions(+), 35 deletions(-) diff --git a/DashAI/back/pipeline/validator/nodes_definitions.py b/DashAI/back/pipeline/validator/nodes_definitions.py index 8d427ff7b..55fce100f 100644 --- a/DashAI/back/pipeline/validator/nodes_definitions.py +++ b/DashAI/back/pipeline/validator/nodes_definitions.py @@ -94,7 +94,7 @@ class NodeDefinition(BaseModel): predecessors=["TaskAndModel"], successors=["Prediction"], description=( - "Evaluates the trained model using the selected metrics " + "Evaluates the trained model using the selected metrics" "on each data split." ), input="Trained model and split data", diff --git a/DashAI/back/tasks/tabular_classification_task.py b/DashAI/back/tasks/tabular_classification_task.py index 4255e08ab..ea9b3ef20 100644 --- a/DashAI/back/tasks/tabular_classification_task.py +++ b/DashAI/back/tasks/tabular_classification_task.py @@ -1,6 +1,6 @@ +import logging from typing import List, Union -import logging from datasets import DatasetDict from DashAI.back.core.utils import MultilingualString @@ -78,7 +78,9 @@ def prepare_for_task( if isinstance(col_type, Text): # Get unique values using dataset API unique_values = [ - value for value in dashai_dataset.unique(col) if value is not None + value + for value in dashai_dataset.unique(col) + if value is not None ] log.info( f"Converting Text column '{col}' to Categorical with " @@ -91,6 +93,4 @@ def prepare_for_task( # Now call parent's prepare_for_task for validation # This will validate that all input columns are now of allowed types - return super().prepare_for_task( - dashai_dataset, input_columns, output_columns - ) + return super().prepare_for_task(dashai_dataset, input_columns, output_columns) diff --git a/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx b/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx index 3040d02e2..4c8615c5a 100644 --- a/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx @@ -81,8 +81,8 @@ const MetricsEvalNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { {!task && ( - ⚠️ Connect a Task & Model node first so compatible metrics - can be listed. + ⚠️ Connect a Task & Model node first so compatible metrics can + be listed. )} diff --git a/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx b/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx index df6cd2c14..efe69591d 100644 --- a/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx @@ -78,9 +78,12 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { if (!datasetId) { setInfoLoading(false); if (!hasWarnedRef.current) { - enqueueSnackbar("Missing dataset – connect a DataSelector node first.", { - variant: "warning", - }); + enqueueSnackbar( + "Missing dataset – connect a DataSelector node first.", + { + variant: "warning", + }, + ); hasWarnedRef.current = true; } return; @@ -206,9 +209,7 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { onChange={(e) => setInputColumns(e.target.value)} margin="normal" error={inputError} - helperText={ - inputError ? inputErrorMessage : 'e.g. 1-4 or 1,3,5' - } + helperText={inputError ? inputErrorMessage : "e.g. 1-4 or 1,3,5"} /> @@ -220,9 +221,7 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { onChange={(e) => setOutputColumns(e.target.value)} margin="normal" error={outputError} - helperText={ - outputError ? outputErrorMessage : 'e.g. 5 or 5-6' - } + helperText={outputError ? outputErrorMessage : "e.g. 5 or 5-6"} /> diff --git a/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx index d3c6bb0fa..2ed0c566e 100644 --- a/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx @@ -26,7 +26,13 @@ import { useSnackbar } from "notistack"; * Receives from SplitData (via prevNodes): * input_columns, output_columns, splits, id (dataset id), file_path */ -const TaskAndModelNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { +const TaskAndModelNode = ({ + open, + onClose, + onSave, + savedConfig, + prevNodes, +}) => { const [task, setTask] = useState(savedConfig?.task || ""); const [model, setModel] = useState(savedConfig?.model || ""); const [openSettings, setOpenSettings] = useState(false); @@ -58,10 +64,9 @@ const TaskAndModelNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => if (!datasetId) { setInfoLoading(false); if (!hasWarnedRef.current) { - enqueueSnackbar( - "Missing dataset – connect a SplitData node first.", - { variant: "warning" }, - ); + enqueueSnackbar("Missing dataset – connect a SplitData node first.", { + variant: "warning", + }); hasWarnedRef.current = true; } return; @@ -157,8 +162,14 @@ const TaskAndModelNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => ); } else { try { - const parsedInput = parseRangeToIndex(String(inputColumnsRaw), maxValue); - const parsedOutput = parseRangeToIndex(String(outputColumnsRaw), maxValue); + const parsedInput = parseRangeToIndex( + String(inputColumnsRaw), + maxValue, + ); + const parsedOutput = parseRangeToIndex( + String(outputColumnsRaw), + maxValue, + ); inputColumnNames = parsedInput.map( (idx) => columnNames[idx - 1] || `Column_${idx}`, ); @@ -183,9 +194,17 @@ const TaskAndModelNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => if (validation.dataset_status === "valid") { return { name: taskObj.name, valid: true }; } - return { name: taskObj.name, valid: false, error: validation.error }; + return { + name: taskObj.name, + valid: false, + error: validation.error, + }; } catch (error) { - return { name: taskObj.name, valid: false, error: "Request failed" }; + return { + name: taskObj.name, + valid: false, + error: "Request failed", + }; } }), ); @@ -200,7 +219,13 @@ const TaskAndModelNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => }; validateAvailableTasks(); - }, [availableTasks, inputColumnsRaw, outputColumnsRaw, datasetId, datasetInfo]); + }, [ + availableTasks, + inputColumnsRaw, + outputColumnsRaw, + datasetId, + datasetInfo, + ]); // ---------- Save ---------- const handleSave = async () => { diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 2b6c70068..0a6acc9f1 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -102,21 +102,20 @@ function PipelineResults({ pipelineId, onClose }) { const hasAtomizedTrain = (results.task_and_model && Object.keys(results.task_and_model).length > 0) || - (results.metrics_result && - Object.keys(results.metrics_result).length > 0); + (results.metrics_result && Object.keys(results.metrics_result).length > 0); const hasPrediction = results.prediction && results.prediction !== "No prediction data"; // Derive unified values from whichever format is available const trainModelName = hasTrain ? results.train.info - : results.task_and_model?.model ?? "-"; + : (results.task_and_model?.model ?? "-"); const trainParameters = hasTrain ? results.train.parameters - : results.task_and_model?.parameters ?? null; + : (results.task_and_model?.parameters ?? null); const trainMetrics = hasTrain ? results.train.metrics - : results.metrics_result?.results ?? null; + : (results.metrics_result?.results ?? null); const paramData = { parameters: trainParameters }; @@ -197,9 +196,7 @@ function PipelineResults({ pipelineId, onClose }) { )} {trainTab === 2 && ( - + )} {trainTab === 3 && ( From 32282bddf424220af2b6115695446fd07cde24be Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 15:04:35 -0300 Subject: [PATCH 30/88] fix: Improve message formatting for ruff-format --- DashAI/back/pipeline/task_and_model_node.py | 4 +--- DashAI/back/pipeline/validator/nodes_definitions.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/DashAI/back/pipeline/task_and_model_node.py b/DashAI/back/pipeline/task_and_model_node.py index 5e78e71b1..0ffde98be 100644 --- a/DashAI/back/pipeline/task_and_model_node.py +++ b/DashAI/back/pipeline/task_and_model_node.py @@ -197,9 +197,7 @@ async def run( # `train/`. This node now expects that location to be # a directory, so replace legacy file with a directory. if os.path.isfile(path): - log.warning( - "Replacing legacy model file path with directory: %s", path - ) + log.warning("Replacing legacy model file path with directory: %s", path) os.remove(path) os.makedirs(path, exist_ok=True) diff --git a/DashAI/back/pipeline/validator/nodes_definitions.py b/DashAI/back/pipeline/validator/nodes_definitions.py index 55fce100f..b562645dc 100644 --- a/DashAI/back/pipeline/validator/nodes_definitions.py +++ b/DashAI/back/pipeline/validator/nodes_definitions.py @@ -94,8 +94,7 @@ class NodeDefinition(BaseModel): predecessors=["TaskAndModel"], successors=["Prediction"], description=( - "Evaluates the trained model using the selected metrics" - "on each data split." + "Evaluates the trained model using the selected metricson each data split." ), input="Trained model and split data", output="Evaluation results", From 3437c8aae5d899eb5e6e106a3c16f1a951859c8a Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 30 Mar 2026 16:46:23 -0300 Subject: [PATCH 31/88] fix: Update expected input types in tabular class task metadata test --- tests/back/tasks/test_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/back/tasks/test_tasks.py b/tests/back/tasks/test_tasks.py index ad5847c1f..dbcf9938c 100644 --- a/tests/back/tasks/test_tasks.py +++ b/tests/back/tasks/test_tasks.py @@ -136,7 +136,7 @@ def test_get_tabular_class_task_metadata(): metadata = tabular_class_task.get_metadata() assert len(metadata.keys()) == 4 - assert metadata["inputs_types"] == ["Float", "Integer", "Categorical"] + assert metadata["inputs_types"] == ["Float", "Integer", "Categorical", "Text"] assert metadata["outputs_types"] == ["Categorical"] assert metadata["inputs_cardinality"] == "n" assert metadata["outputs_cardinality"] == 1 From e6fee7428287ff84866776dbac87c39f1bad88a9 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Thu, 2 Apr 2026 18:44:50 -0300 Subject: [PATCH 32/88] feat: Enhance pipeline functionality with new nodes and improve type inference --- DashAI/back/api/api_v1/endpoints/datasets.py | 16 ++++++++++++++++ DashAI/back/initial_components.py | 3 +++ DashAI/back/job/pipeline_job.py | 16 ++++++++-------- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/datasets.py b/DashAI/back/api/api_v1/endpoints/datasets.py index 60e4c73d5..991338908 100644 --- a/DashAI/back/api/api_v1/endpoints/datasets.py +++ b/DashAI/back/api/api_v1/endpoints/datasets.py @@ -1,5 +1,10 @@ +import contextlib import hashlib +import json import logging +import os +import shutil +import tempfile import time import zipfile from collections import OrderedDict @@ -21,6 +26,8 @@ DatasetUpdateParams, ) from DashAI.back.dependencies.database.models import Dataset, ModelSession +from DashAI.back.types.inf.type_inference import infer_types +from DashAI.back.types.utils import arrow_to_dashai_schema if TYPE_CHECKING: from sqlalchemy.orm.session import sessionmaker @@ -1676,6 +1683,12 @@ async def load_preview( - inferred_types: Detailed type inference (DashAI types) - preview_row_count: Number of rows used for inference """ + import contextlib + import os + import shutil + import tempfile + from DashAI.back.types.utils import arrow_to_dashai_schema + try: parsed_params = json.loads(params) if params else {} @@ -1835,6 +1848,9 @@ async def infer_datatypes( Dict A dictionary containing inferred types for each column. """ + import os + import tempfile + try: parsed_params = json.loads(params) if params else {} diff --git a/DashAI/back/initial_components.py b/DashAI/back/initial_components.py index 0d83d860b..5df5082b9 100644 --- a/DashAI/back/initial_components.py +++ b/DashAI/back/initial_components.py @@ -194,8 +194,11 @@ # Pipeline nodes from DashAI.back.pipeline.data_selector_node import DataSelector from DashAI.back.pipeline.exploration_node import DataExploration +from DashAI.back.pipeline.metrics_node import MetricsEval from DashAI.back.pipeline.prediction_node import Prediction from DashAI.back.pipeline.retrieve_model_node import RetrieveModel +from DashAI.back.pipeline.split_data_node import SplitData +from DashAI.back.pipeline.task_and_model_node import TaskAndModel from DashAI.back.pipeline.train_node import Train # Plugins diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index f73526bf7..dca6082b2 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -2,12 +2,14 @@ import logging from typing import TYPE_CHECKING, Any, Dict, List -from kink import di +from kink import di, inject from DashAI.back.dependencies.database.models import Pipeline from DashAI.back.job.base_job import BaseJob, JobError if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + from sqlalchemy.orm import Session from DashAI.back.dependencies.registry import ComponentRegistry @@ -31,7 +33,7 @@ def get_job_name(self) -> str: return "Pipeline" try: - session_factory: sessionmaker = di["session_factory"] + session_factory: "sessionmaker" = di["session_factory"] with session_factory() as db: pipeline: Pipeline | None = db.get(Pipeline, pipeline_id) if pipeline and getattr(pipeline, "name", None): @@ -41,15 +43,13 @@ def get_job_name(self) -> str: return f"Pipeline ({pipeline_id})" + @inject def run( self, + session_factory: "sessionmaker" = lambda di: di["session_factory"], component_registry: "ComponentRegistry" = lambda di: di["component_registry"], ) -> None: - db: "Session" = self.kwargs["db"] - id: int = self.kwargs.get("id", None) - pipeline: Pipeline = db.get(Pipeline, id) - steps: List[Dict[str, Any]] = self.kwargs.get("steps", []) or pipeline.steps - + pipeline_id: int | None = self.kwargs.get("id") if not pipeline_id: raise JobError("No id provided to execute the pipeline.") @@ -80,7 +80,7 @@ async def _execute_steps() -> None: ) try: - node_class = component_registry(di)[node_type]["class"] + node_class = component_registry[node_type]["class"] except KeyError as e: error_msg = f"Component type {node_type} not found in registry" raise JobError( From eba51240dc87023260a9ab4fa160d09707d41747 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Thu, 2 Apr 2026 18:45:20 -0300 Subject: [PATCH 33/88] feat: Add error handling and job name methods to pipeline nodes --- DashAI/back/pipeline/data_selector_node.py | 6 ++++++ DashAI/back/pipeline/exploration_node.py | 12 +++++++++--- DashAI/back/pipeline/prediction_node.py | 6 ++++++ DashAI/back/pipeline/retrieve_model_node.py | 8 +++++++- DashAI/back/pipeline/train_node.py | 12 +++++++++--- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/DashAI/back/pipeline/data_selector_node.py b/DashAI/back/pipeline/data_selector_node.py index e3e5d7e28..1585ed451 100644 --- a/DashAI/back/pipeline/data_selector_node.py +++ b/DashAI/back/pipeline/data_selector_node.py @@ -34,6 +34,12 @@ def __init__(self, **kwargs) -> None: def set_status_as_delivered(self) -> None: log.debug("DataSelector executed successfully.") + def set_status_as_error(self) -> None: + log.error("DataSelector encountered an error.") + + def get_job_name(self) -> str: + return "DataSelector" + @inject async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: from pathlib import Path diff --git a/DashAI/back/pipeline/exploration_node.py b/DashAI/back/pipeline/exploration_node.py index 388ffafee..3b2daf5a2 100644 --- a/DashAI/back/pipeline/exploration_node.py +++ b/DashAI/back/pipeline/exploration_node.py @@ -44,6 +44,12 @@ def __init__( def set_status_as_delivered(self) -> None: log.debug("DataExploration executed successfully.") + def set_status_as_error(self) -> None: + log.error("DataExploration encountered an error.") + + def get_job_name(self) -> str: + return "DataExploration" + @inject async def run( self, @@ -74,9 +80,9 @@ async def run( explorer_id = option.get("id", str(idx)) name = option.get("name", "") - explorer_component_class: Type[BaseExplorer] = component_registry( - di - )[exploration_type]["class"] + explorer_component_class: Type[BaseExplorer] = component_registry[ + exploration_type + ]["class"] explorer_instance = explorer_component_class(**parameters) assert isinstance(explorer_instance, BaseExplorer) diff --git a/DashAI/back/pipeline/prediction_node.py b/DashAI/back/pipeline/prediction_node.py index 6f245a102..7f928c4ed 100644 --- a/DashAI/back/pipeline/prediction_node.py +++ b/DashAI/back/pipeline/prediction_node.py @@ -28,6 +28,12 @@ def __init__(self, **kwargs) -> None: def set_status_as_delivered(self) -> None: log.debug("Prediction executed successfully.") + def set_status_as_error(self) -> None: + log.error("Prediction encountered an error.") + + def get_job_name(self) -> str: + return "Prediction" + @inject async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: import json diff --git a/DashAI/back/pipeline/retrieve_model_node.py b/DashAI/back/pipeline/retrieve_model_node.py index 2e20f9187..5109033d1 100644 --- a/DashAI/back/pipeline/retrieve_model_node.py +++ b/DashAI/back/pipeline/retrieve_model_node.py @@ -39,6 +39,12 @@ def __init__( def set_status_as_delivered(self) -> None: log.debug("RetrieveModel executed successfully.") + def set_status_as_error(self) -> None: + log.error("RetrieveModel encountered an error.") + + def get_job_name(self) -> str: + return f"RetrieveModel: {self.model}" + @inject async def run( self, @@ -60,7 +66,7 @@ async def run( ) context["input_columns"] = input_columns_names - model_class = component_registry(di)[self.model]["class"] + model_class = component_registry[self.model]["class"] context["model_class"] = model_class return { diff --git a/DashAI/back/pipeline/train_node.py b/DashAI/back/pipeline/train_node.py index 144fcbd86..5920eeafa 100644 --- a/DashAI/back/pipeline/train_node.py +++ b/DashAI/back/pipeline/train_node.py @@ -70,6 +70,12 @@ def __init__( def set_status_as_delivered(self) -> None: log.debug("Train executed successfully.") + def set_status_as_error(self) -> None: + log.error("Train encountered an error.") + + def get_job_name(self) -> str: + return f"Train: {self.model}" + @inject async def run( self, @@ -90,7 +96,7 @@ async def run( context["model_name"] = self.model pipeline_id = context["pipeline_id"] dataset = context["dataset"] - task: BaseTask = component_registry(di)[self.task]["class"] + task: BaseTask = component_registry[self.task]["class"] task_instance = task() input_columns_names = get_column_names_from_indexes(dataset, self.input_columns) @@ -101,7 +107,7 @@ async def run( all_metrics = { component_dict["name"]: component_dict - for component_dict in component_registry(di).get_components_by_types( + for component_dict in component_registry.get_components_by_types( select="Metric" ) } @@ -148,7 +154,7 @@ async def run( ) from e try: - model_class = component_registry(di)[self.model]["class"] + model_class = component_registry[self.model]["class"] context["model_class"] = model_class except Exception as e: log.exception(e) From 97b5ff59c3ab0c2d9d8ed500645e808140125c85 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Thu, 2 Apr 2026 18:45:49 -0300 Subject: [PATCH 34/88] feat: Add pipelines navigation and button to home page --- DashAI/front/src/components/ResponsiveAppBar.jsx | 1 + DashAI/front/src/pages/home/Home.jsx | 10 ++++++++-- tests/back/tasks/test_tasks.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/DashAI/front/src/components/ResponsiveAppBar.jsx b/DashAI/front/src/components/ResponsiveAppBar.jsx index 501664a2f..718588033 100644 --- a/DashAI/front/src/components/ResponsiveAppBar.jsx +++ b/DashAI/front/src/components/ResponsiveAppBar.jsx @@ -31,6 +31,7 @@ function ResponsiveAppBar() { const pages = [ { name: t("common:datasets"), to: "/app/data", disabled: false }, { name: t("common:models"), to: "/app/models", disabled: false }, + { name: t("common:pipelines"), to: "/app/pipelines", disabled: false }, { name: t("common:generative"), to: "/app/generative", disabled: false }, { name: t("common:plugins"), to: "/app/plugins/browse", disabled: false }, ]; diff --git a/DashAI/front/src/pages/home/Home.jsx b/DashAI/front/src/pages/home/Home.jsx index e5eded4cf..8db11e46c 100644 --- a/DashAI/front/src/pages/home/Home.jsx +++ b/DashAI/front/src/pages/home/Home.jsx @@ -4,9 +4,7 @@ import { FileUpload as FileUploadIcon, Science as ScienceIcon, Extension as ExtensionIcon, - Insights as InsightsIcon, Merge as MergeIcon, - Timeline as TimelineIcon, AutoAwesome as AutoAwesomeIcon, } from "@mui/icons-material"; import HomeButton from "../../components/HomeButton"; @@ -57,6 +55,14 @@ function Home() { Icon={ScienceIcon} /> + + + Date: Fri, 3 Apr 2026 16:25:02 -0300 Subject: [PATCH 35/88] feat: Implement dynamic argument handling for save_notebook in DataExploration --- DashAI/back/pipeline/exploration_node.py | 38 ++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/DashAI/back/pipeline/exploration_node.py b/DashAI/back/pipeline/exploration_node.py index 3b2daf5a2..501c6fcc8 100644 --- a/DashAI/back/pipeline/exploration_node.py +++ b/DashAI/back/pipeline/exploration_node.py @@ -57,6 +57,7 @@ async def run( component_registry: "ComponentRegistry" = lambda di: di["component_registry"], ) -> Dict[str, Any]: try: + import inspect import os from pathlib import Path @@ -115,12 +116,39 @@ def __init__( prepared_dataset, explorer_info ) - save_path = explorer_instance.save_notebook( - __notebook_info__=None, - explorer_info=explorer_info, - save_path=base_path, - result=result, + save_notebook_sig = inspect.signature( + explorer_instance.save_notebook ) + save_notebook_params = set(save_notebook_sig.parameters) + + notebook_info_arg_name = None + for candidate in ( + "__notebook_info__", + "__exploration_info__", + "notebook_info", + ): + if candidate in save_notebook_params: + notebook_info_arg_name = candidate + break + + save_notebook_kwargs = { + "explorer_info": explorer_info, + "save_path": base_path, + "result": result, + } + if notebook_info_arg_name is not None: + save_notebook_kwargs[notebook_info_arg_name] = None + save_path = explorer_instance.save_notebook( + **save_notebook_kwargs + ) + else: + # Fallback to positional first arg for legacy implementations. + save_path = explorer_instance.save_notebook( + None, + explorer_info=explorer_info, + save_path=base_path, + result=result, + ) results[str(explorer_id)] = { "exploration_type": exploration_type, From 0f2125ad9b83ca4eb955b26f183cba9ee119af92 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 3 Apr 2026 16:25:12 -0300 Subject: [PATCH 36/88] feat: Add migration script to merge dag-pipelines and develop heads --- ...3_merge_dag_pipelines_and_develop_heads.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 DashAI/alembic/versions/0dc6fd4637a3_merge_dag_pipelines_and_develop_heads.py diff --git a/DashAI/alembic/versions/0dc6fd4637a3_merge_dag_pipelines_and_develop_heads.py b/DashAI/alembic/versions/0dc6fd4637a3_merge_dag_pipelines_and_develop_heads.py new file mode 100644 index 000000000..1e34682f2 --- /dev/null +++ b/DashAI/alembic/versions/0dc6fd4637a3_merge_dag_pipelines_and_develop_heads.py @@ -0,0 +1,26 @@ +"""merge dag-pipelines and develop heads + +Revision ID: 0dc6fd4637a3 +Revises: b4f9e70098e7, c4f7a8b2d1e3 +Create Date: 2026-04-02 19:32:55.182155 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '0dc6fd4637a3' +down_revision: Union[str, None] = ('b4f9e70098e7', 'c4f7a8b2d1e3') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass From 31a8109ced3d0e294bd21d3a647129aea03b9259 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sat, 4 Apr 2026 19:00:55 -0300 Subject: [PATCH 37/88] feat: Update created and last_modified columns to be non-nullable in upgrade migration --- ...1e3_atomize_train_node_into_split_task_metrics.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py b/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py index 23f66c6cb..426580b6d 100644 --- a/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py +++ b/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py @@ -52,8 +52,8 @@ def upgrade() -> None: sa.Column("input_columns", sa.JSON(), nullable=False), sa.Column("output_columns", sa.JSON(), nullable=False), sa.Column("splits", sa.JSON(), nullable=False), - sa.Column("created", sa.DateTime(), nullable=True), - sa.Column("last_modified", sa.DateTime(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), ) @@ -67,8 +67,8 @@ def upgrade() -> None: sa.Column("model_name", sa.String(), nullable=False), sa.Column("parameters", sa.JSON(), nullable=True), sa.Column("model_path", sa.String(), nullable=True), - sa.Column("created", sa.DateTime(), nullable=True), - sa.Column("last_modified", sa.DateTime(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), ) @@ -80,8 +80,8 @@ def upgrade() -> None: sa.Column("pipeline_id", sa.Integer(), nullable=False), sa.Column("metric_names", sa.JSON(), nullable=False), sa.Column("results", sa.JSON(), nullable=True), - sa.Column("created", sa.DateTime(), nullable=True), - sa.Column("last_modified", sa.DateTime(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), ) From 89dab864ec7bccd11d589ac4f41a298c3be5f5f3 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sat, 4 Apr 2026 19:25:42 -0300 Subject: [PATCH 38/88] refactor: Remove unused imports across multiple files --- DashAI/back/api/api_v1/endpoints/datasets.py | 16 ---------------- DashAI/back/dependencies/database/migrate.py | 1 - DashAI/back/job/pipeline_job.py | 2 -- DashAI/back/pipeline/exploration_node.py | 3 +-- DashAI/back/pipeline/retrieve_model_node.py | 2 +- scripts/ci_alembic_check.py | 3 ++- 6 files changed, 4 insertions(+), 23 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/datasets.py b/DashAI/back/api/api_v1/endpoints/datasets.py index 991338908..ac30a3685 100644 --- a/DashAI/back/api/api_v1/endpoints/datasets.py +++ b/DashAI/back/api/api_v1/endpoints/datasets.py @@ -450,7 +450,6 @@ async def get_sample( Dict A Dict with a sample of 10 rows """ - import os import pyarrow as pa import pyarrow.ipc as ipc @@ -519,7 +518,6 @@ async def get_sample_by_file( Dict A Dict with a sample of 10 rows """ - import os import pyarrow as pa import pyarrow.ipc as ipc @@ -810,7 +808,6 @@ async def copy_dataset( Dataset The newly created dataset. """ - import shutil from DashAI.back.core.enums.status import DatasetStatus @@ -1023,7 +1020,6 @@ async def rename_dataset_column( A dictionary with a success message and updated column types. """ import json - import os import pyarrow as pa import pyarrow.ipc as ipc @@ -1300,7 +1296,6 @@ async def export_dataset_as_csv( StreamingResponse A streaming response with the complete dataset in CSV format. """ - import os import pyarrow as pa import pyarrow.csv as csv @@ -1384,7 +1379,6 @@ async def export_dataset_csv_by_id( StreamingResponse A streaming response that provides the CSV file content. """ - import os import pyarrow as pa import pyarrow.csv as csv @@ -1494,17 +1488,14 @@ async def preview_with_types( - inferred_types: Detailed type inference (DashAI types) - preview_row_count: Number of rows used for inference """ - import contextlib import json import os import shutil - import tempfile import numpy as np import pyarrow as pa from DashAI.back.types.inf.type_inference import infer_types - from DashAI.back.types.utils import arrow_to_dashai_schema try: parsed_params = json.loads(params) if params else {} @@ -1683,11 +1674,8 @@ async def load_preview( - inferred_types: Detailed type inference (DashAI types) - preview_row_count: Number of rows used for inference """ - import contextlib import os import shutil - import tempfile - from DashAI.back.types.utils import arrow_to_dashai_schema try: parsed_params = json.loads(params) if params else {} @@ -1848,8 +1836,6 @@ async def infer_datatypes( Dict A dictionary containing inferred types for each column. """ - import os - import tempfile try: parsed_params = json.loads(params) if params else {} @@ -1928,8 +1914,6 @@ async def validate_type_changes( Validate proposed type changes for dataset columns. """ import json - import os - import tempfile from DashAI.back.types.type_validation import validate_multiple_type_changes diff --git a/DashAI/back/dependencies/database/migrate.py b/DashAI/back/dependencies/database/migrate.py index ffcbda073..4b18202b6 100644 --- a/DashAI/back/dependencies/database/migrate.py +++ b/DashAI/back/dependencies/database/migrate.py @@ -6,7 +6,6 @@ from alembic import command from alembic.config import Config - from DashAI.back.dependencies.database.utils import resolve_db_url logger = logging.getLogger(__name__) diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index dca6082b2..f48dcd99b 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -10,8 +10,6 @@ if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker - from sqlalchemy.orm import Session - from DashAI.back.dependencies.registry import ComponentRegistry log = logging.getLogger(__name__) diff --git a/DashAI/back/pipeline/exploration_node.py b/DashAI/back/pipeline/exploration_node.py index 501c6fcc8..2941be09c 100644 --- a/DashAI/back/pipeline/exploration_node.py +++ b/DashAI/back/pipeline/exploration_node.py @@ -1,10 +1,9 @@ import logging from typing import TYPE_CHECKING, Any, Dict, Type -from kink import di, inject +from kink import inject from DashAI.back.config import DefaultSettings -from DashAI.back.dependencies.database.models import Explorer from DashAI.back.exploration.base_explorer import BaseExplorer from DashAI.back.job.base_job import BaseJob, JobError diff --git a/DashAI/back/pipeline/retrieve_model_node.py b/DashAI/back/pipeline/retrieve_model_node.py index 5109033d1..8f6d9fe00 100644 --- a/DashAI/back/pipeline/retrieve_model_node.py +++ b/DashAI/back/pipeline/retrieve_model_node.py @@ -1,7 +1,7 @@ import logging from typing import TYPE_CHECKING, Any, Dict, List -from kink import di, inject +from kink import inject from DashAI.back.job.base_job import BaseJob, JobError diff --git a/scripts/ci_alembic_check.py b/scripts/ci_alembic_check.py index 84dbdd7bc..29caf15c6 100644 --- a/scripts/ci_alembic_check.py +++ b/scripts/ci_alembic_check.py @@ -2,9 +2,10 @@ import os import sys +from sqlalchemy import create_engine + from alembic.autogenerate import compare_metadata from alembic.migration import MigrationContext -from sqlalchemy import create_engine try: from DashAI.back.dependencies.database.models import Base From a4c629bb63675a4520541cbd138347d2ff43440c Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 5 Apr 2026 20:30:43 -0400 Subject: [PATCH 39/88] refactor: Reorder import statements for consistency in migrate.py and ci_alembic_check.py --- DashAI/back/dependencies/database/migrate.py | 3 ++- scripts/ci_alembic_check.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DashAI/back/dependencies/database/migrate.py b/DashAI/back/dependencies/database/migrate.py index 4b18202b6..cbc37fe59 100644 --- a/DashAI/back/dependencies/database/migrate.py +++ b/DashAI/back/dependencies/database/migrate.py @@ -4,8 +4,9 @@ import time from pathlib import Path -from alembic import command from alembic.config import Config + +from alembic import command from DashAI.back.dependencies.database.utils import resolve_db_url logger = logging.getLogger(__name__) diff --git a/scripts/ci_alembic_check.py b/scripts/ci_alembic_check.py index 29caf15c6..84dbdd7bc 100644 --- a/scripts/ci_alembic_check.py +++ b/scripts/ci_alembic_check.py @@ -2,10 +2,9 @@ import os import sys -from sqlalchemy import create_engine - from alembic.autogenerate import compare_metadata from alembic.migration import MigrationContext +from sqlalchemy import create_engine try: from DashAI.back.dependencies.database.models import Base From d293f48009f655c8192c47bec69ec9d8037c5a27 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 5 Apr 2026 20:55:38 -0400 Subject: [PATCH 40/88] refactor: Reorder import statements for clarity in migrate.py --- DashAI/back/dependencies/database/migrate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DashAI/back/dependencies/database/migrate.py b/DashAI/back/dependencies/database/migrate.py index cbc37fe59..4b18202b6 100644 --- a/DashAI/back/dependencies/database/migrate.py +++ b/DashAI/back/dependencies/database/migrate.py @@ -4,9 +4,8 @@ import time from pathlib import Path -from alembic.config import Config - from alembic import command +from alembic.config import Config from DashAI.back.dependencies.database.utils import resolve_db_url logger = logging.getLogger(__name__) From 46e0c0674c6d7086e0d0d10fb0b71dd669700bd5 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 5 Apr 2026 21:23:24 -0400 Subject: [PATCH 41/88] refactor: Move import statement for create_engine to improve organization in ci_alembic_check.py --- scripts/ci_alembic_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci_alembic_check.py b/scripts/ci_alembic_check.py index 84dbdd7bc..29caf15c6 100644 --- a/scripts/ci_alembic_check.py +++ b/scripts/ci_alembic_check.py @@ -2,9 +2,10 @@ import os import sys +from sqlalchemy import create_engine + from alembic.autogenerate import compare_metadata from alembic.migration import MigrationContext -from sqlalchemy import create_engine try: from DashAI.back.dependencies.database.models import Base From 6e38d39e2ba64f68792a42fec2de58884259054b Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 5 Apr 2026 22:14:10 -0400 Subject: [PATCH 42/88] refactor: Reorder import statements for consistency in migrate.py and ci_alembic_check.py --- DashAI/back/dependencies/database/migrate.py | 3 ++- scripts/ci_alembic_check.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DashAI/back/dependencies/database/migrate.py b/DashAI/back/dependencies/database/migrate.py index 4b18202b6..cbc37fe59 100644 --- a/DashAI/back/dependencies/database/migrate.py +++ b/DashAI/back/dependencies/database/migrate.py @@ -4,8 +4,9 @@ import time from pathlib import Path -from alembic import command from alembic.config import Config + +from alembic import command from DashAI.back.dependencies.database.utils import resolve_db_url logger = logging.getLogger(__name__) diff --git a/scripts/ci_alembic_check.py b/scripts/ci_alembic_check.py index 29caf15c6..84dbdd7bc 100644 --- a/scripts/ci_alembic_check.py +++ b/scripts/ci_alembic_check.py @@ -2,10 +2,9 @@ import os import sys -from sqlalchemy import create_engine - from alembic.autogenerate import compare_metadata from alembic.migration import MigrationContext +from sqlalchemy import create_engine try: from DashAI.back.dependencies.database.models import Base From d6c3322e97a7d84938d49c7af6640fa2721db333 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Sun, 5 Apr 2026 22:37:39 -0400 Subject: [PATCH 43/88] refactor: Reorder import statements for clarity in migrate.py --- DashAI/back/dependencies/database/migrate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DashAI/back/dependencies/database/migrate.py b/DashAI/back/dependencies/database/migrate.py index cbc37fe59..4b18202b6 100644 --- a/DashAI/back/dependencies/database/migrate.py +++ b/DashAI/back/dependencies/database/migrate.py @@ -4,9 +4,8 @@ import time from pathlib import Path -from alembic.config import Config - from alembic import command +from alembic.config import Config from DashAI.back.dependencies.database.utils import resolve_db_url logger = logging.getLogger(__name__) From 5151537efd7f1f92d0aec7cbf3cdebba3b7d00bc Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 8 May 2026 16:56:46 -0400 Subject: [PATCH 44/88] refactor: branch-aware execution and per-node outputs --- DashAI/back/api/api_v1/endpoints/pipelines.py | 106 ++++++++- DashAI/back/job/pipeline_job.py | 211 ++++++++++++++++-- DashAI/back/pipeline/data_selector_node.py | 4 +- DashAI/back/pipeline/exploration_node.py | 7 +- DashAI/back/pipeline/prediction_node.py | 38 +++- DashAI/back/pipeline/split_data_node.py | 11 +- DashAI/back/pipeline/task_and_model_node.py | 7 +- DashAI/back/pipeline/train_node.py | 8 +- 8 files changed, 354 insertions(+), 38 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/pipelines.py b/DashAI/back/api/api_v1/endpoints/pipelines.py index 5c71b707a..572b64e58 100644 --- a/DashAI/back/api/api_v1/endpoints/pipelines.py +++ b/DashAI/back/api/api_v1/endpoints/pipelines.py @@ -280,9 +280,46 @@ async def get_pipeline_dataexploration_results( detail="Pipeline has no valid Data Exploration step", ) - results = {} - - for exploration_id, exploration_info in dataexploration.items(): + grouped_results = {} + steps = pipeline.steps or [] + edges = pipeline.edges or [] + + step_map = { + step.get("id"): step + for step in steps + if isinstance(step, dict) and step.get("id") + } + predecessor_map = {} + for edge in edges: + source = edge.get("source") if isinstance(edge, dict) else None + target = edge.get("target") if isinstance(edge, dict) else None + if source and target: + predecessor_map.setdefault(target, []).append(source) + + def resolve_dataset_name_from_graph(node_id: str | None) -> str | None: + if not node_id: + return None + + current = node_id + visited = set() + while current and current not in visited: + visited.add(current) + step = step_map.get(current) + if isinstance(step, dict) and step.get("type") == "DataSelector": + config = step.get("config") or {} + return config.get("name") + + predecessors = predecessor_map.get(current) or [] + current = predecessors[0] if predecessors else None + + return None + + def append_result( + dataset_name: str, + node_id: str, + exploration_id: str, + exploration_info: dict, + ) -> None: exploration_type = exploration_info["exploration_type"] exploration_path = exploration_info["path"] parameters = exploration_info.get("parameters", {}) @@ -302,11 +339,15 @@ async def get_pipeline_dataexploration_results( exploration_path=exploration_path, options={}, ) - results[exploration_id] = { - "exploration_type": exploration_type, - "results": result, - "name": name, - } + grouped_results.setdefault(dataset_name, []).append( + { + "node_id": node_id, + "exploration_id": exploration_id, + "exploration_type": exploration_type, + "results": result, + "name": name, + } + ) except Exception as e: logger.exception(e) raise HTTPException( @@ -314,7 +355,54 @@ async def get_pipeline_dataexploration_results( detail=(f"Error while getting results for '{exploration_type}'"), ) from e - return results + for node_or_exploration_id, node_or_exploration_info in dataexploration.items(): + if not isinstance(node_or_exploration_info, dict): + continue + + if "explorations" in node_or_exploration_info: + inferred_dataset_name = resolve_dataset_name_from_graph( + node_or_exploration_id + ) + dataset_name = ( + node_or_exploration_info.get("dataset_name") + or inferred_dataset_name + or "Unknown Dataset" + ) + node_explorations = node_or_exploration_info.get("explorations") or {} + if not isinstance(node_explorations, dict): + continue + + for exploration_id, exploration_info in node_explorations.items(): + if not isinstance(exploration_info, dict): + continue + append_result( + dataset_name=dataset_name, + node_id=node_or_exploration_id, + exploration_id=exploration_id, + exploration_info=exploration_info, + ) + continue + + dataexploration_nodes = [ + step + for step in steps + if isinstance(step, dict) and step.get("type") == "DataExploration" + ] + inferred_legacy_node_id = ( + dataexploration_nodes[-1].get("id") if dataexploration_nodes else "legacy" + ) + inferred_legacy_dataset_name = ( + resolve_dataset_name_from_graph(inferred_legacy_node_id) + or "Unknown Dataset" + ) + append_result( + dataset_name=inferred_legacy_dataset_name, + node_id=inferred_legacy_node_id or "legacy", + exploration_id=node_or_exploration_id, + exploration_info=node_or_exploration_info, + ) + + return grouped_results @router.post("/", status_code=status.HTTP_201_CREATED) diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index f48dcd99b..fe8d026a1 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -59,17 +59,31 @@ def run( steps: List[Dict[str, Any]] = self.kwargs.get("steps", []) or ( pipeline.steps or [] ) + edges: List[Dict[str, Any]] = self.kwargs.get("edges", []) or ( + pipeline.edges or [] + ) if not steps: raise JobError("No steps provided to execute the pipeline.") + pipeline.exploration = None + pipeline.train = None + pipeline.prediction = None + pipeline.split_data = None + pipeline.task_and_model = None + pipeline.metrics_result = None + log.info(f"Starting pipeline execution for pipeline {pipeline_id}...") - context: Dict[str, Any] = {"pipeline_id": pipeline_id} + base_context: Dict[str, Any] = {"pipeline_id": pipeline_id} + node_contexts: Dict[str, Dict[str, Any]] = {} + predecessor_map = self._build_predecessor_map(steps, edges) async def _execute_steps() -> None: for idx, step in enumerate(steps): node_id = step.get("id") node_type = step.get("type") node_config = step.get("config", {}) + predecessor_id = None + node_context = None log.debug( f"Pipeline {pipeline_id}: " @@ -93,13 +107,58 @@ async def _execute_steps() -> None: raise JobError(error_msg) from e try: - output = await node_instance.run(context=context) + predecessors = predecessor_map.get(node_id, []) + predecessor_id = predecessors[0] if predecessors else None + node_context = self._build_node_context( + node_id=node_id, + predecessor_map=predecessor_map, + node_contexts=node_contexts, + base_context=base_context, + ) + node_context["_node_id"] = node_id + if node_context.get("dataset") is not None: + dataset_name = node_context.get("dataset_name") + try: + dataset_cols = len( + list(node_context["dataset"].features.keys()) + ) + except Exception: + dataset_cols = "" + log.debug( + "Node %s (%s) context from predecessor=%s, " + "dataset_name=%s, dataset_columns=%s", + node_id, + node_type, + predecessor_id, + dataset_name, + dataset_cols, + ) + output = await node_instance.run(context=node_context) self._update_context( - context, pipeline, node_type, node_id, output + node_context, pipeline, node_type, node_id, output ) + if node_id: + node_contexts[node_id] = node_context log.debug(f"Node {node_id} executed successfully.") except Exception as e: - error_msg = f"Error in node {node_id} ({node_type}): {str(e)}" + dataset_name = "" + dataset_cols = "" + if isinstance(node_context, dict): + dataset_name = node_context.get("dataset_name", "") + if node_context.get("dataset") is not None: + try: + dataset_cols = len( + list(node_context["dataset"].features.keys()) + ) + except Exception: + dataset_cols = "" + + error_msg = ( + f"Error in node {node_id} ({node_type}): {str(e)} " + f"[predecessor={predecessor_id}, " + f"dataset_name={dataset_name}, " + f"dataset_columns={dataset_cols}]" + ) log.exception(error_msg) raise JobError(error_msg) from e @@ -111,6 +170,53 @@ async def _execute_steps() -> None: db.commit() self.set_status_as_delivered() + def _build_predecessor_map( + self, + steps: List[Dict[str, Any]], + edges: List[Dict[str, Any]], + ) -> Dict[str, List[str]]: + """Build a mapping target_node_id -> [source_node_ids].""" + step_ids = {step.get("id") for step in steps if step.get("id")} + predecessor_map: Dict[str, List[str]] = {step_id: [] for step_id in step_ids} + + for edge in edges: + source = edge.get("source") + target = edge.get("target") + if source in step_ids and target in step_ids: + predecessor_map[target].append(source) + + return predecessor_map + + def _build_node_context( + self, + node_id: str, + predecessor_map: Dict[str, List[str]], + node_contexts: Dict[str, Dict[str, Any]], + base_context: Dict[str, Any], + ) -> Dict[str, Any]: + """Build execution context for a node from its unique predecessor branch.""" + if not node_id: + raise JobError("Pipeline step is missing an id.") + + predecessors = predecessor_map.get(node_id, []) + if len(predecessors) > 1: + raise JobError( + f"Node {node_id} has multiple inputs, which is not supported." + ) + + if not predecessors: + return dict(base_context) + + predecessor_id = predecessors[0] + predecessor_context = node_contexts.get(predecessor_id) + if predecessor_context is None: + raise JobError( + f"Cannot resolve predecessor context for node {node_id} " + f"from source {predecessor_id}." + ) + + return dict(predecessor_context) + def _update_context( self, context: Dict[str, Any], @@ -131,25 +237,102 @@ def _update_context( """ if node_type == "DataSelector": context["dataset"] = output.get("dataset") + context["dataset_name"] = output.get( + "dataset_name", context.get("dataset_name") + ) elif node_type == "DataExploration": - context["exploration"] = output.get("exploration") - pipeline.exploration = context["exploration"] + current_node_exploration = output.get("exploration") or {} + context["exploration"] = current_node_exploration + + aggregated_exploration = ( + dict(pipeline.exploration) + if isinstance(pipeline.exploration, dict) + else {} + ) + exploration_key = node_id or f"DataExploration-{len(aggregated_exploration)}" + aggregated_exploration[exploration_key] = { + "dataset_name": context.get("dataset_name") + or output.get("dataset_name") + or "Unknown Dataset", + "explorations": current_node_exploration, + } + pipeline.exploration = aggregated_exploration elif node_type == "Train": - context["train"] = output.get("train") - pipeline.train = context["train"] + current_train = output.get("train") or {} + context["train"] = current_train + context["model_node_id"] = node_id + + aggregated_train = ( + dict(pipeline.train) if isinstance(pipeline.train, dict) else {} + ) + train_key = node_id or f"Train-{len(aggregated_train)}" + aggregated_train[train_key] = { + "node_id": node_id, + "model_node_id": node_id, + "dataset_name": context.get("dataset_name"), + **current_train, + } + pipeline.train = aggregated_train elif node_type == "SplitData": context["split_data"] = output.get("split_data") pipeline.split_data = context["split_data"] elif node_type == "TaskAndModel": - context["task_and_model"] = output.get("task_and_model") - pipeline.task_and_model = context["task_and_model"] + current_task_and_model = output.get("task_and_model") or {} + context["task_and_model"] = current_task_and_model + context["model_node_id"] = node_id + + aggregated_task_and_model = ( + dict(pipeline.task_and_model) + if isinstance(pipeline.task_and_model, dict) + else {} + ) + task_model_key = node_id or f"TaskAndModel-{len(aggregated_task_and_model)}" + aggregated_task_and_model[task_model_key] = { + "node_id": node_id, + "model_node_id": node_id, + "dataset_name": context.get("dataset_name"), + **current_task_and_model, + } + pipeline.task_and_model = aggregated_task_and_model elif node_type == "MetricsEval": - context["metrics_result"] = output.get("metrics_result") - pipeline.metrics_result = context["metrics_result"] + current_metrics_result = output.get("metrics_result") or {} + context["metrics_result"] = current_metrics_result + + aggregated_metrics_result = ( + dict(pipeline.metrics_result) + if isinstance(pipeline.metrics_result, dict) + else {} + ) + metrics_key = node_id or f"MetricsEval-{len(aggregated_metrics_result)}" + aggregated_metrics_result[metrics_key] = { + "node_id": node_id, + "model_node_id": context.get("model_node_id"), + "dataset_name": context.get("dataset_name"), + "model_name": context.get("model_name"), + "task_name": context.get("task_name"), + **current_metrics_result, + } + pipeline.metrics_result = aggregated_metrics_result elif node_type == "RetrieveModel": context["retrieve"] = output.get("retrieve") elif node_type == "Prediction": - context["prediction"] = output.get("prediction") - pipeline.prediction = context["prediction"] + current_prediction = output.get("prediction") + context["prediction"] = current_prediction + + aggregated_prediction = ( + dict(pipeline.prediction) + if isinstance(pipeline.prediction, dict) + else {} + ) + prediction_key = node_id or f"Prediction-{len(aggregated_prediction)}" + aggregated_prediction[prediction_key] = { + "node_id": node_id, + "model_node_id": context.get("model_node_id"), + "dataset_name": context.get("dataset_name"), + "model_name": context.get("model_name"), + "task_name": context.get("task_name"), + "prediction": current_prediction, + } + pipeline.prediction = aggregated_prediction else: context[node_id] = output diff --git a/DashAI/back/pipeline/data_selector_node.py b/DashAI/back/pipeline/data_selector_node.py index 1585ed451..78750b257 100644 --- a/DashAI/back/pipeline/data_selector_node.py +++ b/DashAI/back/pipeline/data_selector_node.py @@ -46,7 +46,8 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: from DashAI.back.dataloaders.classes.dashai_dataset import load_dataset - context["dataset_name"] = self.kwargs["name"] + dataset_name = self.kwargs["name"] + context["dataset_name"] = dataset_name dataset_dir = Path(self.kwargs["file_path"]) data_path = dataset_dir / "dataset/data.arrow" @@ -63,4 +64,5 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: return { "dataset": loaded_dataset, + "dataset_name": dataset_name, } diff --git a/DashAI/back/pipeline/exploration_node.py b/DashAI/back/pipeline/exploration_node.py index 2941be09c..e8d6a14c3 100644 --- a/DashAI/back/pipeline/exploration_node.py +++ b/DashAI/back/pipeline/exploration_node.py @@ -61,12 +61,17 @@ async def run( from pathlib import Path pipeline_id = context.get("pipeline_id") + node_id = context.get("_node_id", "DataExploration") loaded_dataset = context.get("dataset") settings = DefaultSettings() sqlite_local = os.path.expanduser(settings.LOCAL_PATH) base_path = ( - Path(sqlite_local) / "pipelines" / "exploration" / str(pipeline_id) + Path(sqlite_local) + / "pipelines" + / "exploration" + / str(pipeline_id) + / str(node_id) ) base_path.mkdir(parents=True, exist_ok=True) diff --git a/DashAI/back/pipeline/prediction_node.py b/DashAI/back/pipeline/prediction_node.py index 7f928c4ed..1509f7c6a 100644 --- a/DashAI/back/pipeline/prediction_node.py +++ b/DashAI/back/pipeline/prediction_node.py @@ -43,25 +43,46 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: loaded_dataset = context["dataset"] model = context["model_class"] - model_path = context["model_path"] - trained_model: BaseModel = model.load(model_path) config = di["config"] + sqlite_local = config["LOCAL_PATH"] + pipeline_id = context.get("pipeline_id") + model_node_id = context.get("model_node_id") + + model_path = context["model_path"] + if pipeline_id is not None and model_node_id: + node_model_path = os.path.join( + sqlite_local, + "pipelines", + "train", + str(pipeline_id), + str(model_node_id), + "model", + ) + if os.path.exists(node_model_path): + model_path = node_model_path + + try: + trained_model: BaseModel = model.load(model_path) + except Exception as e: + log.exception(e) + raise JobError( + "Failed to load trained model for Prediction. " + f"model_name={context.get('model_name')}, " + f"model_path={model_path}, " + f"model_node_id={model_node_id}, " + f"pipeline_id={pipeline_id}" + ) from e + try: prepared_dataset = loaded_dataset.select_columns(context["input_columns"]) y_pred_proba = np.array(trained_model.predict(prepared_dataset)) - # Check if it's classification (2D array with probabilities) - # or regression (1D array) if isinstance(y_pred_proba[0], str): - # Text/categorical predictions y_pred = y_pred_proba elif y_pred_proba.ndim == 2: - # Classification: multi-class probabilities, - # get class with highest probability y_pred = np.argmax(y_pred_proba, axis=1) else: - # Regression: predictions are already the final values y_pred = y_pred_proba except ValueError as ve: @@ -77,7 +98,6 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: ) from e try: - sqlite_local = config["LOCAL_PATH"] path = os.path.join(sqlite_local, "pipelines", "predictions") os.makedirs(path, exist_ok=True) existing_ids = [] diff --git a/DashAI/back/pipeline/split_data_node.py b/DashAI/back/pipeline/split_data_node.py index 860f5cdc2..31a707312 100644 --- a/DashAI/back/pipeline/split_data_node.py +++ b/DashAI/back/pipeline/split_data_node.py @@ -82,11 +82,18 @@ async def run( ) except Exception as e: log.exception(e) + dataset_name = context.get("dataset_name", "") + try: + total_columns = len(list(dataset.features.keys())) + except Exception: + total_columns = "" raise JobError( - "Cannot resolve column names from the dataset.", + "Cannot resolve column names from the dataset. " + f"dataset_name={dataset_name}, total_columns={total_columns}, " + f"input_columns={self.input_columns}, " + f"output_columns={self.output_columns}", ) from e - # Populate context for downstream nodes context["input_columns"] = input_columns_names context["output_columns"] = output_columns_names context["input_columns_idx"] = self.input_columns diff --git a/DashAI/back/pipeline/task_and_model_node.py b/DashAI/back/pipeline/task_and_model_node.py index 0ffde98be..1b9627fe6 100644 --- a/DashAI/back/pipeline/task_and_model_node.py +++ b/DashAI/back/pipeline/task_and_model_node.py @@ -191,6 +191,8 @@ async def run( train_root = os.path.join(sqlite_local, "pipelines", "train") os.makedirs(train_root, exist_ok=True) + node_id = context.get("_node_id", "TaskAndModel") + path = os.path.join(train_root, str(pipeline_id)) # Backward-compatibility: older runs may have saved a file in @@ -201,7 +203,10 @@ async def run( os.remove(path) os.makedirs(path, exist_ok=True) - model_path = os.path.join(path, "model") + + node_path = os.path.join(path, str(node_id)) + os.makedirs(node_path, exist_ok=True) + model_path = os.path.join(node_path, "model") # Ensure clean overwrite when retraining same pipeline id. if os.path.isdir(model_path): diff --git a/DashAI/back/pipeline/train_node.py b/DashAI/back/pipeline/train_node.py index 5920eeafa..4de50d5a3 100644 --- a/DashAI/back/pipeline/train_node.py +++ b/DashAI/back/pipeline/train_node.py @@ -219,7 +219,13 @@ async def run( sqlite_local = config["LOCAL_PATH"] path = os.path.join(sqlite_local, "pipelines", "train") os.makedirs(path, exist_ok=True) - model_path = os.path.join(path, str(pipeline_id)) + + node_id = context.get("_node_id", "Train") + pipeline_path = os.path.join(path, str(pipeline_id)) + os.makedirs(pipeline_path, exist_ok=True) + model_path = os.path.join(pipeline_path, str(node_id), "model") + + os.makedirs(os.path.dirname(model_path), exist_ok=True) model.save(model_path) context["model_path"] = model_path except Exception as e: From c71a6e610a410e57cecd728e51dca9288abf3922 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 8 May 2026 16:57:37 -0400 Subject: [PATCH 45/88] refactor: update NodeDefinition to include sourceHandles and maxOutputs; remove duplicate validation logic in PipelineValidator --- .../pipeline/validator/nodes_definitions.py | 12 +++++-- .../pipeline/validator/pipeline_validator.py | 33 +++++-------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/DashAI/back/pipeline/validator/nodes_definitions.py b/DashAI/back/pipeline/validator/nodes_definitions.py index b562645dc..2e1cc42c0 100644 --- a/DashAI/back/pipeline/validator/nodes_definitions.py +++ b/DashAI/back/pipeline/validator/nodes_definitions.py @@ -7,7 +7,7 @@ class NodeDefinition(BaseModel): type: str name: str icon: str - requiresConfiguration: bool # noqa: N815 + requiresConfiguration: bool source: bool target: bool predecessors: List[str] @@ -15,7 +15,9 @@ class NodeDefinition(BaseModel): description: str input: Optional[str] = None output: Optional[str] = None - configType: str # noqa: N815 + sourceHandles: int = 1 + maxOutputs: Optional[int] = None + configType: str NODES: List[NodeDefinition] = [ @@ -32,6 +34,8 @@ class NodeDefinition(BaseModel): "Loads a dataset stored in the system to be used in the pipeline." ), output="Dataset object", + sourceHandles=2, + maxOutputs=2, configType="custom", ), NodeDefinition( @@ -65,6 +69,8 @@ class NodeDefinition(BaseModel): ), input="Dataset object", output="Split dataset subsets", + sourceHandles=2, + maxOutputs=2, configType="custom", ), NodeDefinition( @@ -82,6 +88,8 @@ class NodeDefinition(BaseModel): ), input="Split dataset subsets", output="Trained model object", + sourceHandles=2, + maxOutputs=2, configType="custom", ), NodeDefinition( diff --git a/DashAI/back/pipeline/validator/pipeline_validator.py b/DashAI/back/pipeline/validator/pipeline_validator.py index c0d54f1db..0b98a6b69 100644 --- a/DashAI/back/pipeline/validator/pipeline_validator.py +++ b/DashAI/back/pipeline/validator/pipeline_validator.py @@ -9,19 +9,18 @@ def __init__(self, nodes: List[Dict], edges: List[Dict]): self.edges = edges self.errors: Dict[str, List[str]] = {} self.node_map = {n["id"]: n for n in nodes} - self.duplicated_ids = set() self.type_to_rule = { nt["type"]: { "predecessors": set(nt.get("predecessors", [])), "successors": set(nt.get("successors", [])), + "max_outputs": nt.get("maxOutputs"), } for nt in NODE_TYPES } self.type_to_name = {nt["type"]: nt["name"] for nt in NODE_TYPES} def validate(self) -> Dict[str, List[str]]: - self._validate_duplicates() self._validate_structure() return self.errors @@ -32,31 +31,9 @@ def _get_node_display_name(self, node: Dict) -> str: def _get_type_display_name(self, node_type: str) -> str: return self.type_to_name.get(node_type, node_type) - def _validate_duplicates(self): - def extract_number(id_str): - import re - - match = re.search(r"-(\d+)$", id_str) - return int(match.group(1)) if match else float("inf") - - type_to_ids = {} - for node in self.nodes: - node_type = node["type"] - node_name = self._get_node_display_name(node) - type_to_ids.setdefault(node_type, []).append((node["id"], node_name)) - - for _, ids in type_to_ids.items(): - sorted_ids = sorted(ids, key=lambda x: extract_number(x[0])) - for id, name in sorted_ids[1:]: - self.duplicated_ids.add(id) - self.errors.setdefault(id, []).append(f"{name} already exists.") - def _validate_structure(self): for node in self.nodes: node_id = node["id"] - if node_id in self.duplicated_ids: - continue - node_type = node["type"] node_name = self._get_node_display_name(node) rule = self.type_to_rule.get(node_type) @@ -99,3 +76,11 @@ def _validate_structure(self): self.errors.setdefault(node_id, []).append( f"{node_name} should not have any outputs." ) + + max_outputs = rule.get("max_outputs") + if max_outputs is not None: + outgoing_edges = [e for e in self.edges if e["source"] == node_id] + if len(outgoing_edges) > max_outputs: + self.errors.setdefault(node_id, []).append( + f"{node_name} supports up to {max_outputs} outputs." + ) From 149b53c5d3c0c71b4421e6e2db841023a9a5fe2d Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 8 May 2026 16:58:23 -0400 Subject: [PATCH 46/88] refactor: enhance result normalization and grouping in Results components; update pipeline types for better flexibility --- .../components/pipelines/results/Results.jsx | 239 +++++++++++++----- .../pipelines/results/ResultsExploration.jsx | 59 ++++- DashAI/front/src/types/pipeline.ts | 8 +- 3 files changed, 225 insertions(+), 81 deletions(-) diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 0a6acc9f1..d8f296cb6 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -21,13 +21,97 @@ import ResultsExploration from "./ResultsExploration"; const POLL_INTERVAL_MS = 3000; +function normalizeNodeResults(raw, valueKey = null) { + if (!raw) { + return []; + } + + if (typeof raw === "string") { + return valueKey + ? [{ node_id: "legacy", [valueKey]: raw }] + : [{ node_id: "legacy", value: raw }]; + } + + if (typeof raw !== "object") { + return []; + } + + const entries = Object.entries(raw); + if (entries.length === 0) { + return []; + } + + const looksGrouped = entries.every( + ([, value]) => value && typeof value === "object" && !Array.isArray(value), + ); + + if (looksGrouped) { + return entries.map(([nodeId, value]) => ({ + node_id: value.node_id || nodeId, + ...value, + })); + } + + return [{ node_id: "legacy", ...raw }]; +} + +function buildBranchResults({ trainEntries, taskEntries, metricsEntries, predictionEntries }) { + const branches = new Map(); + + const ensureBranch = (branchKey) => { + if (!branches.has(branchKey)) { + branches.set(branchKey, { + branchKey, + modelName: "-", + taskName: "-", + datasetName: "Unknown Dataset", + parameters: null, + metrics: null, + prediction: null, + }); + } + return branches.get(branchKey); + }; + + [...trainEntries, ...taskEntries].forEach((entry, idx) => { + const branchKey = entry.model_node_id || entry.node_id || `train-${idx}`; + const branch = ensureBranch(branchKey); + + branch.modelName = entry.model || entry.info || branch.modelName; + branch.taskName = entry.task || branch.task_name || branch.taskName; + branch.datasetName = entry.dataset_name || branch.datasetName; + branch.parameters = entry.parameters ?? branch.parameters; + }); + + metricsEntries.forEach((entry, idx) => { + const branchKey = entry.model_node_id || entry.node_id || `metrics-${idx}`; + const branch = ensureBranch(branchKey); + + branch.datasetName = entry.dataset_name || branch.datasetName; + branch.modelName = entry.model_name || branch.modelName; + branch.taskName = entry.task_name || branch.taskName; + branch.metrics = entry.results || branch.metrics; + }); + + predictionEntries.forEach((entry, idx) => { + const branchKey = entry.model_node_id || entry.node_id || `prediction-${idx}`; + const branch = ensureBranch(branchKey); + + branch.datasetName = entry.dataset_name || branch.datasetName; + branch.modelName = entry.model_name || branch.modelName; + branch.taskName = entry.task_name || branch.taskName; + branch.prediction = entry.prediction || branch.prediction; + }); + + return Array.from(branches.values()); +} + function PipelineResults({ pipelineId, onClose }) { const [results, setResults] = useState(null); const [trainTab, setTrainTab] = useState(0); const [polling, setPolling] = useState(false); const pollingRef = useRef(null); - /** Returns true when the pipeline has at least one result section populated. */ const hasAnyResult = useCallback((data) => { if (!data) return false; const hasExpl = @@ -49,10 +133,8 @@ function PipelineResults({ pipelineId, onClose }) { if (cancelled) return; setResults(response); - // If results are not yet available, start polling if (!hasAnyResult(response)) { setPolling(true); - // Clear any previous interval before setting a new one if (pollingRef.current) clearInterval(pollingRef.current); pollingRef.current = setInterval(async () => { try { @@ -95,31 +177,26 @@ function PipelineResults({ pipelineId, onClose }) { return Loading results...; } + const taskEntries = normalizeNodeResults(results.task_and_model); + const trainEntries = normalizeNodeResults(results.train); + const metricsEntries = normalizeNodeResults(results.metrics_result); + const predictionEntries = normalizeNodeResults(results.prediction, "prediction"); + + const branchResults = buildBranchResults({ + trainEntries, + taskEntries, + metricsEntries, + predictionEntries, + }); + const hasExploration = results.exploration && results.exploration !== "No exploration data"; - const hasTrain = results.train && Object.keys(results.train).length > 0; - // Atomized nodes: task_and_model + metrics_result - const hasAtomizedTrain = - (results.task_and_model && - Object.keys(results.task_and_model).length > 0) || - (results.metrics_result && Object.keys(results.metrics_result).length > 0); + const hasTrain = + branchResults.filter((branch) => branch.parameters || branch.metrics).length > 0; const hasPrediction = - results.prediction && results.prediction !== "No prediction data"; - - // Derive unified values from whichever format is available - const trainModelName = hasTrain - ? results.train.info - : (results.task_and_model?.model ?? "-"); - const trainParameters = hasTrain - ? results.train.parameters - : (results.task_and_model?.parameters ?? null); - const trainMetrics = hasTrain - ? results.train.metrics - : (results.metrics_result?.results ?? null); - - const paramData = { parameters: trainParameters }; - - if (!hasExploration && !hasTrain && !hasAtomizedTrain && !hasPrediction) { + branchResults.filter((branch) => !!branch.prediction).length > 0; + + if (!hasExploration && !hasTrain && !hasPrediction) { return ( {onClose && ( @@ -162,50 +239,68 @@ function PipelineResults({ pipelineId, onClose }) { )} - {(hasTrain || hasAtomizedTrain) && ( + {hasTrain && ( }> Train - - - - - - - - - - {trainTab === 0 && ( - - Model Name - - {trainModelName} - - - )} - {trainTab === 1 && ( - - - - )} - {trainTab === 2 && ( - - - - )} - {trainTab === 3 && ( - - - - )} - - + + {branchResults + .filter((branch) => branch.parameters || branch.metrics) + .map((branch, branchIndex) => { + const paramData = { parameters: branch.parameters }; + + return ( + + + + Path {branchIndex + 1} | Dataset: {branch.datasetName} + + + {branch.taskName} | {branch.modelName} + + + + + + + + + + + + {trainTab === 0 && ( + + Model Name + + {branch.modelName} + + + )} + {trainTab === 1 && ( + + + + )} + {trainTab === 2 && ( + + + + )} + {trainTab === 3 && ( + + + + )} + + + ); + })} @@ -217,8 +312,20 @@ function PipelineResults({ pipelineId, onClose }) { Prediction - - + + {branchResults + .filter((branch) => !!branch.prediction) + .map((branch, branchIndex) => ( + + + Path {branchIndex + 1} | Dataset: {branch.datasetName} + + + {branch.taskName} | {branch.modelName} + + + + ))} diff --git a/DashAI/front/src/components/pipelines/results/ResultsExploration.jsx b/DashAI/front/src/components/pipelines/results/ResultsExploration.jsx index 79a9793c5..0271cf7fd 100644 --- a/DashAI/front/src/components/pipelines/results/ResultsExploration.jsx +++ b/DashAI/front/src/components/pipelines/results/ResultsExploration.jsx @@ -172,10 +172,36 @@ function Results({ pipelineId }) { return null; }; + const normalizeGroupedResults = (results) => { + if (!results || typeof results !== "object") { + return {}; + } + + const entries = Object.entries(results); + if (entries.length === 0) { + return {}; + } + + const alreadyGrouped = entries.every(([, value]) => Array.isArray(value)); + if (alreadyGrouped) { + return results; + } + + return { + "Unknown Dataset": entries.map(([explorationId, result]) => ({ + ...result, + exploration_id: explorationId, + node_id: "legacy", + })), + }; + }; + if (!explorationResults) { return Loading...; } + const groupedResults = normalizeGroupedResults(explorationResults); + return ( - {Object.entries(explorationResults).map( - ([explorationName, result], i) => ( - - - {i}: {result.exploration_type} - {result.name ? ` | ${result.name}` : ""} - - {renderVisualizer(result.results.type, result.results)} - - ), - )} + {Object.entries(groupedResults).map(([datasetName, datasetResults]) => ( + + + {datasetName} + + + {datasetResults.map((result, i) => ( + + + {i + 1}: {result.exploration_type} + {result.name ? ` | ${result.name}` : ""} + + {result.results + ? renderVisualizer(result.results.type, result.results) + : null} + + ))} + + ))} ); } diff --git a/DashAI/front/src/types/pipeline.ts b/DashAI/front/src/types/pipeline.ts index a7453b3b3..d3b78bdbe 100644 --- a/DashAI/front/src/types/pipeline.ts +++ b/DashAI/front/src/types/pipeline.ts @@ -20,18 +20,18 @@ export interface IPipeline { model_path?: string; input_columns?: string[]; task?: string; - }; + } | Record; task_and_model?: { task?: string; model?: string; parameters?: Record; model_path?: string; input_columns?: string[]; - }; + } | Record; metrics_result?: { metric_names?: string[]; results?: Record; - }; + } | Record; split_data?: Record; - prediction?: string; + prediction?: string | Record; } From 3d6e349b8275d3572e55f36a60624640682376d4 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 8 May 2026 16:59:08 -0400 Subject: [PATCH 47/88] refactor: add sourceHandles and targetHandles to node properties; normalize handles in usePipelineState --- .../src/components/pipelines/CustomNode.jsx | 104 ++++++++++++------ .../components/pipelines/PipelineDesigner.jsx | 2 + DashAI/front/src/hooks/usePipelineState.js | 54 ++++++++- 3 files changed, 126 insertions(+), 34 deletions(-) diff --git a/DashAI/front/src/components/pipelines/CustomNode.jsx b/DashAI/front/src/components/pipelines/CustomNode.jsx index efe6c1914..0d336851a 100644 --- a/DashAI/front/src/components/pipelines/CustomNode.jsx +++ b/DashAI/front/src/components/pipelines/CustomNode.jsx @@ -33,8 +33,6 @@ const CustomNode = ({ data, isConnectable }) => { const isDisabled = data.errors?.some((err) => err.includes("already exists")) ?? false; - // Light canvas mode: white nodes, yellow borders, black text - // Dark canvas mode: theme-aware colors const borderColor = isLightCanvas ? data.notConfigured && !isDisabled ? "2px solid #f9a825" @@ -67,6 +65,40 @@ const CustomNode = ({ data, isConnectable }) => { const handleDisabledColor = isLightCanvas ? "#cccccc" : theme.palette.ui.border; + const sourceHandles = Math.max(1, Number(data.sourceHandles) || 1); + const targetHandles = Math.max(1, Number(data.targetHandles) || 1); + + const getSourceHandleStyle = (index) => { + if (sourceHandles === 1) { + return { + top: "50%", + transform: "translateY(-50%)", + }; + } + + const spacing = sourceHandles + 1; + const top = ((index + 1) / spacing) * 100; + return { + top: `${top}%`, + transform: "translateY(-50%)", + }; + }; + + const getTargetHandleStyle = (index) => { + if (targetHandles === 1) { + return { + top: "50%", + transform: "translateY(-50%)", + }; + } + + const spacing = targetHandles + 1; + const top = ((index + 1) / spacing) * 100; + return { + top: `${top}%`, + transform: "translateY(-50%)", + }; + }; const nodeContent = ( { )} - {data.target && ( - - )} + {data.target && + Array.from({ length: targetHandles }).map((_, index) => ( + + ))} - {data.source && ( - - )} + {data.source && + Array.from({ length: sourceHandles }).map((_, index) => ( + + ))} ); diff --git a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx index 35735edba..1ea2fd9d3 100644 --- a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx +++ b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx @@ -101,6 +101,8 @@ function PipelineDesigner({ errors: nodeErrors, source: nodeInfo?.source || false, target: nodeInfo?.target || false, + sourceHandles: nodeInfo?.sourceHandles || 1, + targetHandles: nodeInfo?.targetHandles || 1, canvasMode, onDelete: () => { setNodes((nds) => nds.filter((n) => n.id !== nodeId)); diff --git a/DashAI/front/src/hooks/usePipelineState.js b/DashAI/front/src/hooks/usePipelineState.js index d29fe770b..626a25e12 100644 --- a/DashAI/front/src/hooks/usePipelineState.js +++ b/DashAI/front/src/hooks/usePipelineState.js @@ -12,6 +12,34 @@ import { getPipelineById, getPipelines } from "../api/pipeline"; import { generateSequentialName } from "../utils/nameGenerator"; import RunPipeline from "../components/pipelines/Run"; +const TWO_OUTPUT_NODE_TYPES = new Set([ + "DataSelector", + "SplitData", + "TaskAndModel", +]); + +const TWO_INPUT_NODE_TYPES = new Set(["MetricsEval"]); + +const normalizeNodeHandles = (nodes = []) => + nodes.map((node) => { + const shouldNormalizeSource = TWO_OUTPUT_NODE_TYPES.has(node.type); + const shouldNormalizeTarget = TWO_INPUT_NODE_TYPES.has(node.type); + + if (!shouldNormalizeSource && !shouldNormalizeTarget) { + return node; + } + + return { + ...node, + ...(shouldNormalizeSource + ? { sourceHandles: Math.max(2, Number(node.sourceHandles) || 0) } + : {}), + ...(shouldNormalizeTarget + ? { targetHandles: Math.max(2, Number(node.targetHandles) || 0) } + : {}), + }; + }); + export function usePipelineState(pipelineId, location, navigate) { const [nodes, setNodes, onNodesChange] = useNodesState([]); const [edges, setEdges, onEdgesChange] = useEdgesState([]); @@ -68,7 +96,7 @@ export function usePipelineState(pipelineId, location, navigate) { // Load node types and available nodes useEffect(() => { const fetchData = async () => { - const nodes = await getNodeTypes(); + const nodes = normalizeNodeHandles(await getNodeTypes()); setAvailableNodes(nodes); buildNodeHelp(nodes); }; @@ -119,7 +147,7 @@ export function usePipelineState(pipelineId, location, navigate) { const pipeline = await getPipelineById(pipelineId); // Get node types to assign source/target properties - const nodeTypes = await getNodeTypes(); + const nodeTypes = normalizeNodeHandles(await getNodeTypes()); const loadedNodes = pipeline.steps.map((step, idx) => { const nodeInfo = nodeTypes.find((n) => n.type === step.type); @@ -131,6 +159,8 @@ export function usePipelineState(pipelineId, location, navigate) { label: step.label, source: nodeInfo?.source || false, target: nodeInfo?.target || false, + sourceHandles: nodeInfo?.sourceHandles || 1, + targetHandles: nodeInfo?.targetHandles || 1, onDelete: () => { setNodes((nds) => nds.filter((n) => n.id !== step.id)); setNodeData((prev) => { @@ -174,6 +204,24 @@ export function usePipelineState(pipelineId, location, navigate) { } }, [pipelineId]); + // Reset editor when switching to new pipeline route + useEffect(() => { + if (!pipelineId) { + setNodes([]); + setEdges([]); + setNodeData({}); + setSelectedNode(null); + setResultId(null); + setValidationErrors({}); + setHoveredNode(null); + setNodeHelp({}); + setNodeIdCounter(0); + setUserHasModifiedName(false); + setPipelineName("undefined"); + setActiveTab("flow"); + } + }, [pipelineId]); + // Validate pipeline useEffect(() => { const validate = async () => { @@ -203,6 +251,8 @@ export function usePipelineState(pipelineId, location, navigate) { name: nodeInfo?.name || node.type, source: nodeInfo?.source || false, target: nodeInfo?.target || false, + sourceHandles: nodeInfo?.sourceHandles || 1, + targetHandles: nodeInfo?.targetHandles || 1, type: nodeInfo?.type || node.type, configType: nodeInfo?.configType, configSchema: nodeInfo?.configSchema || null, From f0561d27a42d4c6ec0ba20d04372c3d4dcc8e4c5 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 8 May 2026 16:59:39 -0400 Subject: [PATCH 48/88] refactor: enhance layout and styling of MetricsEvalNode and TaskAndModelNode components; improve task and model selection UI --- .../pipelines/nodes/MetricsEvalNode.jsx | 86 +++++++++--- .../pipelines/nodes/TaskAndModelNode.jsx | 131 +++++++++++++----- 2 files changed, 156 insertions(+), 61 deletions(-) diff --git a/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx b/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx index 4c8615c5a..3d51221d7 100644 --- a/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx @@ -3,6 +3,7 @@ import PropTypes from "prop-types"; import { Box, Button, + Chip, DialogContent, Typography, TextField, @@ -73,12 +74,70 @@ const MetricsEvalNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { }; return ( - + - - Metric Evaluation - + + + + Metric Evaluation + + + + setMetrics(e.target.value)} + margin="normal" + disabled={!task} + slotProps={{ + select: { + multiple: true, + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, + minWidth: 560, + }, + }, + }, + renderValue: (selected) => ( + + {selected.map((metricName) => ( + + ))} + + ), + }, + }} + sx={{ + minWidth: { xs: "100%", md: 560 }, + "& .MuiInputBase-root": { + minHeight: 58, + }, + "& .MuiSelect-select": { + minHeight: "56px", + display: "flex", + alignItems: "center", + }, + }} + > + {availableMetrics.map((metric) => ( + + {metric.name} + + ))} + + + + {!task && ( ⚠️ Connect a Task & Model node first so compatible metrics can @@ -86,25 +145,6 @@ const MetricsEvalNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { )} - - - setMetrics(e.target.value)} - margin="normal" - disabled={!task} - slotProps={{ select: { multiple: true } }} - > - {availableMetrics.map((metric) => ( - - {metric.name} - - ))} - - diff --git a/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx index 2ed0c566e..bd9e0056f 100644 --- a/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx @@ -253,7 +253,12 @@ const TaskAndModelNode = ({ return ( <> - + @@ -261,7 +266,6 @@ const TaskAndModelNode = ({ - {/* Warnings when no compatible tasks */} {validTasks.length === 0 && Object.keys(validationErrors).length > 0 && ( @@ -283,37 +287,73 @@ const TaskAndModelNode = ({ )} - {/* Task selector */} - { - setTask(e.target.value); - setModel(""); - setModelParams({}); - }} - margin="normal" - disabled={validTasks.length === 0} - > - {availableTasks.map((taskObj) => ( - + + { + setTask(e.target.value); + setModel(""); + setModelParams({}); + }} + margin="normal" + disabled={validTasks.length === 0} + slotProps={{ + select: { + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, + minWidth: 560, + }, + }, + }, + }, + }} + sx={{ + minWidth: { xs: "100%", md: 560 }, + "& .MuiInputBase-root": { + minHeight: 58, + }, + "& .MuiSelect-select": { + whiteSpace: "normal", + minHeight: "1.4375em !important", + display: "flex", + alignItems: "center", + }, + }} + > + {availableTasks.map((taskObj) => ( + + {taskObj.name} + + ))} + + + + setOpenSettings(true)} + disabled={!model} + aria-label="model settings" + sx={{ mt: "8px", ml: { xs: 0, md: 0.5 } }} > - {taskObj.name} - - ))} - + + + + - {/* Model selector + settings */} - - + + setModel(e.target.value)} margin="normal" disabled={!task} + slotProps={{ + select: { + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, + minWidth: 560, + }, + }, + }, + }, + }} + sx={{ + minWidth: { xs: "100%", md: 560 }, + "& .MuiInputBase-root": { + minHeight: 58, + }, + "& .MuiSelect-select": { + whiteSpace: "normal", + minHeight: "1.4375em !important", + display: "flex", + alignItems: "center", + }, + }} > {availableModels.map((m) => ( @@ -330,16 +394,7 @@ const TaskAndModelNode = ({ ))} - - setOpenSettings(true)} - disabled={!model} - aria-label="model settings" - sx={{ mt: "8px" }} - > - - - + From 7cd6dacfe039ef5fc1006bb2479f1b9f9026449f Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 8 May 2026 17:00:12 -0400 Subject: [PATCH 49/88] feat: implement new pipeline sidebar components and enhance NewPipeline page layout; integrate pipeline templates and chat functionality --- DashAI/front/src/App.jsx | 3 +- .../pipelines/PipelineChatSidebar.jsx | 101 ++++ .../pipelines/PipelineHistorySidebar.jsx | 217 ++++++++ .../pipelines/PipelineTemplatesSidebar.jsx | 89 +++ .../front/src/components/pipelines/index.jsx | 3 + .../front/src/hooks/useThreePanelsLayout.js | 7 +- .../front/src/pages/pipelines/NewPipeline.jsx | 509 ++++++++++++++---- 7 files changed, 821 insertions(+), 108 deletions(-) create mode 100644 DashAI/front/src/components/pipelines/PipelineChatSidebar.jsx create mode 100644 DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx create mode 100644 DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx diff --git a/DashAI/front/src/App.jsx b/DashAI/front/src/App.jsx index f0aa151c8..2819db857 100644 --- a/DashAI/front/src/App.jsx +++ b/DashAI/front/src/App.jsx @@ -8,7 +8,6 @@ import ModelsPage from "./pages/models/Models"; import Home from "./pages/home/Home"; import ResponsiveAppBar from "./components/ResponsiveAppBar"; import PluginsPage from "./pages/plugins/Plugins"; -import PipelinesPage from "./pages/pipelines/Pipelines"; import PluginsDetails from "./pages/plugins/components/PluginsDetails"; import Generative from "./pages/generative/Generative"; import NewPipelineWrapper from "./pages/pipelines/newPipelineWrapper"; @@ -26,7 +25,7 @@ function App() { } /> } /> } /> - } /> + } /> } /> + + + + Pipeline Chatbot + + {onToggle && ( + + + + )} + + + + + + + + Ideas de prompts: + + + - "Revisa este pipeline y sugiere mejoras" + + + - "Explica por que falla la validacion" + + + - "Sugiere el siguiente nodo" + + + + + + + + setMessage(event.target.value)} + /> + + + + ); +} + +export default PipelineChatSidebar; diff --git a/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx new file mode 100644 index 000000000..1cbbd2c4e --- /dev/null +++ b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx @@ -0,0 +1,217 @@ +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { + Box, + Button, + Divider, + IconButton, + LinearProgress, + Typography, +} from "@mui/material"; +import { + AddCircleOutline as AddIcon, + MoreVert as MoreVertIcon, + Update as UpdateIcon, + Visibility as ViewIcon, +} from "@mui/icons-material"; +import HistoryIcon from "@mui/icons-material/History"; +import { ChevronLeft } from "@mui/icons-material"; +import { useTheme } from "@mui/material/styles"; +import { DataGrid, GridActionsCellItem, GridToolbar } from "@mui/x-data-grid"; +import { Menu, MenuItem } from "@mui/material"; +import { useNavigate } from "react-router-dom"; +import { useSnackbar } from "notistack"; +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; + +import { deletePipeline, getPipelines } from "../../api/pipeline"; +import { formatDate } from "../../utils"; +import DeleteItemModal from "../custom/DeleteItemModal"; + +function PipelineHistorySidebar({ currentPipelineId, onToggle }) { + const theme = useTheme(); + const navigate = useNavigate(); + const { enqueueSnackbar } = useSnackbar(); + const [loading, setLoading] = useState(true); + const [pipelines, setPipelines] = useState([]); + const [rowMenuAnchor, setRowMenuAnchor] = useState(null); + const [rowMenuPipeline, setRowMenuPipeline] = useState(null); + + const fetchPipelines = useCallback(async () => { + setLoading(true); + try { + const data = await getPipelines(); + setPipelines(data || []); + } catch (error) { + enqueueSnackbar("Error while fetching pipelines", { variant: "error" }); + console.error(error); + } finally { + setLoading(false); + } + }, [enqueueSnackbar]); + + useEffect(() => { + fetchPipelines(); + }, [fetchPipelines]); + + const handleDelete = async (id) => { + try { + await deletePipeline(id); + enqueueSnackbar("Pipeline deleted successfully", { variant: "success" }); + if (String(currentPipelineId) === String(id)) { + navigate("/app/pipelines"); + } + fetchPipelines(); + } catch (error) { + enqueueSnackbar("Failed to delete pipeline", { variant: "error" }); + console.error(error); + } + }; + + const selectedPipeline = useMemo(() => { + const parsedId = Number(currentPipelineId); + return Number.isFinite(parsedId) ? [parsedId] : []; + }, [currentPipelineId]); + + const handleOpenRowMenu = (event, pipeline) => { + event.stopPropagation(); + setRowMenuAnchor(event.currentTarget); + setRowMenuPipeline(pipeline); + }; + + const handleCloseRowMenu = () => { + setRowMenuAnchor(null); + setRowMenuPipeline(null); + }; + + const columns = useMemo( + () => [ + { + field: "name", + headerName: "Name", + minWidth: 150, + flex: 1, + }, + { + field: "actions", + type: "actions", + minWidth: 120, + getActions: (params) => [ + } + label="View" + onClick={() => navigate(`/app/pipelines/${params.id}`)} + />, + } + label="Details" + onClick={(event) => handleOpenRowMenu(event, params.row)} + />, + handleDelete(params.id)} + />, + ], + }, + ], + [navigate, handleDelete], + ); + + return ( + + + + + Pipelines History + + {onToggle && ( + + + + )} + + + + + + + + + + + + + + + + Created: {rowMenuPipeline ? formatDate(rowMenuPipeline.created) : "-"} + + + Edited: {rowMenuPipeline ? formatDate(rowMenuPipeline.last_modified) : "-"} + + + + + ); +} + +export default PipelineHistorySidebar; diff --git a/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx b/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx new file mode 100644 index 000000000..2323588a1 --- /dev/null +++ b/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx @@ -0,0 +1,89 @@ +import React from "react"; +import { + Box, + Button, + Card, + CardActions, + CardContent, + Chip, + IconButton, + Typography, +} from "@mui/material"; +import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; +import { ChevronRight } from "@mui/icons-material"; +import { useTheme } from "@mui/material/styles"; +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; + +function PipelineTemplatesSidebar({ onToggle, templates = [], onSelectTemplate }) { + const theme = useTheme(); + + return ( + + + + + Prefabricated Pipelines + + {onToggle && ( + + + + )} + + + + {templates.map((template) => ( + onSelectTemplate?.(template.id)} + > + + + {template.name} + + + {(template.labels || template.steps).join(" + ")} + + + {(template.labels || template.steps).map((step) => ( + + ))} + + + + + + + ))} + + + ); +} + +export default PipelineTemplatesSidebar; diff --git a/DashAI/front/src/components/pipelines/index.jsx b/DashAI/front/src/components/pipelines/index.jsx index 818688d71..87a7c982f 100644 --- a/DashAI/front/src/components/pipelines/index.jsx +++ b/DashAI/front/src/components/pipelines/index.jsx @@ -3,6 +3,9 @@ export { default as PipelineHeader } from "./PipelineHeader"; export { default as PipelineToolbar } from "./PipelineToolbar"; export { default as PipelineDesigner } from "./PipelineDesigner"; export { default as NodeSidebar } from "./NodeSidebar"; +export { default as PipelineHistorySidebar } from "./PipelineHistorySidebar"; +export { default as PipelineTemplatesSidebar } from "./PipelineTemplatesSidebar"; +export { default as PipelineChatSidebar } from "./PipelineChatSidebar"; export { default as PipelinesTable } from "./PipelinesTable"; export { default as CustomNode } from "./CustomNode"; export { default as Run } from "./Run"; diff --git a/DashAI/front/src/hooks/useThreePanelsLayout.js b/DashAI/front/src/hooks/useThreePanelsLayout.js index cc1041bc3..9e169f462 100644 --- a/DashAI/front/src/hooks/useThreePanelsLayout.js +++ b/DashAI/front/src/hooks/useThreePanelsLayout.js @@ -1,10 +1,11 @@ import { useState, useRef, useEffect, useCallback } from "react"; -export function useThreePanelLayout() { +export function useThreePanelLayout(options = {}) { + const { initialLeftWidth = 20, initialRightWidth = 20 } = options; const [leftBarVisible, setLeftBarVisible] = useState(true); const [rightBarVisible, setRightBarVisible] = useState(true); - const [leftBarWidth, setLeftBarWidth] = useState(20); - const [rightBarWidth, setRightBarWidth] = useState(20); + const [leftBarWidth, setLeftBarWidth] = useState(initialLeftWidth); + const [rightBarWidth, setRightBarWidth] = useState(initialRightWidth); const [isTogglingLeft, setIsTogglingLeft] = useState(false); const [isTogglingRight, setIsTogglingRight] = useState(false); diff --git a/DashAI/front/src/pages/pipelines/NewPipeline.jsx b/DashAI/front/src/pages/pipelines/NewPipeline.jsx index 031a3cd0f..2bbab60c5 100644 --- a/DashAI/front/src/pages/pipelines/NewPipeline.jsx +++ b/DashAI/front/src/pages/pipelines/NewPipeline.jsx @@ -1,6 +1,12 @@ import React, { useRef, useState } from "react"; import { Box, + Button, + Card, + CardActions, + CardContent, + Chip, + Grid, Typography, Dialog, DialogTitle, @@ -10,25 +16,62 @@ import { useTheme } from "@mui/material/styles"; import CloseIcon from "@mui/icons-material/Close"; import { ReactFlowProvider } from "reactflow"; import CustomLayout from "../../components/custom/CustomLayout"; +import ModuleContainer from "../../components/layout/ModuleContainer"; +import LeftPanel from "../../components/threeSectionLayout/panels/LeftPanel"; +import CenterPanel from "../../components/threeSectionLayout/panels/CenterPanel"; +import RightPanel from "../../components/threeSectionLayout/panels/RightPanel"; +import { ThreePanelLayoutContext } from "../../components/threeSectionLayout/panels/ThreePanelLayoutContext"; +import OptionBox from "../../components/threeSectionLayout/OptionBox"; import { Results as PipelineResults } from "../../components/pipelines"; import { useParams, useNavigate, useLocation } from "react-router-dom"; import { usePipelineState } from "../../hooks/usePipelineState"; import { useConnectedNodeData } from "../../hooks/useConnectedNodeData"; +import { useThreePanelLayout } from "../../hooks/useThreePanelsLayout"; import { PipelineHeader, PipelineToolbar, PipelineDesigner, NodeSidebar, + PipelineHistorySidebar, + PipelineChatSidebar, nodeRegistry, } from "../../components/pipelines"; +const PIPELINE_TEMPLATES = [ + { + id: "data_exploration", + name: "Data Exploration", + steps: ["DataSelector", "DataExploration"], + labels: ["Data Selector", "Data Exploration"], + }, + { + id: "train_model", + name: "Train Model", + steps: ["DataSelector", "SplitData", "TaskAndModel", "MetricsEval", "Prediction"], + labels: ["Data Selector", "Split Data", "Task & Model", "Metrics", "Prediction"], + }, + { + id: "retrieve_model", + name: "Retrieve Model", + steps: ["DataSelector", "RetrieveModel", "Prediction"], + labels: ["Data Selector", "Retrieve Model", "Prediction"], + }, +]; + function NewPipeline() { const location = useLocation(); const { pipelineId } = useParams(); const navigate = useNavigate(); + const isNewPipelineRoute = location.pathname.endsWith("/new"); + const isWorkspaceOpen = Boolean(pipelineId) || isNewPipelineRoute; const flowWrapperRef = useRef(null); const theme = useTheme(); const [canvasMode, setCanvasMode] = useState("dark"); + const [centerMenuView, setCenterMenuView] = useState("home"); + const threePanelLayout = useThreePanelLayout({ + initialLeftWidth: 20, + initialRightWidth: 20, + }); const { // State @@ -80,6 +123,97 @@ function NewPipeline() { const { getConnectedNodeData } = useConnectedNodeData(nodes, nodeData, edges); + const handleApplyTemplate = (templateId) => { + const selectedTemplate = PIPELINE_TEMPLATES.find((t) => t.id === templateId); + if (!selectedTemplate) return; + + if (!isWorkspaceOpen) { + navigate("/app/pipelines/new", { state: { initialTemplateId: templateId } }); + return; + } + + const startX = 120; + const startY = 170; + const spacingX = 230; + + const createdNodes = selectedTemplate.steps.map((nodeType, index) => { + const nodeInfo = availableNodes.find((node) => node.type === nodeType); + const nodeId = `${nodeType}-${index}`; + + return { + id: nodeId, + type: nodeType, + position: { + x: startX + index * spacingX, + y: startY, + }, + data: { + label: nodeInfo?.name || nodeType, + source: nodeInfo?.source || false, + target: nodeInfo?.target || false, + sourceHandles: nodeInfo?.sourceHandles || 1, + targetHandles: nodeInfo?.targetHandles || 1, + canvasMode, + onDelete: () => { + setNodes((nds) => nds.filter((n) => n.id !== nodeId)); + setNodeData((prev) => { + const newData = { ...prev }; + delete newData[nodeId]; + return newData; + }); + setEdges((eds) => + eds.filter((edge) => edge.source !== nodeId && edge.target !== nodeId), + ); + }, + }, + sourcePosition: "right", + targetPosition: "left", + }; + }); + + const createdEdges = createdNodes.slice(0, -1).map((node, index) => ({ + id: `e-${node.id}-${createdNodes[index + 1].id}`, + source: node.id, + target: createdNodes[index + 1].id, + markerEnd: { + type: "arrowclosed", + }, + })); + + setSelectedNode(null); + setDragging(null); + setNodeHelp(null); + setHoveredNode(null); + setValidationErrors({}); + setResultId(null); + setNodeData({}); + setNodes(createdNodes); + setEdges(createdEdges); + setNodeIdCounter(createdNodes.length); + setActiveTab("flow"); + }; + + React.useEffect(() => { + const templateIdFromState = location.state?.initialTemplateId; + + if (templateIdFromState && isWorkspaceOpen && availableNodes.length > 0) { + handleApplyTemplate(templateIdFromState); + navigate(location.pathname, { replace: true, state: {} }); + } + }, [ + availableNodes.length, + isWorkspaceOpen, + location.pathname, + location.state, + navigate, + ]); + + React.useEffect(() => { + if (isWorkspaceOpen) { + setCenterMenuView("home"); + } + }, [isWorkspaceOpen]); + const renderNodeDialogContent = () => { if (!selectedNode) return null; @@ -108,119 +242,288 @@ function NewPipeline() { }; return ( - + - - - {activeTab === "flow" ? ( - <> - - + + + + - - - - - - - {renderNodeDialogContent() && ( - {}} - disableEscapeKeyDown - maxWidth="md" - fullWidth + + {!isWorkspaceOpen ? ( + - + {centerMenuView === "home" ? ( + <> + + Pipelines Module + + + Create and manage your pipelines. + + + + + navigate("/app/pipelines/new")} + /> + + + setCenterMenuView("prefabricated")} + /> + + + + ) : ( + <> + + Choose a Prefabricated Pipeline + + + Select one of the templates to open it in the workspace. + + + + {PIPELINE_TEMPLATES.map((template) => ( + + handleApplyTemplate(template.id)} + > + + + {template.name} + + + {(template.labels || template.steps).join(" + ")} + + + {(template.labels || template.steps).map((step) => ( + + ))} + + + + + + + + ))} + + + + + + + )} + + ) : ( + + + + + + {activeTab === "flow" ? ( - - {selectedNode?.type === "Train" - ? "Select Train Parameters" - : selectedNode?.type === "Exploration" - ? "Exploration Configuration" - : selectedNode?.type === "SplitData" - ? "Configure Data Splits" - : selectedNode?.type === "TaskAndModel" - ? "Select Task & Model" - : selectedNode?.type === "MetricsEval" - ? "Select Metrics" - : `Configure ${selectedNode?.type || "Node"}`} - - + + - - + + + + + + ) : ( + + {resultId ? ( + + ) : ( + No results yet. + )} - - {renderNodeDialogContent()} - + )} + )} - - - ) : ( - - {resultId ? ( - - ) : ( - No results yet. + + + + + + + {isWorkspaceOpen && renderNodeDialogContent() && ( + {}} + disableEscapeKeyDown + maxWidth={ + selectedNode?.type === "TaskAndModel" || + selectedNode?.type === "MetricsEval" + ? "lg" + : "md" + } + fullWidth + PaperProps={{ + sx: + selectedNode?.type === "TaskAndModel" || + selectedNode?.type === "MetricsEval" + ? { + width: { xs: "calc(100% - 24px)", md: 920 }, + maxWidth: { xs: "calc(100% - 24px)", md: 920 }, + } + : undefined, + }} + > + + + + {selectedNode?.type === "Train" + ? "Select Train Parameters" + : selectedNode?.type === "Exploration" + ? "Exploration Configuration" + : selectedNode?.type === "SplitData" + ? "Configure Data Splits" + : selectedNode?.type === "TaskAndModel" + ? "Select Task & Model" + : selectedNode?.type === "MetricsEval" + ? "Select Metrics" + : `Configure ${selectedNode?.type || "Node"}`} + + + + + + + {renderNodeDialogContent()} + )} - - )} + + ); From d9ecf90992525819bcb5c1767d87641c2fc9437a Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 22 Jun 2026 16:17:03 -0400 Subject: [PATCH 50/88] feat: add node contract system with backend-side edge validation --- DashAI/back/api/api_v1/endpoints/pipelines.py | 28 ++ .../api/api_v1/schemas/pipelines_params.py | 12 + DashAI/back/pipeline/contracts.py | 446 ++++++++++++++++++ .../pipeline/validator/nodes_definitions.py | 137 +----- .../pipeline/validator/pipeline_validator.py | 12 +- DashAI/front/src/api/pipeline.ts | 10 + DashAI/front/src/index.css | 15 + 7 files changed, 523 insertions(+), 137 deletions(-) create mode 100644 DashAI/back/pipeline/contracts.py diff --git a/DashAI/back/api/api_v1/endpoints/pipelines.py b/DashAI/back/api/api_v1/endpoints/pipelines.py index 572b64e58..73ae30d8c 100644 --- a/DashAI/back/api/api_v1/endpoints/pipelines.py +++ b/DashAI/back/api/api_v1/endpoints/pipelines.py @@ -9,10 +9,12 @@ DatasetFilterParams, PipelineCreateParams, PipelineUpdateParams, + ValidateEdgeParams, ValidateNodeParams, ValidatePipelineParams, ) from DashAI.back.dependencies.database.models import Dataset, Pipeline +from DashAI.back.pipeline.contracts import get_contracts_payload, validate_edge from DashAI.back.pipeline.validator.nodes_definitions import NODES from DashAI.back.pipeline.validator.pipeline_validator import PipelineValidator from DashAI.back.pipeline.validator.validator import VALIDATOR_MAP @@ -76,6 +78,18 @@ async def get_nodes() -> List[Dict[str, Any]]: ) from e +@router.get("/contracts") +async def get_node_contracts() -> List[Dict[str, Any]]: + """Retrieve pipeline node contracts.""" + try: + return get_contracts_payload() + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to load node contracts", + ) from e + + @router.get("/predict_summary") @inject async def pipeline_predict_summary( @@ -647,6 +661,20 @@ async def validate_pipeline( return validator.validate() +@router.post("/validate_edge") +async def validate_edge_connection(params: ValidateEdgeParams) -> Dict[str, Any]: + """Validate a single edge connection between two node ports.""" + result = validate_edge( + source_type=params.source.type, + target_type=params.target.type, + source_port=params.source.port, + target_port=params.target.port, + source_current_outputs=params.source.currentConnections, + target_current_inputs=params.target.currentConnections, + ) + return result + + @router.post("/filter_models") async def filter_models_endpoint( params: DatasetFilterParams, diff --git a/DashAI/back/api/api_v1/schemas/pipelines_params.py b/DashAI/back/api/api_v1/schemas/pipelines_params.py index 993754754..bca8c2be9 100644 --- a/DashAI/back/api/api_v1/schemas/pipelines_params.py +++ b/DashAI/back/api/api_v1/schemas/pipelines_params.py @@ -35,3 +35,15 @@ class ValidateNodeParams(BaseModel): class ValidatePipelineParams(BaseModel): nodes: List[Dict[str, Any]] edges: List[Dict[str, Any]] + + +class EdgeEndpointRef(BaseModel): + nodeId: Optional[str] = None + type: str + port: Optional[str] = None + currentConnections: Optional[int] = None + + +class ValidateEdgeParams(BaseModel): + source: EdgeEndpointRef + target: EdgeEndpointRef diff --git a/DashAI/back/pipeline/contracts.py b/DashAI/back/pipeline/contracts.py new file mode 100644 index 000000000..b0a143fc9 --- /dev/null +++ b/DashAI/back/pipeline/contracts.py @@ -0,0 +1,446 @@ +from __future__ import annotations + +from typing import Dict, List, Literal, Optional, TYPE_CHECKING + +from pydantic import BaseModel + +if TYPE_CHECKING: + from DashAI.back.pipeline.validator.nodes_definitions import NodeDefinition + +DataKind = Literal[ + "DatasetRef", + "DatasetSplit", + "ModelArtifact", + "MetricsResult", + "PredictionArtifact", + "ExplorationResult", + "Any", +] + +class PortSpec(BaseModel): + name: str + kind: DataKind + accepts: Optional[List[DataKind]] = None + required: bool = True + max_connections: Optional[int] = 1 + description: Optional[str] = None + + +class NodeContract(BaseModel): + type: str + name: str + icon: str + description: str + requiresConfiguration: bool + source: bool + target: bool + predecessors: List[str] + successors: List[str] + input: Optional[str] = None + output: Optional[str] = None + sourceHandles: int = 1 + maxInputs: Optional[int] = 1 + maxOutputs: Optional[int] = None + configType: str = "custom" + inputs: List[PortSpec] + outputs: List[PortSpec] + + +NODE_CONTRACTS: List[NodeContract] = [ + NodeContract( + type="DataSelector", + name="Data Selector", + icon="FolderIcon", + description="Loads a dataset stored in the system to be used in the pipeline.", + requiresConfiguration=True, + source=True, + target=False, + predecessors=[], + successors=["DataExploration", "Train", "SplitData"], + output="Dataset object", + sourceHandles=2, + maxOutputs=2, + inputs=[], + outputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + max_connections=2, + description="Reference to the dataset loaded from storage.", + ) + ], + ), + NodeContract( + type="DataExploration", + name="Data Exploration", + icon="InsertChartIcon", + description=( + "Visualize and analyze your dataset to gain insights into its structure " + "and content." + ), + requiresConfiguration=True, + source=False, + target=True, + predecessors=["DataSelector"], + successors=[], + input="Dataset object", + inputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + description="Dataset used for exploration.", + ) + ], + outputs=[ + PortSpec( + name="exploration", + kind="ExplorationResult", + description="Exploration outputs and charts.", + ) + ], + ), + NodeContract( + type="SplitData", + name="Split Data", + icon="CallSplitIcon", + description=( + "Configures the partition of the dataset into train / validation / test " + "splits." + ), + requiresConfiguration=True, + source=True, + target=True, + predecessors=["DataSelector"], + successors=["TaskAndModel"], + input="Dataset object", + output="Split dataset subsets", + sourceHandles=2, + maxOutputs=2, + inputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + description="Dataset to split.", + ) + ], + outputs=[ + PortSpec( + name="split", + kind="DatasetSplit", + max_connections=2, + description="Train/validation/test splits.", + ) + ], + ), + NodeContract( + type="TaskAndModel", + name="Task & Model", + icon="ModelTrainingIcon", + description=( + "Selects the ML task and model, configures hyper-parameters, and trains " + "the model on the split data." + ), + requiresConfiguration=True, + source=True, + target=True, + predecessors=["SplitData"], + successors=["MetricsEval", "Prediction"], + input="Split dataset subsets", + output="Trained model object", + sourceHandles=2, + maxOutputs=2, + inputs=[ + PortSpec( + name="split", + kind="DatasetSplit", + description="Split dataset used for training.", + ) + ], + outputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + max_connections=2, + description="Trained model artifact.", + ) + ], + ), + NodeContract( + type="MetricsEval", + name="Metrics", + icon="AssessmentIcon", + description="Evaluates the trained model using the selected metrics.", + requiresConfiguration=True, + source=True, + target=True, + predecessors=["TaskAndModel"], + successors=["Prediction"], + input="Trained model and split data", + output="Model artifact (metrics stored in pipeline results)", + maxInputs=2, + inputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + max_connections=2, + description="Model to evaluate.", + ) + ], + outputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + description="Model artifact passthrough for downstream nodes.", + ) + ], + ), + NodeContract( + type="Train", + name="Train Model", + icon="SettingsIcon", + description=( + "Trains a model using a selected dataset, with configurable columns, " + "splits, task, model, and metrics." + ), + requiresConfiguration=True, + source=True, + target=True, + predecessors=["DataSelector"], + successors=["Prediction"], + input="Dataset object", + output="Trained model object and evaluation results", + inputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + description="Dataset used for training.", + ) + ], + outputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + description="Trained model artifact.", + ) + ], + ), + NodeContract( + type="Prediction", + name="Prediction", + icon="EmojiObjectsIcon", + description="Generates predictions using a trained model.", + requiresConfiguration=False, + source=False, + target=True, + predecessors=["Train", "RetrieveModel", "TaskAndModel", "MetricsEval"], + successors=[], + input="Trained model and dataset object", + inputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + description="Model used for predictions.", + ) + ], + outputs=[ + PortSpec( + name="prediction", + kind="PredictionArtifact", + description="Prediction output artifact.", + ) + ], + ), + NodeContract( + type="RetrieveModel", + name="Retrieve Model", + icon="ManageHistoryIcon", + description="Loads a trained model from saved pipelines.", + requiresConfiguration=True, + source=True, + target=True, + predecessors=["DataSelector"], + successors=["Prediction"], + input="Dataset object", + output="Trained model object", + inputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + description="Dataset used for compatibility checks.", + ) + ], + outputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + description="Loaded model artifact.", + ) + ], + ), +] + +NODE_CONTRACTS_MAP: Dict[str, NodeContract] = { + contract.type: contract for contract in NODE_CONTRACTS +} + + +def get_contracts_payload() -> List[Dict[str, object]]: + payload = [contract.model_dump() for contract in NODE_CONTRACTS] + type_to_name = {contract.type: contract.name for contract in NODE_CONTRACTS} + for entry in payload: + entry["next"] = [type_to_name.get(t, t) for t in entry.get("successors", [])] + return payload + + +def _resolve_port(ports: List[PortSpec], port_name: Optional[str], role: str) -> PortSpec: + if port_name: + for port in ports: + if port.name == port_name: + return port + raise ValueError(f"Unknown {role} port: {port_name}") + if len(ports) == 1: + return ports[0] + raise ValueError(f"Ambiguous {role} port: specify a port name") + + +def validate_edge( + source_type: str, + target_type: str, + source_port: Optional[str] = None, + target_port: Optional[str] = None, + source_current_outputs: Optional[int] = None, + target_current_inputs: Optional[int] = None, +) -> Dict[str, object]: + source_contract = NODE_CONTRACTS_MAP.get(source_type) + target_contract = NODE_CONTRACTS_MAP.get(target_type) + + if source_contract is None: + return _error_response( + "Unknown source node type.", + reason="UNKNOWN_SOURCE", + ) + if target_contract is None: + return _error_response( + "Unknown target node type.", + reason="UNKNOWN_TARGET", + ) + + if target_contract.predecessors and source_type not in target_contract.predecessors: + return _error_response( + f"{target_contract.name} does not accept {source_contract.name} as input.", + reason="INVALID_PREDECESSOR", + ) + if source_contract.successors and target_type not in source_contract.successors: + return _error_response( + f"{source_contract.name} cannot connect to {target_contract.name}.", + reason="INVALID_SUCCESSOR", + ) + + try: + target_port_spec = _resolve_port( + target_contract.inputs, target_port, role="target" + ) + if source_port: + source_port_spec = _resolve_port( + source_contract.outputs, source_port, role="source" + ) + else: + accepts = target_port_spec.accepts or [target_port_spec.kind] + compatible = [ + port for port in source_contract.outputs if port.kind in accepts + ] + if len(compatible) == 1: + source_port_spec = compatible[0] + else: + source_port_spec = _resolve_port( + source_contract.outputs, source_port, role="source" + ) + except ValueError as exc: + return _error_response(str(exc), reason="INVALID_PORT") + + if target_port_spec.max_connections is not None and target_current_inputs is not None: + if target_current_inputs >= target_port_spec.max_connections: + return _error_response( + f"{target_contract.name} does not allow more connections on this input.", + reason="MAX_INPUTS_EXCEEDED", + severity="warning", + edge_color="#F57C00", + edge_class="edge-warning", + ) + + if source_current_outputs is not None and source_port_spec.max_connections is not None: + if source_current_outputs >= source_port_spec.max_connections: + return _error_response( + f"{source_contract.name} does not allow more outputs on this port.", + reason="MAX_OUTPUTS_EXCEEDED", + severity="warning", + edge_color="#F57C00", + edge_class="edge-warning", + ) + + accepts = target_port_spec.accepts or [target_port_spec.kind] + if source_port_spec.kind not in accepts: + return _error_response( + ( + f"{source_contract.name} outputs {source_port_spec.kind}, but " + f"{target_contract.name} expects {', '.join(accepts)}." + ), + reason="TYPE_MISMATCH", + ) + + return { + "ok": True, + "severity": "info", + "message": "Compatible connection.", + "reason": None, + "style": { + "edgeColor": "#4CAF50", + "edgeClass": "edge-ok", + }, + } + + +def _error_response( + message: str, + reason: str, + severity: str = "error", + edge_color: str = "#D32F2F", + edge_class: str = "edge-error", +) -> Dict[str, object]: + return { + "ok": False, + "severity": severity, + "message": message, + "reason": reason, + "style": { + "edgeColor": edge_color, + "edgeClass": edge_class, + }, + } + + +def contracts_to_node_definitions() -> List["NodeDefinition"]: + from DashAI.back.pipeline.validator.nodes_definitions import NodeDefinition + + definitions: List[NodeDefinition] = [] + for contract in NODE_CONTRACTS: + definitions.append( + NodeDefinition( + type=contract.type, + name=contract.name, + icon=contract.icon, + requiresConfiguration=contract.requiresConfiguration, + source=contract.source, + target=contract.target, + predecessors=contract.predecessors, + successors=contract.successors, + description=contract.description, + input=contract.input, + output=contract.output, + sourceHandles=contract.sourceHandles, + maxInputs=contract.maxInputs, + maxOutputs=contract.maxOutputs, + configType=contract.configType, + ) + ) + return definitions diff --git a/DashAI/back/pipeline/validator/nodes_definitions.py b/DashAI/back/pipeline/validator/nodes_definitions.py index 2e1cc42c0..a46931846 100644 --- a/DashAI/back/pipeline/validator/nodes_definitions.py +++ b/DashAI/back/pipeline/validator/nodes_definitions.py @@ -2,6 +2,8 @@ from pydantic import BaseModel +from DashAI.back.pipeline.contracts import contracts_to_node_definitions + class NodeDefinition(BaseModel): type: str @@ -16,142 +18,11 @@ class NodeDefinition(BaseModel): input: Optional[str] = None output: Optional[str] = None sourceHandles: int = 1 + maxInputs: Optional[int] = 1 maxOutputs: Optional[int] = None configType: str -NODES: List[NodeDefinition] = [ - NodeDefinition( - type="DataSelector", - name="Data Selector", - icon="FolderIcon", - requiresConfiguration=True, - source=True, - target=False, - predecessors=[], - successors=["DataExploration", "Train", "SplitData"], - description=( - "Loads a dataset stored in the system to be used in the pipeline." - ), - output="Dataset object", - sourceHandles=2, - maxOutputs=2, - configType="custom", - ), - NodeDefinition( - type="DataExploration", - name="Data Exploration", - icon="InsertChartIcon", - requiresConfiguration=True, - source=False, - target=True, - predecessors=["DataSelector"], - successors=[], - description=( - "Visualize and analyze your dataset to gain insights " - "into its structure and content." - ), - input="Dataset object", - configType="custom", - ), - NodeDefinition( - type="SplitData", - name="Split Data", - icon="CallSplitIcon", - requiresConfiguration=True, - source=True, - target=True, - predecessors=["DataSelector"], - successors=["TaskAndModel"], - description=( - "Configures the partition of the dataset into " - "train / validation / test splits." - ), - input="Dataset object", - output="Split dataset subsets", - sourceHandles=2, - maxOutputs=2, - configType="custom", - ), - NodeDefinition( - type="TaskAndModel", - name="Task & Model", - icon="ModelTrainingIcon", - requiresConfiguration=True, - source=True, - target=True, - predecessors=["SplitData"], - successors=["MetricsEval", "Prediction"], - description=( - "Selects the ML task and model, configures hyper-parameters, " - "and trains the model on the split data." - ), - input="Split dataset subsets", - output="Trained model object", - sourceHandles=2, - maxOutputs=2, - configType="custom", - ), - NodeDefinition( - type="MetricsEval", - name="Metrics", - icon="AssessmentIcon", - requiresConfiguration=True, - source=True, - target=True, - predecessors=["TaskAndModel"], - successors=["Prediction"], - description=( - "Evaluates the trained model using the selected metricson each data split." - ), - input="Trained model and split data", - output="Evaluation results", - configType="custom", - ), - NodeDefinition( - type="Train", - name="Train Model", - icon="SettingsIcon", - requiresConfiguration=True, - source=True, - target=True, - predecessors=["DataSelector"], - successors=["Prediction"], - description=( - "Trains a model using a selected dataset, with configurable " - "columns, splits, task, model, and metrics." - ), - input="Dataset object", - output="Trained model object and evaluation results", - configType="custom", - ), - NodeDefinition( - type="Prediction", - name="Prediction", - icon="EmojiObjectsIcon", - requiresConfiguration=False, - source=False, - target=True, - predecessors=["Train", "RetrieveModel", "TaskAndModel", "MetricsEval"], - successors=[], - description="Generates predictions using a trained model.", - input="Trained model and dataset object", - configType="custom", - ), - NodeDefinition( - type="RetrieveModel", - name="Retrieve Model", - icon="ManageHistoryIcon", - requiresConfiguration=True, - source=True, - target=True, - predecessors=["DataSelector"], - successors=["Prediction"], - description="Loads a trained model from saved pipelines.", - input="Dataset object", - output="Trained model object", - configType="custom", - ), -] +NODES: List[NodeDefinition] = contracts_to_node_definitions() NODE_TYPES = [node.model_dump() for node in NODES] diff --git a/DashAI/back/pipeline/validator/pipeline_validator.py b/DashAI/back/pipeline/validator/pipeline_validator.py index 0b98a6b69..78a62b597 100644 --- a/DashAI/back/pipeline/validator/pipeline_validator.py +++ b/DashAI/back/pipeline/validator/pipeline_validator.py @@ -14,6 +14,7 @@ def __init__(self, nodes: List[Dict], edges: List[Dict]): nt["type"]: { "predecessors": set(nt.get("predecessors", [])), "successors": set(nt.get("successors", [])), + "max_inputs": nt.get("maxInputs"), "max_outputs": nt.get("maxOutputs"), } for nt in NODE_TYPES @@ -43,10 +44,13 @@ def _validate_structure(self): expected_predecessors = rule["predecessors"] predecessors = [e for e in self.edges if e["target"] == node_id] - if len(predecessors) > 1: - self.errors.setdefault(node_id, []).append( - f"{node_name} cannot have more than one input." - ) + max_inputs = rule.get("max_inputs", 1) + if max_inputs is not None and len(predecessors) > max_inputs: + if max_inputs == 1: + message = f"{node_name} cannot have more than one input." + else: + message = f"{node_name} supports up to {max_inputs} inputs." + self.errors.setdefault(node_id, []).append(message) predecessor_types = [ self.node_map[e["source"]]["type"] diff --git a/DashAI/front/src/api/pipeline.ts b/DashAI/front/src/api/pipeline.ts index 9ad5e7812..4de877701 100644 --- a/DashAI/front/src/api/pipeline.ts +++ b/DashAI/front/src/api/pipeline.ts @@ -23,6 +23,11 @@ export const getNodes = async () => { return response.data; }; +export const getNodeContracts = async () => { + const response = await api.get(`${pipelineEndpoint}/contracts`); + return response.data; +}; + export const updatePipeline = async ( id: number, formData: object, @@ -61,6 +66,11 @@ export const validatePipeline = async ( return response.data; }; +export const validateEdge = async (payload: object) => { + const response = await api.post(`${pipelineEndpoint}/validate_edge`, payload); + return response.data; +}; + export const getPipelinePredictionSummary = async (predictionId: string) => { const response = await api.get(`${pipelineEndpoint}/predict_summary`, { params: { diff --git a/DashAI/front/src/index.css b/DashAI/front/src/index.css index 3e669fe67..26d20df62 100644 --- a/DashAI/front/src/index.css +++ b/DashAI/front/src/index.css @@ -11,3 +11,18 @@ code { font-family: "IBM Plex Mono", source-code-pro, Menlo, Monaco, Consolas, "Courier New", monospace; } + +.react-flow__edge.edge-error .react-flow__edge-path { + stroke: #d32f2f; + stroke-width: 2; +} + +.react-flow__edge.edge-warning .react-flow__edge-path { + stroke: #f57c00; + stroke-width: 2; +} + +.react-flow__edge.edge-ok .react-flow__edge-path { + stroke: #4caf50; + stroke-width: 2; +} From c1938a34f3726441cfda6c6edb18c9803d151f46 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 22 Jun 2026 16:17:40 -0400 Subject: [PATCH 51/88] feat: track pipeline and node executions in the database --- .../d9c8f2a1b7e4_add_pipeline_run_tracking.py | 108 ++++++++++++ DashAI/back/core/enums/status.py | 16 ++ DashAI/back/dependencies/database/models.py | 165 +++++++++++++++--- 3 files changed, 262 insertions(+), 27 deletions(-) create mode 100644 DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py diff --git a/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py b/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py new file mode 100644 index 000000000..76a19a81f --- /dev/null +++ b/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py @@ -0,0 +1,108 @@ +"""Add pipeline run tracking tables. + +Revision ID: d9c8f2a1b7e4 +Revises: 0dc6fd4637a3 +Create Date: 2026-04-10 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "d9c8f2a1b7e4" +down_revision: Union[str, None] = "0dc6fd4637a3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +def _table_exists(table_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return table_name in inspector.get_table_names() + +def upgrade() -> None: + if not _table_exists("pipeline_run"): + op.create_table( + "pipeline_run", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.Column("steps", sa.JSON(), nullable=True), + sa.Column("edges", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), + sa.Column("delivery_time", sa.DateTime(), nullable=True), + sa.Column("start_time", sa.DateTime(), nullable=True), + sa.Column("end_time", sa.DateTime(), nullable=True), + sa.Column( + "status", + sa.Enum( + "NOT_STARTED", + "DELIVERED", + "STARTED", + "FINISHED", + "ERROR", + name="pipelinerunstatus", + ), + nullable=False, + ), + sa.Column("error_message", sa.String(), nullable=True), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + + if not _table_exists("pipeline_node_run"): + op.create_table( + "pipeline_node_run", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_run_id", sa.Integer(), nullable=False), + sa.Column("node_id", sa.String(), nullable=False), + sa.Column("node_type", sa.String(), nullable=False), + sa.Column("config", sa.JSON(), nullable=True), + sa.Column("input", sa.JSON(), nullable=True), + sa.Column("output", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), + sa.Column("delivery_time", sa.DateTime(), nullable=True), + sa.Column("start_time", sa.DateTime(), nullable=True), + sa.Column("end_time", sa.DateTime(), nullable=True), + sa.Column( + "status", + sa.Enum( + "NOT_STARTED", + "DELIVERED", + "STARTED", + "FINISHED", + "ERROR", + name="noderunstatus", + ), + nullable=False, + ), + sa.Column("error_message", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["pipeline_run_id"], ["pipeline_run.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + if not _table_exists("pipeline_node_artifact"): + op.create_table( + "pipeline_node_artifact", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("node_run_id", sa.Integer(), nullable=False), + sa.Column("key", sa.String(), nullable=False), + sa.Column("value", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["node_run_id"], ["pipeline_node_run.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + +def downgrade() -> None: + if _table_exists("pipeline_node_artifact"): + op.drop_table("pipeline_node_artifact") + + if _table_exists("pipeline_node_run"): + op.drop_table("pipeline_node_run") + + if _table_exists("pipeline_run"): + op.drop_table("pipeline_run") \ No newline at end of file diff --git a/DashAI/back/core/enums/status.py b/DashAI/back/core/enums/status.py index b66e70fd2..6685750a0 100644 --- a/DashAI/back/core/enums/status.py +++ b/DashAI/back/core/enums/status.py @@ -53,3 +53,19 @@ class PredictionStatus(Enum): STARTED = 2 FINISHED = 3 ERROR = 4 + + +class PipelineRunStatus(Enum): + NOT_STARTED = 0 + DELIVERED = 1 + STARTED = 2 + FINISHED = 3 + ERROR = 4 + + +class NodeRunStatus(Enum): + NOT_STARTED = 0 + DELIVERED = 1 + STARTED = 2 + FINISHED = 3 + ERROR = 4 diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index 125109f0a..61df6cc20 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -24,6 +24,8 @@ DatasetStatus, ExplainerStatus, ExplorerStatus, + NodeRunStatus, + PipelineRunStatus, PluginStatus, PredictionStatus, RunStatus, @@ -112,8 +114,6 @@ class ModelSession(Base): task_name: Mapped[str] = mapped_column(String, nullable=False) input_columns: Mapped[str] = mapped_column(JSON, nullable=False) output_columns: Mapped[str] = mapped_column(JSON, nullable=False) - - # Metrics per split train_metrics: Mapped[list[str]] = mapped_column(JSON, nullable=True) validation_metrics: Mapped[list[str]] = mapped_column(JSON, nullable=True) test_metrics: Mapped[list[str]] = mapped_column(JSON, nullable=True) @@ -147,22 +147,17 @@ class Run(Base): default=datetime.now, onupdate=datetime.now, ) - # model and parameters model_name: Mapped[str] = mapped_column(String) parameters: Mapped[JSON] = mapped_column(JSON) split_indexes: Mapped[str] = mapped_column(JSON, nullable=True) - # optimizer optimizer_name: Mapped[str] = mapped_column(String) optimizer_parameters: Mapped[JSON] = mapped_column(JSON) plot_history_path: Mapped[str] = mapped_column(String, nullable=True) plot_slice_path: Mapped[str] = mapped_column(String, nullable=True) plot_contour_path: Mapped[str] = mapped_column(String, nullable=True) plot_importance_path: Mapped[str] = mapped_column(String, nullable=True) - # goal metrics goal_metric: Mapped[str] = mapped_column(String) - # artifacts artifacts: Mapped[str] = mapped_column(JSON, nullable=True) - # metadata name: Mapped[str] = mapped_column(String) description: Mapped[str] = mapped_column(String, nullable=True) run_path: Mapped[str] = mapped_column(String, nullable=True) @@ -220,8 +215,6 @@ class Prediction(Base): delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) - - # Relationships run: Mapped["Run"] = relationship("Run", back_populates="predictions") dataset: Mapped["Dataset"] = relationship("Dataset", back_populates="predictions") @@ -408,7 +401,6 @@ class GenerativeProcess(Base): default=datetime.now, onupdate=datetime.now, ) - # metadata session_id: Mapped[int] = mapped_column( ForeignKey("generative_session.id", ondelete="CASCADE") ) @@ -496,16 +488,12 @@ class GenerativeSession(Base): default=datetime.now, onupdate=datetime.now, ) - # task name task_name: Mapped[str] = mapped_column(String, nullable=False) - # model and parameters model_name: Mapped[str] = mapped_column(String) parameters: Mapped[JSON] = mapped_column(JSON) - # metadata name: Mapped[str] = mapped_column(String, unique=True, nullable=False) description: Mapped[str] = mapped_column(String, nullable=True) - # Relationship with GenerativeSessionParameterHistory parameters_history: Mapped[List["GenerativeSessionParameterHistory"]] = ( relationship( "GenerativeSessionParameterHistory", @@ -514,7 +502,6 @@ class GenerativeSession(Base): ) ) - # Relationship with GenerativeProcess processes: Mapped[List["GenerativeProcess"]] = relationship( "GenerativeProcess", cascade="all, delete-orphan", back_populates="session" ) @@ -532,13 +519,10 @@ class Pipeline(Base): exploration: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) train: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) prediction: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) - - # New JSON columns for atomized Train sub-node results split_data: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) task_and_model: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) metrics_result: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) - # Relationships to new node tables split_data_nodes: Mapped[List["SplitDataNode"]] = relationship( "SplitDataNode", cascade="all, delete-orphan", back_populates="pipeline" ) @@ -548,6 +532,9 @@ class Pipeline(Base): metrics_nodes: Mapped[List["MetricsNode"]] = relationship( "MetricsNode", cascade="all, delete-orphan", back_populates="pipeline" ) + pipeline_runs: Mapped[List["PipelineRun"]] = relationship( + "PipelineRun", cascade="all, delete-orphan", back_populates="pipeline" + ) class SplitDataNode(Base): @@ -625,6 +612,139 @@ class MetricsNode(Base): ) +class PipelineRun(Base): + __tablename__ = "pipeline_run" + """ + Table to store execution runs for a pipeline. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_id: Mapped[int] = mapped_column( + ForeignKey("pipeline.id", ondelete="CASCADE"), nullable=False + ) + steps: Mapped[List[Dict[str, Any]]] = mapped_column(JSON, nullable=True) + edges: Mapped[List[Dict[str, Any]]] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + status: Mapped[Enum] = mapped_column( + Enum(PipelineRunStatus), + nullable=False, + default=PipelineRunStatus.NOT_STARTED, + ) + error_message: Mapped[str] = mapped_column(String, nullable=True) + + pipeline: Mapped["Pipeline"] = relationship( + "Pipeline", back_populates="pipeline_runs" + ) + node_runs: Mapped[List["NodeRun"]] = relationship( + "NodeRun", cascade="all, delete-orphan", back_populates="pipeline_run" + ) + + def set_status_as_delivered(self) -> None: + """ + Update the status of the pipeline run to delivered and set delivery_time to now. + """ + self.status = PipelineRunStatus.DELIVERED + self.delivery_time = datetime.now() + + def set_status_as_started(self) -> None: + """Update the status of the pipeline run to started and set start_time to now.""" + self.status = PipelineRunStatus.STARTED + self.start_time = datetime.now() + + def set_status_as_finished(self) -> None: + """Update the status of the pipeline run to finished and set end_time to now.""" + self.status = PipelineRunStatus.FINISHED + self.end_time = datetime.now() + + def set_status_as_error(self, error_message: str | None = None) -> None: + """Update the status of the pipeline run to error.""" + self.status = PipelineRunStatus.ERROR + self.error_message = error_message + self.end_time = datetime.now() + + +class NodeRun(Base): + __tablename__ = "pipeline_node_run" + """ + Table to store execution runs for individual pipeline nodes. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_run_id: Mapped[int] = mapped_column( + ForeignKey("pipeline_run.id", ondelete="CASCADE"), nullable=False + ) + node_id: Mapped[str] = mapped_column(String, nullable=False) + node_type: Mapped[str] = mapped_column(String, nullable=False) + config: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + input: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + output: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + status: Mapped[Enum] = mapped_column( + Enum(NodeRunStatus), + nullable=False, + default=NodeRunStatus.NOT_STARTED, + ) + error_message: Mapped[str] = mapped_column(String, nullable=True) + + pipeline_run: Mapped["PipelineRun"] = relationship( + "PipelineRun", back_populates="node_runs" + ) + artifacts: Mapped[List["NodeArtifact"]] = relationship( + "NodeArtifact", cascade="all, delete-orphan", back_populates="node_run" + ) + + def set_status_as_delivered(self) -> None: + """ + Update the status of the node run to delivered and set delivery_time to now. + """ + self.status = NodeRunStatus.DELIVERED + self.delivery_time = datetime.now() + + def set_status_as_started(self) -> None: + """Update the status of the node run to started and set start_time to now.""" + self.status = NodeRunStatus.STARTED + self.start_time = datetime.now() + + def set_status_as_finished(self) -> None: + """Update the status of the node run to finished and set end_time to now.""" + self.status = NodeRunStatus.FINISHED + self.end_time = datetime.now() + + def set_status_as_error(self, error_message: str | None = None) -> None: + """Update the status of the node run to error.""" + self.status = NodeRunStatus.ERROR + self.error_message = error_message + self.end_time = datetime.now() + + +class NodeArtifact(Base): + __tablename__ = "pipeline_node_artifact" + """ + Table to store artifacts emitted by a node run. + """ + id: Mapped[int] = mapped_column(primary_key=True) + node_run_id: Mapped[int] = mapped_column( + ForeignKey("pipeline_node_run.id", ondelete="CASCADE"), nullable=False + ) + key: Mapped[str] = mapped_column(String, nullable=False) + value: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + + node_run: Mapped["NodeRun"] = relationship( + "NodeRun", back_populates="artifacts" + ) + + class Converter(Base): __tablename__ = "converter" """ @@ -651,8 +771,6 @@ class Converter(Base): delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) - - # Relationships notebook: Mapped["Notebook"] = relationship(back_populates="converters") def set_status_as_delivered(self) -> None: @@ -699,7 +817,6 @@ class Notebook(Base): file_path: Mapped[str] = mapped_column(String, nullable=False) name: Mapped[str] = mapped_column(String, nullable=True) description: Mapped[str] = mapped_column(String, nullable=True) - # Relationships explorers: Mapped[List["Explorer"]] = relationship( back_populates="notebook", cascade="all, delete-orphan" ) @@ -725,8 +842,6 @@ class GenerativeSessionParameterHistory(Base): DateTime, default=datetime.now, ) - - # Relationship with GenerativeSession session = relationship( "GenerativeSession", back_populates="parameters_history", @@ -750,21 +865,17 @@ class Explorer(Base): default=datetime.now, onupdate=datetime.now, ) - # explorer columns: Mapped[JSON] = mapped_column(JSON, nullable=False) exploration_type: Mapped[str] = mapped_column(String, nullable=False) parameters: Mapped[JSON] = mapped_column(JSON, nullable=False) exploration_path: Mapped[str] = mapped_column(String, nullable=True) - # Metadata name: Mapped[str] = mapped_column(String, nullable=True) - delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) status: Mapped[Enum] = mapped_column( Enum(ExplorerStatus), nullable=False, default=ExplorerStatus.NOT_STARTED ) - # Relationships notebook: Mapped["Notebook"] = relationship(back_populates="explorers") def set_status_as_delivered(self) -> None: From a8127c06ea76c0367cb332ab506c0fb62d1af648 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 22 Jun 2026 16:18:03 -0400 Subject: [PATCH 52/88] feat: turn PipelineJob into a per-node execution orchestrator --- .../dependencies/database/sqlite_database.py | 13 +- DashAI/back/job/node_job.py | 222 +++++++++++++ DashAI/back/job/pipeline_job.py | 313 +++++++++++------- DashAI/back/pipeline/metrics_node.py | 65 +++- DashAI/back/pipeline/prediction_node.py | 157 +++++---- 5 files changed, 566 insertions(+), 204 deletions(-) create mode 100644 DashAI/back/job/node_job.py diff --git a/DashAI/back/dependencies/database/sqlite_database.py b/DashAI/back/dependencies/database/sqlite_database.py index e450a184f..6ce857681 100644 --- a/DashAI/back/dependencies/database/sqlite_database.py +++ b/DashAI/back/dependencies/database/sqlite_database.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Tuple -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker @@ -41,9 +41,18 @@ def setup_sqlite_db(config: Dict[str, str]) -> Tuple[Engine, sessionmaker]: engine: Engine = create_engine( db_url, echo=config["LOGGING_LEVEL"] == logging.DEBUG, - connect_args={"check_same_thread": False}, + connect_args={"check_same_thread": False, "timeout": 30}, + pool_pre_ping=True, ) + @event.listens_for(engine, "connect") + def _set_sqlite_pragma(dbapi_connection, connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL;") + cursor.execute("PRAGMA synchronous=NORMAL;") + cursor.execute("PRAGMA busy_timeout=30000;") + cursor.close() + session_factory = sessionmaker( autocommit=False, autoflush=True, diff --git a/DashAI/back/job/node_job.py b/DashAI/back/job/node_job.py new file mode 100644 index 000000000..9ec0da131 --- /dev/null +++ b/DashAI/back/job/node_job.py @@ -0,0 +1,222 @@ +import asyncio +import json +import logging +import threading +import time +from typing import TYPE_CHECKING, Any, Dict + +from kink import di, inject +from sqlalchemy import exc + +from DashAI.back.dependencies.database.models import NodeRun +from DashAI.back.job.base_job import BaseJob, JobError + +if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + + from DashAI.back.dependencies.registry import ComponentRegistry + + +log = logging.getLogger(__name__) + +_DB_WRITE_LOCK = threading.Lock() + + +class NodeJob(BaseJob): + """Execute a single pipeline node and persist run metadata.""" + + @inject + def set_status_as_delivered( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + node_run_id = self.kwargs.get("node_run_id") + if not node_run_id: + return + + with session_factory() as db: + node_run: NodeRun | None = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError(f"NodeRun with id {node_run_id} not found.") + + try: + node_run.set_status_as_delivered() + self._commit_with_retry(db) + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Error while setting node run status as delivered.") from e + + @inject + def set_status_as_error( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + node_run_id = self.kwargs.get("node_run_id") + if not node_run_id: + return + + with session_factory() as db: + node_run: NodeRun | None = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError(f"NodeRun with id {node_run_id} not found.") + + try: + node_run.set_status_as_error("Node job failed.") + self._commit_with_retry(db) + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Error while setting node run status as error.") from e + + def get_job_name(self) -> str: + node_type = self.kwargs.get("node_type", "Node") + node_id = self.kwargs.get("node_id") + if node_id: + return f"{node_type} ({node_id})" + return str(node_type) + + @inject + def run( + self, + session_factory: "sessionmaker" = lambda di: di["session_factory"], + component_registry: "ComponentRegistry" = lambda di: di["component_registry"], + ) -> Dict[str, Any]: + node_type = self.kwargs.get("node_type") + node_config = self.kwargs.get("node_config", {}) + context = self.kwargs.get("context", {}) + node_run_id = self.kwargs.get("node_run_id") + + if not node_type: + raise JobError("Node type is required to execute a node job.") + if not node_run_id: + try: + node_class = component_registry[node_type]["class"] + except KeyError as e: + raise JobError( + f"Component type {node_type} not found in registry" + ) from e + + try: + node_instance = node_class(**node_config) + except Exception as e: + raise JobError( + f"Error instantiating node {node_type}: {str(e)}" + ) from e + + return self._run_node_instance(node_instance, context) + + with session_factory() as db: + node_run: NodeRun | None = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError(f"NodeRun with id {node_run_id} not found.") + + try: + node_run.set_status_as_started() + node_run.input = self._safe_payload(context) + self._commit_with_retry(db) + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Error while setting node run status as started.") from e + + try: + node_class = component_registry[node_type]["class"] + except KeyError as e: + raise JobError(f"Component type {node_type} not found in registry") from e + + try: + node_instance = node_class(**node_config) + except Exception as e: + raise JobError( + f"Error instantiating node {node_type}: {str(e)}" + ) from e + + try: + output = self._run_node_instance(node_instance, context) + except Exception as e: + with session_factory() as db: + node_run = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError( + f"NodeRun with id {node_run_id} not found after failure." + ) from e + + try: + node_run.set_status_as_error(str(e)) + self._commit_with_retry(db) + except exc.SQLAlchemyError as db_error: + log.exception(db_error) + raise JobError( + "Error while setting node run status as error." + ) from db_error + + raise + + with session_factory() as db: + node_run = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError(f"NodeRun with id {node_run_id} not found.") + + try: + node_run.output = self._safe_payload(output) + node_run.set_status_as_finished() + self._commit_with_retry(db) + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Error while setting node run status as finished.") from e + + return output + + @staticmethod + def _run_node_instance(node_instance: Any, context: Dict[str, Any]) -> Dict[str, Any]: + result = node_instance.run(context=context) + if asyncio.iscoroutine(result): + return asyncio.run(result) + return result + + @staticmethod + def _commit_with_retry( + db, + attempts: int = 5, + base_delay: float = 0.1, + readd: list[Any] | None = None, + ) -> None: + readd = readd or [] + with _DB_WRITE_LOCK: + for attempt in range(attempts): + try: + for obj in readd: + if obj is not None: + db.add(obj) + db.commit() + return + except exc.OperationalError as error: + if "database is locked" not in str(error).lower(): + raise + db.rollback() + if attempt >= attempts - 1: + raise + time.sleep(base_delay * (2**attempt)) + + @staticmethod + def _safe_payload(value: Any) -> Any: + if isinstance(value, dict): + sanitized: Dict[str, Any] = {} + for key, item in value.items(): + if key == "dataset": + continue + sanitized_item = NodeJob._safe_payload(item) + if sanitized_item is not None: + sanitized[key] = sanitized_item + return sanitized + if isinstance(value, list): + sanitized_list = [] + for item in value: + sanitized_item = NodeJob._safe_payload(item) + if sanitized_item is not None: + sanitized_list.append(sanitized_item) + return sanitized_list + if isinstance(value, (str, int, float, bool)) or value is None: + return value + + try: + json.dumps(value) + return value + except TypeError: + return str(value) diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index fe8d026a1..0cc0e840f 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -4,14 +4,15 @@ from kink import di, inject -from DashAI.back.dependencies.database.models import Pipeline +from sqlalchemy import exc + +from DashAI.back.dependencies.database.models import NodeRun, Pipeline, PipelineRun from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.job.node_job import NodeJob if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker - from DashAI.back.dependencies.registry import ComponentRegistry - log = logging.getLogger(__name__) @@ -45,7 +46,6 @@ def get_job_name(self) -> str: def run( self, session_factory: "sessionmaker" = lambda di: di["session_factory"], - component_registry: "ComponentRegistry" = lambda di: di["component_registry"], ) -> None: pipeline_id: int | None = self.kwargs.get("id") if not pipeline_id: @@ -72,103 +72,149 @@ def run( pipeline.task_and_model = None pipeline.metrics_result = None + pipeline_run = PipelineRun( + pipeline_id=pipeline_id, + steps=steps, + edges=edges, + ) + pipeline_run.set_status_as_started() + db.add(pipeline_run) + NodeJob._commit_with_retry(db, readd=[pipeline_run]) + db.refresh(pipeline_run) + log.info(f"Starting pipeline execution for pipeline {pipeline_id}...") base_context: Dict[str, Any] = {"pipeline_id": pipeline_id} node_contexts: Dict[str, Dict[str, Any]] = {} predecessor_map = self._build_predecessor_map(steps, edges) + successor_map = self._build_successor_map(steps, edges) + step_map = {step.get("id"): step for step in steps if step.get("id")} + + async def _execute_graph() -> None: + pending = set(step_map.keys()) + if not pending: + raise JobError("Pipeline steps are missing ids.") + + ready = [ + node_id + for node_id in pending + if not predecessor_map.get(node_id) + ] + if not ready: + raise JobError( + "No starting nodes found. Check for cycles or missing roots." + ) + + task_map: Dict[asyncio.Task, str] = {} + update_lock = asyncio.Lock() + exclusive_lock = asyncio.Lock() + exclusive_types = {"TaskAndModel", "Train"} - async def _execute_steps() -> None: - for idx, step in enumerate(steps): - node_id = step.get("id") + async def _run_node(node_id: str) -> None: + step = step_map.get(node_id, {}) node_type = step.get("type") node_config = step.get("config", {}) - predecessor_id = None - node_context = None - - log.debug( - f"Pipeline {pipeline_id}: " - f"Executing step {idx + 1}/{len(steps)} - " - f"{node_type} ({node_id})" + node_context = self._build_node_context( + node_id=node_id, + predecessor_map=predecessor_map, + node_contexts=node_contexts, + base_context=base_context, ) + node_context["_node_id"] = node_id + node_run_id = None try: - node_class = component_registry[node_type]["class"] - except KeyError as e: - error_msg = f"Component type {node_type} not found in registry" - raise JobError( - f"Error in node {node_id} ({node_type}): {error_msg}" - ) from e + with session_factory() as run_db: + node_run = NodeRun( + pipeline_run_id=pipeline_run.id, + node_id=node_id, + node_type=node_type or "Unknown", + config=node_config, + ) + node_run.set_status_as_delivered() + run_db.add(node_run) + NodeJob._commit_with_retry(run_db, readd=[node_run]) + node_run_id = node_run.id + except exc.OperationalError as e: + if "database is locked" in str(e).lower(): + log.warning( + "NodeRun insert skipped due to database lock." + ) + node_run_id = None + else: + raise + + node_job = NodeJob( + node_run_id=node_run_id, + node_id=node_id, + node_type=node_type, + node_config=node_config, + context=node_context, + ) - try: - node_instance = node_class(**node_config) - except Exception as e: - error_msg = f"Error in node {node_id} ({node_type}): {str(e)}" - log.exception(error_msg) - raise JobError(error_msg) from e + if node_type in exclusive_types: + async with exclusive_lock: + output = await asyncio.to_thread(node_job.run) + else: + output = await asyncio.to_thread(node_job.run) - try: - predecessors = predecessor_map.get(node_id, []) - predecessor_id = predecessors[0] if predecessors else None - node_context = self._build_node_context( - node_id=node_id, - predecessor_map=predecessor_map, - node_contexts=node_contexts, - base_context=base_context, - ) - node_context["_node_id"] = node_id - if node_context.get("dataset") is not None: - dataset_name = node_context.get("dataset_name") - try: - dataset_cols = len( - list(node_context["dataset"].features.keys()) - ) - except Exception: - dataset_cols = "" - log.debug( - "Node %s (%s) context from predecessor=%s, " - "dataset_name=%s, dataset_columns=%s", - node_id, - node_type, - predecessor_id, - dataset_name, - dataset_cols, - ) - output = await node_instance.run(context=node_context) + async with update_lock: self._update_context( node_context, pipeline, node_type, node_id, output ) - if node_id: - node_contexts[node_id] = node_context - log.debug(f"Node {node_id} executed successfully.") - except Exception as e: - dataset_name = "" - dataset_cols = "" - if isinstance(node_context, dict): - dataset_name = node_context.get("dataset_name", "") - if node_context.get("dataset") is not None: - try: - dataset_cols = len( - list(node_context["dataset"].features.keys()) - ) - except Exception: - dataset_cols = "" - - error_msg = ( - f"Error in node {node_id} ({node_type}): {str(e)} " - f"[predecessor={predecessor_id}, " - f"dataset_name={dataset_name}, " - f"dataset_columns={dataset_cols}]" - ) - log.exception(error_msg) - raise JobError(error_msg) from e + node_contexts[node_id] = node_context + + async def _schedule(node_id: str) -> None: + task = asyncio.create_task(_run_node(node_id)) + task_map[task] = node_id - # Run async nodes in a new event loop - asyncio.run(_execute_steps()) + for node_id in ready: + await _schedule(node_id) - log.info(f"Pipeline {pipeline_id} execution completed successfully.") - db.add(pipeline) - db.commit() - self.set_status_as_delivered() + while task_map: + done, _ = await asyncio.wait( + task_map.keys(), return_when=asyncio.FIRST_COMPLETED + ) + + for task in done: + node_id = task_map.pop(task) + pending.discard(node_id) + + try: + task.result() + except Exception as e: + for pending_task in task_map.keys(): + pending_task.cancel() + raise JobError( + f"Error executing node {node_id}: {str(e)}" + ) from e + + for successor in successor_map.get(node_id, []): + if successor not in pending: + continue + predecessors = predecessor_map.get(successor, []) + if all(pred in node_contexts for pred in predecessors): + await _schedule(successor) + + if pending: + raise JobError( + "Pipeline execution stalled: unresolved dependencies remain." + ) + + try: + asyncio.run(_execute_graph()) + pipeline_run.set_status_as_finished() + db.add(pipeline) + db.add(pipeline_run) + NodeJob._commit_with_retry(db, readd=[pipeline_run, pipeline]) + self.set_status_as_delivered() + log.info( + f"Pipeline {pipeline_id} execution completed successfully." + ) + except Exception as e: + pipeline_run.set_status_as_error(str(e)) + db.add(pipeline_run) + NodeJob._commit_with_retry(db, readd=[pipeline_run]) + raise def _build_predecessor_map( self, @@ -187,6 +233,23 @@ def _build_predecessor_map( return predecessor_map + def _build_successor_map( + self, + steps: List[Dict[str, Any]], + edges: List[Dict[str, Any]], + ) -> Dict[str, List[str]]: + """Build a mapping source_node_id -> [target_node_ids].""" + step_ids = {step.get("id") for step in steps if step.get("id")} + successor_map: Dict[str, List[str]] = {step_id: [] for step_id in step_ids} + + for edge in edges: + source = edge.get("source") + target = edge.get("target") + if source in step_ids and target in step_ids: + successor_map[source].append(target) + + return successor_map + def _build_node_context( self, node_id: str, @@ -194,28 +257,28 @@ def _build_node_context( node_contexts: Dict[str, Dict[str, Any]], base_context: Dict[str, Any], ) -> Dict[str, Any]: - """Build execution context for a node from its unique predecessor branch.""" + """Build execution context for a node from all predecessor branches.""" if not node_id: raise JobError("Pipeline step is missing an id.") predecessors = predecessor_map.get(node_id, []) - if len(predecessors) > 1: - raise JobError( - f"Node {node_id} has multiple inputs, which is not supported." - ) - if not predecessors: return dict(base_context) - predecessor_id = predecessors[0] - predecessor_context = node_contexts.get(predecessor_id) - if predecessor_context is None: - raise JobError( - f"Cannot resolve predecessor context for node {node_id} " - f"from source {predecessor_id}." - ) - - return dict(predecessor_context) + merged_context: Dict[str, Any] = dict(base_context) + branches: List[Dict[str, Any]] = [] + for predecessor_id in predecessors: + predecessor_context = node_contexts.get(predecessor_id) + if predecessor_context is None: + raise JobError( + f"Cannot resolve predecessor context for node {node_id} " + f"from source {predecessor_id}." + ) + branches.append(dict(predecessor_context)) + merged_context.update(predecessor_context) + + merged_context["_branches"] = branches + return merged_context def _update_context( self, @@ -303,15 +366,23 @@ def _update_context( if isinstance(pipeline.metrics_result, dict) else {} ) - metrics_key = node_id or f"MetricsEval-{len(aggregated_metrics_result)}" - aggregated_metrics_result[metrics_key] = { - "node_id": node_id, - "model_node_id": context.get("model_node_id"), - "dataset_name": context.get("dataset_name"), - "model_name": context.get("model_name"), - "task_name": context.get("task_name"), - **current_metrics_result, - } + if isinstance(current_metrics_result, list): + for idx, entry in enumerate(current_metrics_result): + metrics_key = entry.get("model_node_id") or f"{node_id}-{idx}" + aggregated_metrics_result[metrics_key] = { + "node_id": node_id, + **entry, + } + else: + metrics_key = node_id or f"MetricsEval-{len(aggregated_metrics_result)}" + aggregated_metrics_result[metrics_key] = { + "node_id": node_id, + "model_node_id": context.get("model_node_id"), + "dataset_name": context.get("dataset_name"), + "model_name": context.get("model_name"), + "task_name": context.get("task_name"), + **current_metrics_result, + } pipeline.metrics_result = aggregated_metrics_result elif node_type == "RetrieveModel": context["retrieve"] = output.get("retrieve") @@ -324,15 +395,23 @@ def _update_context( if isinstance(pipeline.prediction, dict) else {} ) - prediction_key = node_id or f"Prediction-{len(aggregated_prediction)}" - aggregated_prediction[prediction_key] = { - "node_id": node_id, - "model_node_id": context.get("model_node_id"), - "dataset_name": context.get("dataset_name"), - "model_name": context.get("model_name"), - "task_name": context.get("task_name"), - "prediction": current_prediction, - } + if isinstance(current_prediction, list): + for idx, entry in enumerate(current_prediction): + prediction_key = entry.get("model_node_id") or f"{node_id}-{idx}" + aggregated_prediction[prediction_key] = { + "node_id": node_id, + **entry, + } + else: + prediction_key = node_id or f"Prediction-{len(aggregated_prediction)}" + aggregated_prediction[prediction_key] = { + "node_id": node_id, + "model_node_id": context.get("model_node_id"), + "dataset_name": context.get("dataset_name"), + "model_name": context.get("model_name"), + "task_name": context.get("task_name"), + "prediction": current_prediction, + } pipeline.prediction = aggregated_prediction else: context[node_id] = output diff --git a/DashAI/back/pipeline/metrics_node.py b/DashAI/back/pipeline/metrics_node.py index 255b55d05..fc8a464f1 100644 --- a/DashAI/back/pipeline/metrics_node.py +++ b/DashAI/back/pipeline/metrics_node.py @@ -9,7 +9,6 @@ log = logging.getLogger(__name__) - class MetricsEval(BaseJob): """Pipeline node that evaluates the trained model with selected metrics. @@ -28,8 +27,6 @@ def __init__(self, metrics: List[str]) -> None: super().__init__(kwargs={"metrics": metrics}) self.metrics = metrics - # -- BaseJob interface --------------------------------------------------- - def set_status_as_delivered(self) -> None: log.debug("MetricsEval executed successfully.") @@ -39,8 +36,6 @@ def set_status_as_error(self) -> None: def get_job_name(self) -> str: return "MetricsEval" - # -- Execution ----------------------------------------------------------- - async def run( self, context: Dict[str, Any], @@ -57,17 +52,6 @@ async def run( dict ``{"metrics_result": { ... }}`` with per-metric scores. """ - factory = context.get("model_factory") - x = context.get("trained_x") - y = context.get("trained_y") - - if factory is None or x is None or y is None: - raise JobError( - "MetricsEval requires a trained model. " - "Make sure TaskAndModel runs before this node." - ) - - # Resolve metric classes from registry all_metrics = { component_dict["name"]: component_dict for component_dict in component_registry(di).get_components_by_types( @@ -86,6 +70,55 @@ async def run( if not metric_classes: raise JobError("No valid metrics found to evaluate.") + branches = context.get("_branches") + if branches: + entries: List[Dict[str, Any]] = [] + for branch in branches: + factory = branch.get("model_factory") + x = branch.get("trained_x") + y = branch.get("trained_y") + + if factory is None or x is None or y is None: + log.warning( + "Skipping metrics for branch without a trained model." + ) + continue + + try: + model_metrics = factory.evaluate(x, y, metric_classes) + except Exception as e: + log.exception(e) + raise JobError("Metrics calculation failed") from e + + entries.append( + { + "model_node_id": branch.get("model_node_id"), + "dataset_name": branch.get("dataset_name"), + "model_name": branch.get("model_name"), + "task_name": branch.get("task_name"), + "metric_names": self.metrics, + "results": model_metrics, + } + ) + + if not entries: + raise JobError( + "MetricsEval requires a trained model. " + "Make sure TaskAndModel runs before this node." + ) + + return {"metrics_result": entries} + + factory = context.get("model_factory") + x = context.get("trained_x") + y = context.get("trained_y") + + if factory is None or x is None or y is None: + raise JobError( + "MetricsEval requires a trained model. " + "Make sure TaskAndModel runs before this node." + ) + try: model_metrics = factory.evaluate(x, y, metric_classes) except Exception as e: diff --git a/DashAI/back/pipeline/prediction_node.py b/DashAI/back/pipeline/prediction_node.py index 1509f7c6a..028ab8a46 100644 --- a/DashAI/back/pipeline/prediction_node.py +++ b/DashAI/back/pipeline/prediction_node.py @@ -41,61 +41,8 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: import numpy as np - loaded_dataset = context["dataset"] - model = context["model_class"] config = di["config"] - sqlite_local = config["LOCAL_PATH"] - pipeline_id = context.get("pipeline_id") - model_node_id = context.get("model_node_id") - - model_path = context["model_path"] - if pipeline_id is not None and model_node_id: - node_model_path = os.path.join( - sqlite_local, - "pipelines", - "train", - str(pipeline_id), - str(model_node_id), - "model", - ) - if os.path.exists(node_model_path): - model_path = node_model_path - - try: - trained_model: BaseModel = model.load(model_path) - except Exception as e: - log.exception(e) - raise JobError( - "Failed to load trained model for Prediction. " - f"model_name={context.get('model_name')}, " - f"model_path={model_path}, " - f"model_node_id={model_node_id}, " - f"pipeline_id={pipeline_id}" - ) from e - - try: - prepared_dataset = loaded_dataset.select_columns(context["input_columns"]) - y_pred_proba = np.array(trained_model.predict(prepared_dataset)) - - if isinstance(y_pred_proba[0], str): - y_pred = y_pred_proba - elif y_pred_proba.ndim == 2: - y_pred = np.argmax(y_pred_proba, axis=1) - else: - y_pred = y_pred_proba - - except ValueError as ve: - log.error(f"Validation Error: {ve}") - raise HTTPException( - status_code=400, - detail=f"Invalid columns selected: {str(ve)}", - ) from ve - except Exception as e: - log.error(e) - raise JobError( - "Model prediction failed", - ) from e try: path = os.path.join(sqlite_local, "pipelines", "predictions") @@ -111,22 +58,94 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: continue next_id = max(existing_ids, default=0) + 1 - json_name = f"prediction_{next_id}.json" - - json_data = { - "metadata": { - "id": next_id, - "model_name": context["model_name"], - "dataset_name": context["dataset_name"], - "task_name": context["task_name"], - }, - "prediction": y_pred.tolist(), - } - - with open(os.path.join(path, json_name), "w") as json_file: - json.dump(json_data, json_file, indent=4) - - return {"prediction": json_name} + + def _predict_for_context(pred_context: Dict[str, Any]) -> Dict[str, Any]: + nonlocal next_id + loaded_dataset = pred_context["dataset"] + model = pred_context["model_class"] + pipeline_id = pred_context.get("pipeline_id") + model_node_id = pred_context.get("model_node_id") + + model_path = pred_context["model_path"] + if pipeline_id is not None and model_node_id: + node_model_path = os.path.join( + sqlite_local, + "pipelines", + "train", + str(pipeline_id), + str(model_node_id), + "model", + ) + if os.path.exists(node_model_path): + model_path = node_model_path + + try: + trained_model: BaseModel = model.load(model_path) + except Exception as e: + log.exception(e) + raise JobError( + "Failed to load trained model for Prediction. " + f"model_name={pred_context.get('model_name')}, " + f"model_path={model_path}, " + f"model_node_id={model_node_id}, " + f"pipeline_id={pipeline_id}" + ) from e + + try: + prepared_dataset = loaded_dataset.select_columns( + pred_context["input_columns"] + ) + y_pred_proba = np.array(trained_model.predict(prepared_dataset)) + + if isinstance(y_pred_proba[0], str): + y_pred = y_pred_proba + elif y_pred_proba.ndim == 2: + y_pred = np.argmax(y_pred_proba, axis=1) + else: + y_pred = y_pred_proba + + except ValueError as ve: + log.error(f"Validation Error: {ve}") + raise HTTPException( + status_code=400, + detail=f"Invalid columns selected: {str(ve)}", + ) from ve + except Exception as e: + log.error(e) + raise JobError( + "Model prediction failed", + ) from e + + json_name = f"prediction_{next_id}.json" + json_data = { + "metadata": { + "id": next_id, + "model_name": pred_context["model_name"], + "dataset_name": pred_context["dataset_name"], + "task_name": pred_context["task_name"], + }, + "prediction": y_pred.tolist(), + } + + with open(os.path.join(path, json_name), "w") as json_file: + json.dump(json_data, json_file, indent=4) + + next_id += 1 + return { + "prediction": json_name, + "model_node_id": pred_context.get("model_node_id"), + "dataset_name": pred_context.get("dataset_name"), + "model_name": pred_context.get("model_name"), + "task_name": pred_context.get("task_name"), + } + + branches = context.get("_branches") + if branches: + entries = [_predict_for_context(branch) for branch in branches] + return {"prediction": entries} + + entry = _predict_for_context(context) + return {"prediction": entry["prediction"]} except Exception as e: log.exception(e) raise JobError("Can not save prediction to json file") from e From da7be6e8ffcc531101c6c5b8b54b3a523dc8e66c Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 22 Jun 2026 16:19:06 -0400 Subject: [PATCH 53/88] feat: support branched flows with multiple node output handles --- .../components/pipelines/PipelineDesigner.jsx | 37 ++- .../src/components/pipelines/nodeHelp.jsx | 3 +- .../src/components/pipelines/nodeTypes.jsx | 4 +- .../pipelines/nodes/DataSelectorNode.jsx | 7 +- .../pipelines/nodes/MetricsEvalNode.jsx | 141 +++++----- .../pipelines/nodes/SplitDataNode.jsx | 175 +++++++------ .../pipelines/nodes/TaskAndModelNode.jsx | 241 ++++++++++-------- DashAI/front/src/hooks/usePipelineState.js | 119 +++++++-- 8 files changed, 413 insertions(+), 314 deletions(-) diff --git a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx index 1ea2fd9d3..d91ab1838 100644 --- a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx +++ b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx @@ -16,6 +16,8 @@ function PipelineDesigner({ setEdges, onNodesChange, onEdgesChange, + onConnect, + onEdgeDoubleClick, nodeTypes, onNodeClick, onNodeHelp, @@ -53,19 +55,27 @@ function PipelineDesigner({ ); }, [canvasMode, setNodes]); - const onConnect = (params) => { - setEdges((eds) => - addEdge( - { - ...params, - markerEnd: { - type: "arrowclosed", + const handleConnect = useCallback( + (params) => { + if (onConnect) { + onConnect(params); + return; + } + + setEdges((eds) => + addEdge( + { + ...params, + markerEnd: { + type: "arrowclosed", + }, }, - }, - eds, - ), - ); - }; + eds, + ), + ); + }, + [onConnect, setEdges], + ); const onDragOver = useCallback((event) => { event.preventDefault(); @@ -144,7 +154,8 @@ function PipelineDesigner({ edges={edges} onNodesChange={onNodesChange} onEdgesChange={onEdgesChange} - onConnect={onConnect} + onConnect={handleConnect} + onEdgeDoubleClick={onEdgeDoubleClick} onDragOver={onDragOver} onDrop={onDrop} onNodeClick={onNodeHelp} diff --git a/DashAI/front/src/components/pipelines/nodeHelp.jsx b/DashAI/front/src/components/pipelines/nodeHelp.jsx index c0ac8677a..7821631b0 100644 --- a/DashAI/front/src/components/pipelines/nodeHelp.jsx +++ b/DashAI/front/src/components/pipelines/nodeHelp.jsx @@ -2,7 +2,8 @@ let nodeHelp = { Pipeline: { description: `Design a machine learning workflow by dragging and connecting nodes. Single-click a node to see more information. - Double-click a node to configure parameters.`, + Double-click a node to configure parameters. + Double-click an edge to remove the connection.`, }, }; diff --git a/DashAI/front/src/components/pipelines/nodeTypes.jsx b/DashAI/front/src/components/pipelines/nodeTypes.jsx index e53e53486..48510906c 100644 --- a/DashAI/front/src/components/pipelines/nodeTypes.jsx +++ b/DashAI/front/src/components/pipelines/nodeTypes.jsx @@ -1,9 +1,9 @@ -import { getNodes } from "../../api/pipeline"; +import { getNodeContracts } from "../../api/pipeline"; import CustomNode from "./CustomNode"; export async function getNodeTypes() { try { - const response = await getNodes(); + const response = await getNodeContracts(); if (Array.isArray(response)) { return response; } else { diff --git a/DashAI/front/src/components/pipelines/nodes/DataSelectorNode.jsx b/DashAI/front/src/components/pipelines/nodes/DataSelectorNode.jsx index e3ece3ddd..473863102 100644 --- a/DashAI/front/src/components/pipelines/nodes/DataSelectorNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/DataSelectorNode.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useMemo } from "react"; +import React, { useState, useEffect, useMemo, useRef } from "react"; import PropTypes from "prop-types"; import DatasetModal from "../../../components/datasets/DatasetModal"; import { validateNode } from "../../../api/pipeline"; @@ -20,6 +20,7 @@ function DataSelectorNode({ onClose, onSave, savedConfig = null }) { const [openModal, setOpenModal] = useState(false); const [datasets, setDatasets] = useState([]); const [loading, setLoading] = useState(true); + const pollingRef = useRef(null); const { enqueueSnackbar } = useSnackbar(); const [validationStatus, setValidationStatus] = useState(""); const { i18n } = useTranslation(); @@ -49,11 +50,8 @@ function DataSelectorNode({ onClose, onSave, savedConfig = null }) { const latestDatasets = await fetchDatasets(); if (newDatasetId) { setDatasetId(newDatasetId); - - // Check if the dataset is still processing (file_path not yet set) const newDs = latestDatasets.find((d) => d.id === newDatasetId); if (!newDs || !newDs.file_path) { - // Poll every 2 s until the background job sets file_path if (pollingRef.current) clearInterval(pollingRef.current); pollingRef.current = setInterval(async () => { try { @@ -76,7 +74,6 @@ function DataSelectorNode({ onClose, onSave, savedConfig = null }) { const handleCloseModal = () => setOpenModal(false); const handleSave = async () => { - // Re-fetch to obtain the latest file_path (set by the background job) let latestDatasets; try { latestDatasets = await getDatasets(); diff --git a/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx b/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx index 3d51221d7..150b7064d 100644 --- a/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/MetricsEvalNode.jsx @@ -1,59 +1,52 @@ -import React, { useState, useEffect } from "react"; +import React, { useEffect, useState } from "react"; import PropTypes from "prop-types"; import { Box, Button, Chip, DialogContent, - Typography, - TextField, - MenuItem, Grid, + MenuItem, + TextField, + Typography, } from "@mui/material"; import { getComponents as getComponentsRequest } from "../../../api/component"; import { validateNode } from "../../../api/pipeline"; import { useSnackbar } from "notistack"; -/** - * MetricsEvalNode – selects metrics to evaluate the trained model. - * - * Expects a TaskAndModel upstream node (via prevNodes) so it can filter - * compatible metrics by the selected task. - */ const MetricsEvalNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { const [metrics, setMetrics] = useState(savedConfig?.metrics || []); const [availableMetrics, setAvailableMetrics] = useState([]); const { enqueueSnackbar } = useSnackbar(); - // Locate the task from an upstream TaskAndModel (or Train) node const taskNode = prevNodes?.find((node) => node?.task); const task = taskNode?.task || ""; - // ---------- Fetch compatible metrics ---------- + useEffect(() => { + setMetrics(savedConfig?.metrics || []); + }, [savedConfig]); + useEffect(() => { const fetchMetrics = async () => { - if (!task) return; + if (!task) { + setAvailableMetrics([]); + return; + } + try { - const metricsResult = await getComponentsRequest({ + const metricsList = await getComponentsRequest({ selectTypes: ["Metric"], relatedComponent: task, }); - setAvailableMetrics(metricsResult); + setAvailableMetrics(metricsList); } catch (error) { console.error("Error fetching metrics:", error); } }; + fetchMetrics(); }, [task]); - // Restore saved selection - useEffect(() => { - if (savedConfig?.metrics) { - setMetrics(savedConfig.metrics); - } - }, [savedConfig]); - - // ---------- Save ---------- const handleSave = async () => { const payload = { metrics }; @@ -80,63 +73,59 @@ const MetricsEvalNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { maxWidth: "100%", }} > - + + + + Metric Evaluation + + - - - - Metric Evaluation - - - - setMetrics(e.target.value)} - margin="normal" - disabled={!task} - slotProps={{ - select: { - multiple: true, - MenuProps: { - PaperProps: { - sx: { - maxHeight: 420, - minWidth: 560, - }, + + setMetrics(e.target.value)} + margin="normal" + disabled={!task} + slotProps={{ + select: { + multiple: true, + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, }, }, - renderValue: (selected) => ( - - {selected.map((metricName) => ( - - ))} - - ), - }, - }} - sx={{ - minWidth: { xs: "100%", md: 560 }, - "& .MuiInputBase-root": { - minHeight: 58, - }, - "& .MuiSelect-select": { - minHeight: "56px", - display: "flex", - alignItems: "center", }, - }} - > - {availableMetrics.map((metric) => ( - - {metric.name} - - ))} - - - + renderValue: (selected) => ( + + {selected.map((metricName) => ( + + ))} + + ), + }, + }} + sx={{ + "& .MuiInputBase-root": { + minHeight: 58, + }, + "& .MuiSelect-select": { + minHeight: "56px", + display: "flex", + alignItems: "center", + }, + }} + > + {availableMetrics.map((metric) => ( + + {metric.name} + + ))} + + {!task && ( diff --git a/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx b/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx index efe69591d..1f219afbd 100644 --- a/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx @@ -58,7 +58,6 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { const { enqueueSnackbar } = useSnackbar(); const hasWarnedRef = useRef(false); - // ---------- Fetch dataset info ---------- const getDatasetInfo = async () => { setInfoLoading(true); try { @@ -91,7 +90,6 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { getDatasetInfo(); }, [datasetId]); - // ---------- Re-apply saved config when dataset info arrives ---------- useEffect(() => { const total = datasetInfo?.total_columns; setInputColumns( @@ -112,7 +110,6 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { ); }, [savedConfig, datasetInfo]); - // ---------- Validate column ranges on change ---------- useEffect(() => { const maxValue = datasetInfo?.total_columns; if (!maxValue) return; @@ -136,7 +133,6 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { } }, [inputColumns, outputColumns, datasetInfo]); - // ---------- Save ---------- const handleSave = async () => { const maxValue = datasetInfo?.total_columns; const columnNames = datasetInfo?.column_names || []; @@ -193,85 +189,86 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { return ( - - {/* ---- Column selection ---- */} + Column Selection - - setInputColumns(e.target.value)} - margin="normal" - error={inputError} - helperText={inputError ? inputErrorMessage : "e.g. 1-4 or 1,3,5"} - /> - - - - setOutputColumns(e.target.value)} - margin="normal" - error={outputError} - helperText={outputError ? outputErrorMessage : "e.g. 5 or 5-6"} - /> + + + setInputColumns(e.target.value)} + margin="normal" + error={inputError} + helperText={inputError ? inputErrorMessage : "e.g. 1-4 or 1,3,5"} + /> + setOutputColumns(e.target.value)} + margin="normal" + error={outputError} + helperText={outputError ? outputErrorMessage : "e.g. 5 or 5-6"} + /> + - {/* ---- Splits ---- */} Data Splits - - - setSplits({ ...splits, train: parseFloat(e.target.value) }) - } - margin="normal" - inputProps={{ step: 0.05, min: 0, max: 1 }} - /> - + + + + + setSplits({ ...splits, test: parseFloat(e.target.value) }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + - - - setSplits({ ...splits, validation: parseFloat(e.target.value) }) - } - margin="normal" - inputProps={{ step: 0.05, min: 0, max: 1 }} - /> - + + + setSplits({ ...splits, train: parseFloat(e.target.value) }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + - - - setSplits({ ...splits, test: parseFloat(e.target.value) }) - } - margin="normal" - inputProps={{ step: 0.05, min: 0, max: 1 }} - /> + + + setSplits({ ...splits, validation: parseFloat(e.target.value) }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + + {!splitsValid && ( @@ -283,28 +280,36 @@ const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { )} - - setSplits({ ...splits, shuffle: e.target.checked }) + + + + setSplits({ ...splits, shuffle: e.target.checked }) + } + /> } + label="Shuffle" + sx={{ m: 0 }} /> - } - label="Shuffle" - /> - - setSplits({ ...splits, stratify: e.target.checked }) + + + + setSplits({ ...splits, stratify: e.target.checked }) + } + /> } + label="Stratify" + sx={{ m: 0 }} /> - } - label="Stratify" - /> + + diff --git a/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx index bd9e0056f..c565e45b7 100644 --- a/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx @@ -50,7 +50,6 @@ const TaskAndModelNode = ({ const { enqueueSnackbar } = useSnackbar(); const hasWarnedRef = useRef(false); - // Try to locate the dataset info from a SplitData or DataSelector upstream const splitDataNode = prevNodes?.find( (node) => node?.input_columns || node?.splits, ); @@ -59,7 +58,6 @@ const TaskAndModelNode = ({ const inputColumnsRaw = splitDataNode?.input_columns; const outputColumnsRaw = splitDataNode?.output_columns; - // ---------- Fetch dataset info ---------- useEffect(() => { if (!datasetId) { setInfoLoading(false); @@ -86,7 +84,6 @@ const TaskAndModelNode = ({ fetchInfo(); }, [datasetId]); - // ---------- Fetch available tasks / models ---------- useEffect(() => { const fetchComponents = async () => { try { @@ -104,7 +101,6 @@ const TaskAndModelNode = ({ fetchComponents(); }, [task]); - // ---------- Model schema ---------- const { defaultValues, modelSchema, yupSchema, loading } = useSchema({ modelName: model, }); @@ -139,7 +135,6 @@ const TaskAndModelNode = ({ const handleChange = (newValues) => setModelParams(newValues); - // ---------- Validate available tasks against columns ---------- useEffect(() => { const validateAvailableTasks = async () => { if (!datasetId || !inputColumnsRaw || !outputColumnsRaw) { @@ -227,7 +222,6 @@ const TaskAndModelNode = ({ datasetInfo, ]); - // ---------- Save ---------- const handleSave = async () => { const payload = { task, @@ -255,15 +249,37 @@ const TaskAndModelNode = ({ <> - + - - Task and Model - + + + Task and Model + + + {validTasks.length === 0 && @@ -287,115 +303,116 @@ const TaskAndModelNode = ({ )} - - - - { - setTask(e.target.value); - setModel(""); - setModelParams({}); - }} - margin="normal" - disabled={validTasks.length === 0} - slotProps={{ - select: { - MenuProps: { - PaperProps: { - sx: { - maxHeight: 420, - minWidth: 560, - }, + + + { + setTask(e.target.value); + setModel(""); + setModelParams({}); + }} + margin="normal" + disabled={validTasks.length === 0} + slotProps={{ + select: { + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, }, }, }, - }} - sx={{ - minWidth: { xs: "100%", md: 560 }, - "& .MuiInputBase-root": { - minHeight: 58, - }, - "& .MuiSelect-select": { - whiteSpace: "normal", - minHeight: "1.4375em !important", - display: "flex", - alignItems: "center", - }, - }} - > - {availableTasks.map((taskObj) => ( - - {taskObj.name} - - ))} - - - - setOpenSettings(true)} - disabled={!model} - aria-label="model settings" - sx={{ mt: "8px", ml: { xs: 0, md: 0.5 } }} - > - - - - + }, + }} + sx={{ + width: "100%", + minWidth: 0, + display: "block", + "& .MuiInputBase-root": { + width: "100%", + minHeight: 58, + }, + "& .MuiOutlinedInput-root": { + width: "100%", + }, + "& .MuiInputBase-input": { + width: "100%", + }, + "& .MuiSelect-select": { + whiteSpace: "normal", + minHeight: "1.4375em !important", + display: "flex", + alignItems: "center", + }, + }} + > + {availableTasks.map((taskObj) => ( + + {taskObj.name} + + ))} + + - - - - setModel(e.target.value)} - margin="normal" - disabled={!task} - slotProps={{ - select: { - MenuProps: { - PaperProps: { - sx: { - maxHeight: 420, - minWidth: 560, - }, + + + setModel(e.target.value)} + margin="normal" + disabled={!task} + slotProps={{ + select: { + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, }, }, }, - }} - sx={{ - minWidth: { xs: "100%", md: 560 }, - "& .MuiInputBase-root": { - minHeight: 58, - }, - "& .MuiSelect-select": { - whiteSpace: "normal", - minHeight: "1.4375em !important", - display: "flex", - alignItems: "center", - }, - }} - > - {availableModels.map((m) => ( - - {m.name} - - ))} - - - - + }, + }} + sx={{ + width: "100%", + minWidth: 0, + display: "block", + "& .MuiInputBase-root": { + width: "100%", + minHeight: 58, + }, + "& .MuiOutlinedInput-root": { + width: "100%", + }, + "& .MuiInputBase-input": { + width: "100%", + }, + "& .MuiSelect-select": { + whiteSpace: "normal", + minHeight: "1.4375em !important", + display: "flex", + alignItems: "center", + }, + }} + > + {availableModels.map((m) => ( + + {m.name} + + ))} + + diff --git a/DashAI/front/src/hooks/usePipelineState.js b/DashAI/front/src/hooks/usePipelineState.js index 626a25e12..41c45d27e 100644 --- a/DashAI/front/src/hooks/usePipelineState.js +++ b/DashAI/front/src/hooks/usePipelineState.js @@ -1,5 +1,5 @@ import { useState, useEffect, useMemo } from "react"; -import { useEdgesState, useNodesState } from "reactflow"; +import { addEdge, useEdgesState, useNodesState } from "reactflow"; import { useSnackbar } from "notistack"; import { validatePipeline, @@ -8,7 +8,7 @@ import { getNodeTypesMap, buildNodeHelp, } from "../components/pipelines"; -import { getPipelineById, getPipelines } from "../api/pipeline"; +import { getPipelineById, getPipelines, validateEdge } from "../api/pipeline"; import { generateSequentialName } from "../utils/nameGenerator"; import RunPipeline from "../components/pipelines/Run"; @@ -56,8 +56,6 @@ export function usePipelineState(pipelineId, location, navigate) { const [nodeHelp, setNodeHelp] = useState({}); const [availableNodes, setAvailableNodes] = useState([]); const [nodeTypesMap, setNodeTypesMap] = useState([]); - - // Name validation states const [nameError, setNameError] = useState(false); const [nameErrorMessage, setNameErrorMessage] = useState(""); const [userHasModifiedName, setUserHasModifiedName] = useState(false); @@ -65,7 +63,6 @@ export function usePipelineState(pipelineId, location, navigate) { const [nodeIdCounter, setNodeIdCounter] = useState(0); const { enqueueSnackbar } = useSnackbar(); - // Generate default name for new pipelines const { defaultName } = useMemo(() => { if (pipelineId) return { defaultName: null }; @@ -77,7 +74,6 @@ export function usePipelineState(pipelineId, location, navigate) { return result; }, [existingPipelines, pipelineId]); - // Load existing pipelines for name generation useEffect(() => { const fetchPipelines = async () => { try { @@ -93,7 +89,6 @@ export function usePipelineState(pipelineId, location, navigate) { } }, [pipelineId]); - // Load node types and available nodes useEffect(() => { const fetchData = async () => { const nodes = normalizeNodeHandles(await getNodeTypes()); @@ -116,7 +111,6 @@ export function usePipelineState(pipelineId, location, navigate) { [nodeTypesMap], ); - // Set initial name validation and default value for new pipelines useEffect(() => { if (defaultName && !userHasModifiedName && !pipelineId) { if ( @@ -133,20 +127,16 @@ export function usePipelineState(pipelineId, location, navigate) { } }, [defaultName, pipelineName, userHasModifiedName, pipelineId]); - // Handle active tab changes useEffect(() => { if (location.state?.activeTab) { setActiveTab(location.state.activeTab); } }, [location.state?.activeTab]); - // Load existing pipeline useEffect(() => { if (pipelineId) { (async () => { const pipeline = await getPipelineById(pipelineId); - - // Get node types to assign source/target properties const nodeTypes = normalizeNodeHandles(await getNodeTypes()); const loadedNodes = pipeline.steps.map((step, idx) => { @@ -191,7 +181,6 @@ export function usePipelineState(pipelineId, location, navigate) { setPipelineName(pipeline.name); setUserHasModifiedName(false); - // Update nodeIdCounter const maxCounter = Math.max( ...loadedNodes.map((node) => { const match = node.id.match(/-(\d+)$/); @@ -204,7 +193,6 @@ export function usePipelineState(pipelineId, location, navigate) { } }, [pipelineId]); - // Reset editor when switching to new pipeline route useEffect(() => { if (!pipelineId) { setNodes([]); @@ -222,7 +210,6 @@ export function usePipelineState(pipelineId, location, navigate) { } }, [pipelineId]); - // Validate pipeline useEffect(() => { const validate = async () => { const errors = await validatePipeline(nodes, edges); @@ -231,7 +218,6 @@ export function usePipelineState(pipelineId, location, navigate) { validate(); }, [nodes.length, edges]); - // Update node data with validation info useEffect(() => { setNodes((prevNodes) => prevNodes.map((node) => { @@ -262,14 +248,12 @@ export function usePipelineState(pipelineId, location, navigate) { ); }, [validationErrors, nodeData, availableNodes]); - // Event handlers const onDragStart = (event, nodeType) => { setDragging(nodeType); event.dataTransfer.setData("text/plain", nodeType); event.dataTransfer.effectAllowed = "move"; }; - // Handle pipeline name input changes with validation const handlePipelineNameChange = (event) => { setPipelineName(event.target.value); setUserHasModifiedName(true); @@ -296,7 +280,6 @@ export function usePipelineState(pipelineId, location, navigate) { }; const handleRun = async () => { - // Check if name is valid if ( !pipelineName || pipelineName.trim() === "" || @@ -350,7 +333,101 @@ export function usePipelineState(pipelineId, location, navigate) { setNodeHelp(null); }; - // Clean up hovered node when nodes change + const handleEdgeRemove = (event, edge) => { + if (event?.stopPropagation) { + event.stopPropagation(); + } + setEdges((eds) => + eds.filter((e) => { + if (edge?.id && e.id === edge.id) { + return false; + } + return !( + e.source === edge.source && + e.target === edge.target && + e.sourceHandle === edge.sourceHandle && + e.targetHandle === edge.targetHandle + ); + }), + ); + }; + + const handleConnect = async (params) => { + const sourceNode = nodes.find((node) => node.id === params.source); + const targetNode = nodes.find((node) => node.id === params.target); + + if (!sourceNode || !targetNode) { + setEdges((eds) => + addEdge( + { + ...params, + markerEnd: { + type: "arrowclosed", + }, + }, + eds, + ), + ); + return; + } + + const currentOutputs = edges.filter( + (edge) => + edge.source === params.source && + (params.sourceHandle ? edge.sourceHandle === params.sourceHandle : true), + ).length; + const currentInputs = edges.filter( + (edge) => + edge.target === params.target && + (params.targetHandle ? edge.targetHandle === params.targetHandle : true), + ).length; + + let validation = null; + try { + validation = await validateEdge({ + source: { + nodeId: params.source, + type: sourceNode.type, + port: params.sourceHandle || null, + currentConnections: currentOutputs, + }, + target: { + nodeId: params.target, + type: targetNode.type, + port: params.targetHandle || null, + currentConnections: currentInputs, + }, + }); + } catch (error) { + console.error("Edge validation error:", error); + } + + const shouldWarn = validation && validation.ok === false; + const edgeColor = validation?.style?.edgeColor; + const edgeClass = validation?.style?.edgeClass; + + if (shouldWarn && validation?.message) { + enqueueSnackbar(validation.message, { + variant: validation.severity === "warning" ? "warning" : "error", + }); + } + + setEdges((eds) => + addEdge( + { + ...params, + className: shouldWarn ? edgeClass : undefined, + style: shouldWarn && edgeColor ? { stroke: edgeColor, strokeWidth: 2 } : undefined, + markerEnd: { + type: "arrowclosed", + color: shouldWarn && edgeColor ? edgeColor : undefined, + }, + }, + eds, + ), + ); + }; + useEffect(() => { if ( hoveredNode && @@ -406,6 +483,8 @@ export function usePipelineState(pipelineId, location, navigate) { onNodeMouseEnter, onNodeMouseLeave, onPaneClick, + handleConnect, + handleEdgeRemove, handlePipelineNameChange, }; } From 4a5209b23dddd38923259528f569b7dc0a0a5bfb Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 22 Jun 2026 16:23:13 -0400 Subject: [PATCH 54/88] feat: move node configuration into right-side panel --- DashAI/front/package.json | 1 + DashAI/front/src/App.jsx | 2 +- .../src/components/layout/ModuleContainer.jsx | 1 + .../pipelines/PipelineHistorySidebar.jsx | 12 +- .../pipelines/PipelineNodeConfigSidebar.jsx | 71 ++++++++++++ .../front/src/components/pipelines/index.jsx | 1 + .../pipelines/nodes/ConfigurableNode.jsx | 73 +++++++----- .../pipelines/nodes/DataExplorationNode.jsx | 36 ++++-- .../pipelines/nodes/ExplorationModal.jsx | 37 ++++++- .../threeSectionLayout/panels/RightPanel.jsx | 62 +++++++---- .../front/src/hooks/useThreePanelsLayout.js | 24 +++- .../front/src/pages/pipelines/NewPipeline.jsx | 104 +++++++----------- DashAI/front/yarn.lock | 64 ++++++++++- 13 files changed, 342 insertions(+), 146 deletions(-) create mode 100644 DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx diff --git a/DashAI/front/package.json b/DashAI/front/package.json index 3faa32436..cc0f80d2b 100644 --- a/DashAI/front/package.json +++ b/DashAI/front/package.json @@ -10,6 +10,7 @@ "@mui/material": "^7", "@mui/system": "^7", "@mui/utils": "^7", + "@mui/x-data-grid": "^7", "@mui/x-date-pickers": "^8.27.2", "@tanstack/react-table": "^8.21.3", "axios": "^1.3.4", diff --git a/DashAI/front/src/App.jsx b/DashAI/front/src/App.jsx index f13a84910..41b5db4e7 100644 --- a/DashAI/front/src/App.jsx +++ b/DashAI/front/src/App.jsx @@ -47,7 +47,7 @@ function App() { element={} /> } /> - } /> + } /> } /> diff --git a/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx index 1cbbd2c4e..09582f34c 100644 --- a/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx @@ -14,7 +14,6 @@ import { Visibility as ViewIcon, } from "@mui/icons-material"; import HistoryIcon from "@mui/icons-material/History"; -import { ChevronLeft } from "@mui/icons-material"; import { useTheme } from "@mui/material/styles"; import { DataGrid, GridActionsCellItem, GridToolbar } from "@mui/x-data-grid"; import { Menu, MenuItem } from "@mui/material"; @@ -26,7 +25,7 @@ import { deletePipeline, getPipelines } from "../../api/pipeline"; import { formatDate } from "../../utils"; import DeleteItemModal from "../custom/DeleteItemModal"; -function PipelineHistorySidebar({ currentPipelineId, onToggle }) { +function PipelineHistorySidebar({ currentPipelineId }) { const theme = useTheme(); const navigate = useNavigate(); const { enqueueSnackbar } = useSnackbar(); @@ -132,15 +131,6 @@ function PipelineHistorySidebar({ currentPipelineId, onToggle }) { Pipelines History - {onToggle && ( - - - - )} diff --git a/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx b/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx new file mode 100644 index 000000000..9ecbe1180 --- /dev/null +++ b/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx @@ -0,0 +1,71 @@ +import React from "react"; +import { Box, Divider, IconButton, Typography } from "@mui/material"; +import SettingsIcon from "@mui/icons-material/Settings"; +import CloseIcon from "@mui/icons-material/Close"; +import { useTheme } from "@mui/material/styles"; +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; + +function PipelineNodeConfigSidebar({ selectedNode, onClose, children }) { + const theme = useTheme(); + const nodeTitle = + selectedNode?.data?.name || selectedNode?.data?.label || selectedNode?.type; + + return ( + + + + + + {nodeTitle ? "Node Configuration" : "Pipeline Node Configuration"} + + + + {selectedNode && ( + + + + )} + + + + + + + {selectedNode ? ( + children || ( + + This node does not require configuration. + + ) + ) : ( + + Select a node in the canvas to configure it. + + )} + + + ); +} + +export default PipelineNodeConfigSidebar; diff --git a/DashAI/front/src/components/pipelines/index.jsx b/DashAI/front/src/components/pipelines/index.jsx index 87a7c982f..f6daed8e3 100644 --- a/DashAI/front/src/components/pipelines/index.jsx +++ b/DashAI/front/src/components/pipelines/index.jsx @@ -6,6 +6,7 @@ export { default as NodeSidebar } from "./NodeSidebar"; export { default as PipelineHistorySidebar } from "./PipelineHistorySidebar"; export { default as PipelineTemplatesSidebar } from "./PipelineTemplatesSidebar"; export { default as PipelineChatSidebar } from "./PipelineChatSidebar"; +export { default as PipelineNodeConfigSidebar } from "./PipelineNodeConfigSidebar"; export { default as PipelinesTable } from "./PipelinesTable"; export { default as CustomNode } from "./CustomNode"; export { default as Run } from "./Run"; diff --git a/DashAI/front/src/components/pipelines/nodes/ConfigurableNode.jsx b/DashAI/front/src/components/pipelines/nodes/ConfigurableNode.jsx index aa6e88d2b..5bcaee80f 100644 --- a/DashAI/front/src/components/pipelines/nodes/ConfigurableNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/ConfigurableNode.jsx @@ -16,6 +16,7 @@ function ConfigurableNode({ onSave, savedConfig = {}, configSchema, + displayMode = "dialog", }) { const [formValues, setFormValues] = useState({}); @@ -41,37 +42,55 @@ function ConfigurableNode({ onClose(); }; + const fields = (configSchema?.fields || []).map((field) => ( + + {field.type === "select" ? ( + + {field.options.map((opt) => ( + + {opt} + + ))} + + ) : ( + + )} + + )); + + if (displayMode === "panel") { + if (!open) return null; + + return ( + + {fields} + + + + + + ); + } + return ( Configure Node - {(configSchema?.fields || []).map((field) => ( - - {field.type === "select" ? ( - - {field.options.map((opt) => ( - - {opt} - - ))} - - ) : ( - - )} - - ))} + {fields} diff --git a/DashAI/front/src/components/pipelines/nodes/DataExplorationNode.jsx b/DashAI/front/src/components/pipelines/nodes/DataExplorationNode.jsx index 8fbdf83fc..1dcbdbf36 100644 --- a/DashAI/front/src/components/pipelines/nodes/DataExplorationNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/DataExplorationNode.jsx @@ -1,5 +1,6 @@ import React, { useEffect, useRef } from "react"; import PropTypes from "prop-types"; +import { Box, Typography } from "@mui/material"; import { useSnackbar } from "notistack"; import { ExplorationsProvider } from "../../../components/explorations/context"; import ConfigureExplorersModal from "./ExplorationModal"; @@ -10,6 +11,7 @@ const DataExplorationNode = ({ onSave, savedConfig = null, prevNodes = [], + displayMode = "dialog", }) => { const datasetNode = prevNodes?.find((node) => node?.file_path && node?.id); const datasetId = datasetNode?.id ?? null; @@ -35,19 +37,28 @@ const DataExplorationNode = ({ onClose(); }; + if (!open) return null; + + if (!datasetId) { + return ( + + + Missing dataset. Connect a DataSelector node first. + + + ); + } + return ( - <> - {open && datasetId && ( - - - - )} - + + + ); }; @@ -57,6 +68,7 @@ DataExplorationNode.propTypes = { onSave: PropTypes.func.isRequired, savedConfig: PropTypes.object, prevNodes: PropTypes.arrayOf(PropTypes.object), + displayMode: PropTypes.string, }; export default DataExplorationNode; diff --git a/DashAI/front/src/components/pipelines/nodes/ExplorationModal.jsx b/DashAI/front/src/components/pipelines/nodes/ExplorationModal.jsx index d06511cef..e1afe5ea0 100644 --- a/DashAI/front/src/components/pipelines/nodes/ExplorationModal.jsx +++ b/DashAI/front/src/components/pipelines/nodes/ExplorationModal.jsx @@ -10,6 +10,7 @@ import { useMediaQuery, useTheme, Box, + Typography, } from "@mui/material"; import CloseIcon from "@mui/icons-material/Close"; import ConfigureExplorersStep from "../../../components/explorations/Steps/ConfigureExplorersStep"; @@ -17,7 +18,13 @@ import { useExplorationsContext } from "../../../components/explorations/context import { validateNode } from "../../../api/pipeline"; import { useSnackbar } from "notistack"; -function ConfigureExplorersModal({ open, onClose, onSave, savedConfig }) { +function ConfigureExplorersModal({ + open, + onClose, + onSave, + savedConfig, + displayMode = "dialog", +}) { const theme = useTheme(); const fullScreen = useMediaQuery(theme.breakpoints.down("sm")); const [valid, setValid] = useState(false); @@ -83,10 +90,36 @@ function ConfigureExplorersModal({ open, onClose, onSave, savedConfig }) { } }; - if (loading && open) { + if (!open) { return null; } + if (loading) { + return null; + } + + if (displayMode === "panel") { + return ( + + Configure Explorers + + + + + + + + + ); + } + return ( {rightBarVisible && ( <> - + {!isRightOverlayActive && ( + + )} {children} )} diff --git a/DashAI/front/src/hooks/useThreePanelsLayout.js b/DashAI/front/src/hooks/useThreePanelsLayout.js index 273aad5d6..7de93be03 100644 --- a/DashAI/front/src/hooks/useThreePanelsLayout.js +++ b/DashAI/front/src/hooks/useThreePanelsLayout.js @@ -1,4 +1,6 @@ import { useState, useRef, useEffect, useCallback } from "react"; +import { useTheme } from "@mui/material/styles"; +import useMediaQuery from "@mui/material/useMediaQuery"; const DEFAULT_LAYOUT = { leftBarVisible: true, @@ -19,7 +21,11 @@ const readPersisted = (storageKey) => { } }; -export function useThreePanelLayout({ storageKey } = {}) { +export function useThreePanelLayout({ + storageKey, + overlayRightOnCompact = false, + compactBreakpoint = "md", +} = {}) { const initial = readPersisted(storageKey); const [leftBarVisible, setLeftBarVisible] = useState(initial.leftBarVisible); const [rightBarVisible, setRightBarVisible] = useState( @@ -108,13 +114,19 @@ export function useThreePanelLayout({ storageKey } = {}) { }, 300); }, []); + const theme = useTheme(); + const isCompact = useMediaQuery(theme.breakpoints.down(compactBreakpoint)); + const isRightOverlayActive = overlayRightOnCompact && isCompact; + const effectiveRightVisible = isRightOverlayActive ? false : rightBarVisible; + const effectiveRightWidth = isRightOverlayActive ? 0 : rightBarWidth; + const centerWidth = - leftBarVisible && rightBarVisible - ? 100 - leftBarWidth - rightBarWidth + leftBarVisible && effectiveRightVisible + ? 100 - leftBarWidth - effectiveRightWidth : leftBarVisible ? 100 - leftBarWidth - : rightBarVisible - ? 100 - rightBarWidth + : effectiveRightVisible + ? 100 - effectiveRightWidth : 100; return { @@ -123,6 +135,8 @@ export function useThreePanelLayout({ storageKey } = {}) { leftBarWidth, rightBarWidth, centerWidth, + isCompact, + isRightOverlayActive, handleToggleLeft, handleToggleRight, diff --git a/DashAI/front/src/pages/pipelines/NewPipeline.jsx b/DashAI/front/src/pages/pipelines/NewPipeline.jsx index 2bbab60c5..1b4918f62 100644 --- a/DashAI/front/src/pages/pipelines/NewPipeline.jsx +++ b/DashAI/front/src/pages/pipelines/NewPipeline.jsx @@ -8,12 +8,8 @@ import { Chip, Grid, Typography, - Dialog, - DialogTitle, - IconButton, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; -import CloseIcon from "@mui/icons-material/Close"; import { ReactFlowProvider } from "reactflow"; import CustomLayout from "../../components/custom/CustomLayout"; import ModuleContainer from "../../components/layout/ModuleContainer"; @@ -33,7 +29,7 @@ import { PipelineDesigner, NodeSidebar, PipelineHistorySidebar, - PipelineChatSidebar, + PipelineNodeConfigSidebar, nodeRegistry, } from "../../components/pipelines"; @@ -71,6 +67,8 @@ function NewPipeline() { const threePanelLayout = useThreePanelLayout({ initialLeftWidth: 20, initialRightWidth: 20, + overlayRightOnCompact: true, + compactBreakpoint: "md", }); const { @@ -118,6 +116,8 @@ function NewPipeline() { onNodeMouseEnter, onNodeMouseLeave, onPaneClick, + handleConnect, + handleEdgeRemove, handlePipelineNameChange, } = usePipelineState(pipelineId, location, navigate); @@ -214,7 +214,22 @@ function NewPipeline() { } }, [isWorkspaceOpen]); - const renderNodeDialogContent = () => { + const handleNodeConfigOpen = (event, node) => { + if (!threePanelLayout.rightBarVisible) { + threePanelLayout.handleToggleRight(); + } + onNodeClick(event, node); + }; + + const renderNodeConfigContent = () => { + if (activeTab !== "flow") { + return ( + + Switch to the Flow tab to configure nodes. + + ); + } + if (!selectedNode) return null; const { type, id, data } = selectedNode; @@ -227,11 +242,18 @@ function NewPipeline() { NodeComponent = nodeRegistry["Configurable"]; } - if (!NodeComponent) return null; + if (!NodeComponent) { + return ( + + This node does not require configuration. + + ); + } return ( handleSaveNodeData(id, data)} savedConfig={nodeData[id]} @@ -249,7 +271,6 @@ function NewPipeline() { @@ -431,8 +452,10 @@ function NewPipeline() { setEdges={setEdges} onNodesChange={onNodesChange} onEdgesChange={onEdgesChange} + onConnect={handleConnect} + onEdgeDoubleClick={handleEdgeRemove} nodeTypes={nodeTypes} - onNodeClick={onNodeClick} + onNodeClick={handleNodeConfigOpen} onNodeHelp={onNodeHelp} onNodeMouseEnter={onNodeMouseEnter} onNodeMouseLeave={onNodeMouseLeave} @@ -464,64 +487,13 @@ function NewPipeline() { - - - - {isWorkspaceOpen && renderNodeDialogContent() && ( - {}} - disableEscapeKeyDown - maxWidth={ - selectedNode?.type === "TaskAndModel" || - selectedNode?.type === "MetricsEval" - ? "lg" - : "md" - } - fullWidth - PaperProps={{ - sx: - selectedNode?.type === "TaskAndModel" || - selectedNode?.type === "MetricsEval" - ? { - width: { xs: "calc(100% - 24px)", md: 920 }, - maxWidth: { xs: "calc(100% - 24px)", md: 920 }, - } - : undefined, - }} + - - - - {selectedNode?.type === "Train" - ? "Select Train Parameters" - : selectedNode?.type === "Exploration" - ? "Exploration Configuration" - : selectedNode?.type === "SplitData" - ? "Configure Data Splits" - : selectedNode?.type === "TaskAndModel" - ? "Select Task & Model" - : selectedNode?.type === "MetricsEval" - ? "Select Metrics" - : `Configure ${selectedNode?.type || "Node"}`} - - - - - - - {renderNodeDialogContent()} - - )} + {renderNodeConfigContent()} + + diff --git a/DashAI/front/yarn.lock b/DashAI/front/yarn.lock index 05aa10fb0..2b2f731fd 100644 --- a/DashAI/front/yarn.lock +++ b/DashAI/front/yarn.lock @@ -1542,7 +1542,7 @@ __metadata: languageName: node linkType: hard -"@babel/runtime@npm:^7.28.6": +"@babel/runtime@npm:^7.25.7, @babel/runtime@npm:^7.28.6": version: 7.29.2 resolution: "@babel/runtime@npm:7.29.2" checksum: d5548d1165de8995f8afc93a5694b8625409be16cd1f2250ac13e331335858ddac3cb9fd278e6c43956a130101a2203f09417938a1a96f9fb70f02b4b4172e1d @@ -3072,6 +3072,26 @@ __metadata: languageName: node linkType: hard +"@mui/utils@npm:^5.16.6 || ^6.0.0 || ^7.0.0": + version: 7.3.11 + resolution: "@mui/utils@npm:7.3.11" + dependencies: + "@babel/runtime": ^7.28.6 + "@mui/types": ^7.4.12 + "@types/prop-types": ^15.7.15 + clsx: ^2.1.1 + prop-types: ^15.8.1 + react-is: ^19.2.3 + peerDependencies: + "@types/react": ^17.0.0 || ^18.0.0 || ^19.0.0 + react: ^17.0.0 || ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + "@types/react": + optional: true + checksum: 24c4e7a6d9c0c0c40e414a8a39a9f0be389456dc27d80393c9bc135231193487947cec239d11ecd79c72ff653f37c900b0843ec1e5eb7484d580d65f647b9281 + languageName: node + linkType: hard + "@mui/utils@npm:^7, @mui/utils@npm:^7.3.1, @mui/utils@npm:^7.3.3": version: 7.3.3 resolution: "@mui/utils@npm:7.3.3" @@ -3112,6 +3132,33 @@ __metadata: languageName: node linkType: hard +"@mui/x-data-grid@npm:^7": + version: 7.29.13 + resolution: "@mui/x-data-grid@npm:7.29.13" + dependencies: + "@babel/runtime": ^7.25.7 + "@mui/utils": ^5.16.6 || ^6.0.0 || ^7.0.0 + "@mui/x-internals": 7.29.0 + clsx: ^2.1.1 + prop-types: ^15.8.1 + reselect: ^5.1.1 + use-sync-external-store: ^1.0.0 + peerDependencies: + "@emotion/react": ^11.9.0 + "@emotion/styled": ^11.8.1 + "@mui/material": ^5.15.14 || ^6.0.0 || ^7.0.0 + "@mui/system": ^5.15.14 || ^6.0.0 || ^7.0.0 + react: ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^17.0.0 || ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + "@emotion/react": + optional: true + "@emotion/styled": + optional: true + checksum: 0fbd07c4e3ecd6e297d119d8cb5908bd724a56553636ee7bdd5910b666eb350f6a70f9dbed006416147616b62920018caf7f137b7155085737a5361640ece429 + languageName: node + linkType: hard + "@mui/x-date-pickers@npm:^8.27.2": version: 8.27.2 resolution: "@mui/x-date-pickers@npm:8.27.2" @@ -3160,6 +3207,18 @@ __metadata: languageName: node linkType: hard +"@mui/x-internals@npm:7.29.0": + version: 7.29.0 + resolution: "@mui/x-internals@npm:7.29.0" + dependencies: + "@babel/runtime": ^7.25.7 + "@mui/utils": ^5.16.6 || ^6.0.0 || ^7.0.0 + peerDependencies: + react: ^17.0.0 || ^18.0.0 || ^19.0.0 + checksum: 3e246226bddc03bafe2be7128ea22f9c4a44a5e94bc44f67e630e1c2288f601be3514dec034ab1f2b3bef6937715671cbcc65818338b218f50b89b34d9948290 + languageName: node + linkType: hard + "@mui/x-internals@npm:8.26.0": version: 8.26.0 resolution: "@mui/x-internals@npm:8.26.0" @@ -7962,6 +8021,7 @@ __metadata: "@mui/material": ^7 "@mui/system": ^7 "@mui/utils": ^7 + "@mui/x-data-grid": ^7 "@mui/x-date-pickers": ^8.27.2 "@stylistic/eslint-plugin-js": ^1.7.2 "@tanstack/react-table": ^8.21.3 @@ -19590,7 +19650,7 @@ __metadata: languageName: node linkType: hard -"use-sync-external-store@npm:^1.2.2, use-sync-external-store@npm:^1.4.0, use-sync-external-store@npm:^1.6.0": +"use-sync-external-store@npm:^1.0.0, use-sync-external-store@npm:^1.2.2, use-sync-external-store@npm:^1.4.0, use-sync-external-store@npm:^1.6.0": version: 1.6.0 resolution: "use-sync-external-store@npm:1.6.0" peerDependencies: From 38b7387cb20e596e28c666e534c64b967b725f7b Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 22 Jun 2026 16:29:29 -0400 Subject: [PATCH 55/88] fix: correct dataset column validation for explorers --- .../Steps/ConfigureExplorersStep.jsx | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx index 30bd12304..e2fd1a50d 100644 --- a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx +++ b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx @@ -89,9 +89,10 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { */ const validateOptions = useCallback((data) => { const options = data.map((explorer, index) => { - const allowedDtypes = explorer.metadata.allowed_dtypes; - const restrictedDtypes = explorer.metadata.restricted_dtypes; - const inputCardinality = explorer.metadata.input_cardinality; + const metadata = explorer.metadata || {}; + const allowedDtypes = metadata.allowed_dtypes || []; + const restrictedDtypes = metadata.restricted_dtypes || []; + const inputCardinality = metadata.input_cardinality || {}; let disabled = false; let tooltip = ""; @@ -100,29 +101,31 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { tooltip += `\n`; } - let validColumns = datasetColumns; - // check the valid dataset columns for the explorer - if (!allowedDtypes.includes("*")) { - validColumns = datasetColumns.filter((col) => + let validColumns = datasetColumns || []; + const hasDtypeRestriction = + allowedDtypes.length > 0 && !allowedDtypes.includes("*"); + if (hasDtypeRestriction) { + validColumns = validColumns.filter((col) => allowedDtypes.includes(col.dataType), ); } // check the restricted dataset columns for the explorer - if ( - restrictedDtypes.some((dtype) => - datasetColumns.some((col) => col.dataType === dtype), - ) - ) { - validColumns = validColumns.filter( - (col) => !restrictedDtypes.includes(col.dataType), + if (restrictedDtypes.length > 0) { + const hasRestricted = restrictedDtypes.some((dtype) => + (datasetColumns || []).some((col) => col.dataType === dtype), ); + if (hasRestricted) { + validColumns = validColumns.filter( + (col) => !restrictedDtypes.includes(col.dataType), + ); + } } // check the input cardinality if ( - inputCardinality.exact != undefined && - inputCardinality.exact != null + inputCardinality.exact !== undefined && + inputCardinality.exact !== null ) { // if (tooltip) tooltip += "\n"; // tooltip += `This explorer requires exactly ${ @@ -130,7 +133,7 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { // } valid ${inputCardinality.exact === 1 ? "column" : "columns"}.`; if (validColumns.length < inputCardinality.exact) disabled = true; } else { - if (inputCardinality.min != undefined && inputCardinality.min != null) { + if (inputCardinality.min !== undefined && inputCardinality.min !== null) { // if (tooltip) tooltip += "\n"; // tooltip += `This explorer requires at least ${ // inputCardinality.min @@ -139,7 +142,7 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { if (validColumns.length < inputCardinality.min) disabled = true; } - if (inputCardinality.max != undefined && inputCardinality.max != null) { + if (inputCardinality.max !== undefined && inputCardinality.max !== null) { // if (tooltip) tooltip += "\n"; // tooltip += `This explorer requires at most ${ // inputCardinality.max @@ -308,10 +311,9 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { - {/* Form to add a single explorer to the exploration */} - - + + {} }) { /> - + {} }) { /> - + @@ -357,7 +360,6 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { - {/* Explorers table */} {loading && ( Date: Mon, 22 Jun 2026 17:05:42 -0400 Subject: [PATCH 56/88] style: resolve pre-commit ruff and prettier violations --- .../d9c8f2a1b7e4_add_pipeline_run_tracking.py | 2 +- .../api/api_v1/schemas/pipelines_params.py | 4 +- DashAI/back/job/node_job.py | 22 +++---- DashAI/back/job/pipeline_job.py | 19 +++--- DashAI/back/pipeline/contracts.py | 59 +++++++++++-------- .../pipeline/validator/nodes_definitions.py | 10 ++-- .../Steps/ConfigureExplorersStep.jsx | 10 +++- .../pipelines/PipelineChatSidebar.jsx | 5 +- .../pipelines/PipelineHistorySidebar.jsx | 15 +++-- .../pipelines/PipelineNodeConfigSidebar.jsx | 5 +- .../pipelines/PipelineTemplatesSidebar.jsx | 15 ++++- .../pipelines/nodes/ConfigurableNode.jsx | 4 +- .../pipelines/nodes/MetricsEvalNode.jsx | 6 +- .../pipelines/nodes/SplitDataNode.jsx | 5 +- .../components/pipelines/results/Results.jsx | 26 ++++++-- DashAI/front/src/hooks/usePipelineState.js | 13 +++- .../front/src/pages/pipelines/NewPipeline.jsx | 56 ++++++++++++++---- DashAI/front/src/types/pipeline.ts | 44 ++++++++------ 18 files changed, 208 insertions(+), 112 deletions(-) diff --git a/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py b/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py index 76a19a81f..d99ac3ca0 100644 --- a/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py +++ b/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py @@ -105,4 +105,4 @@ def downgrade() -> None: op.drop_table("pipeline_node_run") if _table_exists("pipeline_run"): - op.drop_table("pipeline_run") \ No newline at end of file + op.drop_table("pipeline_run") diff --git a/DashAI/back/api/api_v1/schemas/pipelines_params.py b/DashAI/back/api/api_v1/schemas/pipelines_params.py index bca8c2be9..fdeadd9c5 100644 --- a/DashAI/back/api/api_v1/schemas/pipelines_params.py +++ b/DashAI/back/api/api_v1/schemas/pipelines_params.py @@ -38,10 +38,10 @@ class ValidatePipelineParams(BaseModel): class EdgeEndpointRef(BaseModel): - nodeId: Optional[str] = None + nodeId: Optional[str] = None # noqa: N815 type: str port: Optional[str] = None - currentConnections: Optional[int] = None + currentConnections: Optional[int] = None # noqa: N815 class ValidateEdgeParams(BaseModel): diff --git a/DashAI/back/job/node_job.py b/DashAI/back/job/node_job.py index 9ec0da131..ed1159da6 100644 --- a/DashAI/back/job/node_job.py +++ b/DashAI/back/job/node_job.py @@ -5,7 +5,7 @@ import time from typing import TYPE_CHECKING, Any, Dict -from kink import di, inject +from kink import inject from sqlalchemy import exc from DashAI.back.dependencies.database.models import NodeRun @@ -43,7 +43,9 @@ def set_status_as_delivered( self._commit_with_retry(db) except exc.SQLAlchemyError as e: log.exception(e) - raise JobError("Error while setting node run status as delivered.") from e + raise JobError( + "Error while setting node run status as delivered." + ) from e @inject def set_status_as_error( @@ -96,9 +98,7 @@ def run( try: node_instance = node_class(**node_config) except Exception as e: - raise JobError( - f"Error instantiating node {node_type}: {str(e)}" - ) from e + raise JobError(f"Error instantiating node {node_type}: {str(e)}") from e return self._run_node_instance(node_instance, context) @@ -123,9 +123,7 @@ def run( try: node_instance = node_class(**node_config) except Exception as e: - raise JobError( - f"Error instantiating node {node_type}: {str(e)}" - ) from e + raise JobError(f"Error instantiating node {node_type}: {str(e)}") from e try: output = self._run_node_instance(node_instance, context) @@ -159,12 +157,16 @@ def run( self._commit_with_retry(db) except exc.SQLAlchemyError as e: log.exception(e) - raise JobError("Error while setting node run status as finished.") from e + raise JobError( + "Error while setting node run status as finished." + ) from e return output @staticmethod - def _run_node_instance(node_instance: Any, context: Dict[str, Any]) -> Dict[str, Any]: + def _run_node_instance( + node_instance: Any, context: Dict[str, Any] + ) -> Dict[str, Any]: result = node_instance.run(context=context) if asyncio.iscoroutine(result): return asyncio.run(result) diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index 0cc0e840f..2503333df 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Any, Dict, List from kink import di, inject - from sqlalchemy import exc from DashAI.back.dependencies.database.models import NodeRun, Pipeline, PipelineRun @@ -95,9 +94,7 @@ async def _execute_graph() -> None: raise JobError("Pipeline steps are missing ids.") ready = [ - node_id - for node_id in pending - if not predecessor_map.get(node_id) + node_id for node_id in pending if not predecessor_map.get(node_id) ] if not ready: raise JobError( @@ -136,9 +133,7 @@ async def _run_node(node_id: str) -> None: node_run_id = node_run.id except exc.OperationalError as e: if "database is locked" in str(e).lower(): - log.warning( - "NodeRun insert skipped due to database lock." - ) + log.warning("NodeRun insert skipped due to database lock.") node_run_id = None else: raise @@ -182,7 +177,7 @@ async def _schedule(node_id: str) -> None: try: task.result() except Exception as e: - for pending_task in task_map.keys(): + for pending_task in task_map: pending_task.cancel() raise JobError( f"Error executing node {node_id}: {str(e)}" @@ -207,9 +202,7 @@ async def _schedule(node_id: str) -> None: db.add(pipeline_run) NodeJob._commit_with_retry(db, readd=[pipeline_run, pipeline]) self.set_status_as_delivered() - log.info( - f"Pipeline {pipeline_id} execution completed successfully." - ) + log.info(f"Pipeline {pipeline_id} execution completed successfully.") except Exception as e: pipeline_run.set_status_as_error(str(e)) db.add(pipeline_run) @@ -312,7 +305,9 @@ def _update_context( if isinstance(pipeline.exploration, dict) else {} ) - exploration_key = node_id or f"DataExploration-{len(aggregated_exploration)}" + exploration_key = ( + node_id or f"DataExploration-{len(aggregated_exploration)}" + ) aggregated_exploration[exploration_key] = { "dataset_name": context.get("dataset_name") or output.get("dataset_name") diff --git a/DashAI/back/pipeline/contracts.py b/DashAI/back/pipeline/contracts.py index b0a143fc9..df69d63a5 100644 --- a/DashAI/back/pipeline/contracts.py +++ b/DashAI/back/pipeline/contracts.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Dict, List, Literal, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, List, Literal, Optional from pydantic import BaseModel @@ -17,6 +17,7 @@ "Any", ] + class PortSpec(BaseModel): name: str kind: DataKind @@ -31,17 +32,17 @@ class NodeContract(BaseModel): name: str icon: str description: str - requiresConfiguration: bool + requiresConfiguration: bool # noqa: N815 source: bool target: bool predecessors: List[str] successors: List[str] input: Optional[str] = None output: Optional[str] = None - sourceHandles: int = 1 - maxInputs: Optional[int] = 1 - maxOutputs: Optional[int] = None - configType: str = "custom" + sourceHandles: int = 1 # noqa: N815 + maxInputs: Optional[int] = 1 # noqa: N815 + maxOutputs: Optional[int] = None # noqa: N815 + configType: str = "custom" # noqa: N815 inputs: List[PortSpec] outputs: List[PortSpec] @@ -292,7 +293,9 @@ def get_contracts_payload() -> List[Dict[str, object]]: return payload -def _resolve_port(ports: List[PortSpec], port_name: Optional[str], role: str) -> PortSpec: +def _resolve_port( + ports: List[PortSpec], port_name: Optional[str], role: str +) -> PortSpec: if port_name: for port in ports: if port.name == port_name: @@ -358,25 +361,31 @@ def validate_edge( except ValueError as exc: return _error_response(str(exc), reason="INVALID_PORT") - if target_port_spec.max_connections is not None and target_current_inputs is not None: - if target_current_inputs >= target_port_spec.max_connections: - return _error_response( - f"{target_contract.name} does not allow more connections on this input.", - reason="MAX_INPUTS_EXCEEDED", - severity="warning", - edge_color="#F57C00", - edge_class="edge-warning", - ) + if ( + target_port_spec.max_connections is not None + and target_current_inputs is not None + and target_current_inputs >= target_port_spec.max_connections + ): + return _error_response( + f"{target_contract.name} does not allow more connections on this input.", + reason="MAX_INPUTS_EXCEEDED", + severity="warning", + edge_color="#F57C00", + edge_class="edge-warning", + ) - if source_current_outputs is not None and source_port_spec.max_connections is not None: - if source_current_outputs >= source_port_spec.max_connections: - return _error_response( - f"{source_contract.name} does not allow more outputs on this port.", - reason="MAX_OUTPUTS_EXCEEDED", - severity="warning", - edge_color="#F57C00", - edge_class="edge-warning", - ) + if ( + source_current_outputs is not None + and source_port_spec.max_connections is not None + and source_current_outputs >= source_port_spec.max_connections + ): + return _error_response( + f"{source_contract.name} does not allow more outputs on this port.", + reason="MAX_OUTPUTS_EXCEEDED", + severity="warning", + edge_color="#F57C00", + edge_class="edge-warning", + ) accepts = target_port_spec.accepts or [target_port_spec.kind] if source_port_spec.kind not in accepts: diff --git a/DashAI/back/pipeline/validator/nodes_definitions.py b/DashAI/back/pipeline/validator/nodes_definitions.py index a46931846..d64eff6d0 100644 --- a/DashAI/back/pipeline/validator/nodes_definitions.py +++ b/DashAI/back/pipeline/validator/nodes_definitions.py @@ -9,7 +9,7 @@ class NodeDefinition(BaseModel): type: str name: str icon: str - requiresConfiguration: bool + requiresConfiguration: bool # noqa: N815 source: bool target: bool predecessors: List[str] @@ -17,10 +17,10 @@ class NodeDefinition(BaseModel): description: str input: Optional[str] = None output: Optional[str] = None - sourceHandles: int = 1 - maxInputs: Optional[int] = 1 - maxOutputs: Optional[int] = None - configType: str + sourceHandles: int = 1 # noqa: N815 + maxInputs: Optional[int] = 1 # noqa: N815 + maxOutputs: Optional[int] = None # noqa: N815 + configType: str # noqa: N815 NODES: List[NodeDefinition] = contracts_to_node_definitions() diff --git a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx index e2fd1a50d..d4df0b972 100644 --- a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx +++ b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx @@ -133,7 +133,10 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { // } valid ${inputCardinality.exact === 1 ? "column" : "columns"}.`; if (validColumns.length < inputCardinality.exact) disabled = true; } else { - if (inputCardinality.min !== undefined && inputCardinality.min !== null) { + if ( + inputCardinality.min !== undefined && + inputCardinality.min !== null + ) { // if (tooltip) tooltip += "\n"; // tooltip += `This explorer requires at least ${ // inputCardinality.min @@ -142,7 +145,10 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { if (validColumns.length < inputCardinality.min) disabled = true; } - if (inputCardinality.max !== undefined && inputCardinality.max !== null) { + if ( + inputCardinality.max !== undefined && + inputCardinality.max !== null + ) { // if (tooltip) tooltip += "\n"; // tooltip += `This explorer requires at most ${ // inputCardinality.max diff --git a/DashAI/front/src/components/pipelines/PipelineChatSidebar.jsx b/DashAI/front/src/components/pipelines/PipelineChatSidebar.jsx index a223e001d..e12affff3 100644 --- a/DashAI/front/src/components/pipelines/PipelineChatSidebar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineChatSidebar.jsx @@ -28,7 +28,10 @@ function PipelineChatSidebar({ onToggle }) { }} > - + Pipeline Chatbot {onToggle && ( diff --git a/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx index 09582f34c..43f92b848 100644 --- a/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx @@ -128,7 +128,10 @@ function PipelineHistorySidebar({ currentPipelineId }) { }} > - + Pipelines History @@ -153,7 +156,9 @@ function PipelineHistorySidebar({ currentPipelineId }) { - + - Created: {rowMenuPipeline ? formatDate(rowMenuPipeline.created) : "-"} + Created:{" "} + {rowMenuPipeline ? formatDate(rowMenuPipeline.created) : "-"} - Edited: {rowMenuPipeline ? formatDate(rowMenuPipeline.last_modified) : "-"} + Edited:{" "} + {rowMenuPipeline ? formatDate(rowMenuPipeline.last_modified) : "-"} diff --git a/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx b/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx index 9ecbe1180..214117e5d 100644 --- a/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx @@ -22,7 +22,10 @@ function PipelineNodeConfigSidebar({ selectedNode, onClose, children }) { }} > - + {nodeTitle ? "Node Configuration" : "Pipeline Node Configuration"} diff --git a/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx b/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx index 2323588a1..7322fe055 100644 --- a/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx @@ -14,7 +14,11 @@ import { ChevronRight } from "@mui/icons-material"; import { useTheme } from "@mui/material/styles"; import SideBar from "../threeSectionLayout/panelContainers/SideBar"; -function PipelineTemplatesSidebar({ onToggle, templates = [], onSelectTemplate }) { +function PipelineTemplatesSidebar({ + onToggle, + templates = [], + onSelectTemplate, +}) { const theme = useTheme(); return ( @@ -29,7 +33,10 @@ function PipelineTemplatesSidebar({ onToggle, templates = [], onSelectTemplate } }} > - + Prefabricated Pipelines {onToggle && ( @@ -43,7 +50,9 @@ function PipelineTemplatesSidebar({ onToggle, templates = [], onSelectTemplate } )} - + {templates.map((template) => ( Configure Node - - {fields} - + {fields} Date: Fri, 26 Jun 2026 17:33:26 -0400 Subject: [PATCH 62/88] style: add missing newline at end of status.py --- DashAI/back/core/enums/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DashAI/back/core/enums/status.py b/DashAI/back/core/enums/status.py index e752314cf..d87f1152c 100644 --- a/DashAI/back/core/enums/status.py +++ b/DashAI/back/core/enums/status.py @@ -74,4 +74,4 @@ class NodeRunStatus(Enum): DELIVERED = 1 STARTED = 2 FINISHED = 3 - ERROR = 4 \ No newline at end of file + ERROR = 4 From 34c121427648377153890a0a185eda8641f3d53c Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Thu, 2 Jul 2026 15:06:49 -0400 Subject: [PATCH 63/88] fix: repair dataset upload flow after merge (dataloader params and defaults) --- DashAI/back/api/api_v1/endpoints/datasets.py | 5 +++++ DashAI/back/job/dataset_job.py | 11 ++++++++++- .../src/components/datasets/DatasetModal.jsx | 19 ++++++++++--------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/datasets.py b/DashAI/back/api/api_v1/endpoints/datasets.py index b17651607..8aa53d0a8 100644 --- a/DashAI/back/api/api_v1/endpoints/datasets.py +++ b/DashAI/back/api/api_v1/endpoints/datasets.py @@ -1902,6 +1902,11 @@ async def preview_with_types( detail=f"Dataloader {dataloader_name} not found in registry.", ) + if dataloader_name == "CSVDataLoader" and "separator" not in parsed_params: + parsed_params["separator"] = "," + if dataloader_name == "JSONDataLoader" and "data_key" not in parsed_params: + parsed_params["data_key"] = None + dataloader_cls = component_registry[dataloader_name]["class"] dataloader = dataloader_cls() diff --git a/DashAI/back/job/dataset_job.py b/DashAI/back/job/dataset_job.py index 6d9fc925e..a621c25c1 100644 --- a/DashAI/back/job/dataset_job.py +++ b/DashAI/back/job/dataset_job.py @@ -234,7 +234,16 @@ def run( n_sample=None, ) else: - # --- File / URL upload path (unchanged) --- + if ( + params.get("dataloader") == "CSVDataLoader" + and "separator" not in params + ): + params["separator"] = "," + if ( + params.get("dataloader") == "JSONDataLoader" + and "data_key" not in params + ): + params["data_key"] = None parsed_params = parse_params(DatasetParams, json.dumps(params)) dataloader = component_registry[parsed_params.dataloader][ "class" diff --git a/DashAI/front/src/components/datasets/DatasetModal.jsx b/DashAI/front/src/components/datasets/DatasetModal.jsx index 46cb1ec49..922b43b2f 100644 --- a/DashAI/front/src/components/datasets/DatasetModal.jsx +++ b/DashAI/front/src/components/datasets/DatasetModal.jsx @@ -22,7 +22,7 @@ import ConfigureAndUploadDataset from "./ConfigureAndUploadDataset"; import { useSnackbar } from "notistack"; import { enqueueDatasetJob as enqueueDatasetRequest } from "../../api/job"; import DatasetPreviewStep from "./DatasetPreviewStep"; -import { previewWithTypes } from "../../api/datasets"; +import { previewWithTypes, createDataset } from "../../api/datasets"; const SKIP_PREVIEW_DATALOADERS = new Set([]); @@ -64,18 +64,13 @@ function DatasetModal({ open, setOpen, updateDatasets }) { const handleSubmitNewDataset = async () => { try { - const name = - newDataset.params.name === null - ? newDataset.file.name - : newDataset.params.name; + const name = newDataset.params.name || newDataset.file.name; newDataset.params["dataloader"] = newDataset.dataloader; - // First, create the dataset record in the database const data = await createDataset(name); - - // Then, enqueue the job with the dataset ID await enqueueDatasetRequest(data.id, newDataset.file, newDataset.url, { ...newDataset.params, + name: name, inferred_types: columnsSpec, }); @@ -94,7 +89,13 @@ function DatasetModal({ open, setOpen, updateDatasets }) { const handlePreviewDataset = async (file, url) => { const formData = new FormData(); formData.append("file", newDataset.file); - formData.append("params", JSON.stringify(newDataset.params)); + formData.append( + "params", + JSON.stringify({ + ...newDataset.params, + dataloader_name: newDataset.dataloader, + }), + ); try { const preview = await previewWithTypes(formData); From e3a680f09ad988f5e8ba55e52f14e3e4d5b87feb Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Thu, 2 Jul 2026 15:07:07 -0400 Subject: [PATCH 64/88] fix: serialize plotly explorer figures containing bytes from text columns --- DashAI/back/exploration/base_explorer.py | 17 ++++++++++ .../exploration/explorers/multibox_plot.py | 32 ++++++++++++++++--- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/DashAI/back/exploration/base_explorer.py b/DashAI/back/exploration/base_explorer.py index a77dadef7..e2f126cb2 100644 --- a/DashAI/back/exploration/base_explorer.py +++ b/DashAI/back/exploration/base_explorer.py @@ -12,6 +12,23 @@ from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset +try: + from _plotly_utils.utils import PlotlyJSONEncoder as _PlotlyJSONEncoder + + if not getattr(_PlotlyJSONEncoder, "_dashai_bytes_patch", False): + _plotly_orig_default = _PlotlyJSONEncoder.default + + def _plotly_default_with_bytes(self, obj): + if isinstance(obj, (bytes, bytearray)): + return obj.decode("utf-8", "replace") + return _plotly_orig_default(self, obj) + + _PlotlyJSONEncoder.default = _plotly_default_with_bytes + _PlotlyJSONEncoder._dashai_bytes_patch = True +except Exception: + pass + + class BaseExplorerSchema(BaseSchema): """ Base schema for explorers, it defines the parameters to be used in each explorer. diff --git a/DashAI/back/exploration/explorers/multibox_plot.py b/DashAI/back/exploration/explorers/multibox_plot.py index 60febc426..204e609ea 100644 --- a/DashAI/back/exploration/explorers/multibox_plot.py +++ b/DashAI/back/exploration/explorers/multibox_plot.py @@ -264,16 +264,25 @@ def launch_exploration(self, dataset: "DashAIDataset", explorer_info: Explorer): _df = dataset.to_pandas() cols = [col["columnName"] for col in explorer_info.columns] + def _decode(values): + return [ + v.decode("utf-8", "replace") if isinstance(v, (bytes, bytearray)) else v + for v in values + ] + opposite_axis = ( - _df[self.opposite_axis] if self.opposite_axis is not None else None + _decode(_df[self.opposite_axis].tolist()) + if self.opposite_axis is not None + else None ) fig = go.Figure() for col in cols: + col_values = _decode(_df[col].tolist()) fig.add_trace( go.Box( - x=_df[col] if self.horizontal else opposite_axis, - y=opposite_axis if self.horizontal else _df[col], + x=col_values if self.horizontal else opposite_axis, + y=opposite_axis if self.horizontal else col_values, name=col, boxpoints=self.points, ) @@ -322,13 +331,28 @@ def save_notebook( str The path of the saved file as a POSIX string. """ + import json import os from pathlib import Path + import numpy as np + filename = f"{explorer_info.id}.pickle" path = Path(os.path.join(save_path, filename)) - result.write_json(path.as_posix()) + def _default(o): + if isinstance(o, (bytes, bytearray)): + return o.decode("utf-8", "replace") + if isinstance(o, np.ndarray): + return o.tolist() + if isinstance(o, np.generic): + return o.item() + raise TypeError( + f"Object of type {type(o).__name__} is not JSON serializable" + ) + + with open(path.as_posix(), "w", encoding="utf-8") as f: + json.dump(result.to_dict(), f, default=_default) return path.as_posix() From 948956a93b7491b4404a6466466d626191290653 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Thu, 2 Jul 2026 15:07:38 -0400 Subject: [PATCH 65/88] feat: support TaskAndModel-trained models in Retrieve Model node --- DashAI/back/api/api_v1/endpoints/pipelines.py | 2 +- DashAI/back/pipeline/contracts.py | 4 +- DashAI/back/pipeline/retrieve_model_node.py | 11 ++++-- .../pipelines/nodes/RetrieveModelNode.jsx | 38 +++++++++++++++---- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/pipelines.py b/DashAI/back/api/api_v1/endpoints/pipelines.py index 73ae30d8c..70c6c28a2 100644 --- a/DashAI/back/api/api_v1/endpoints/pipelines.py +++ b/DashAI/back/api/api_v1/endpoints/pipelines.py @@ -723,7 +723,7 @@ async def filter_models_endpoint( if params.pipeline_id and pipeline.id == params.pipeline_id: continue - if not pipeline.train: + if not (pipeline.train or pipeline.task_and_model): continue steps = pipeline.steps diff --git a/DashAI/back/pipeline/contracts.py b/DashAI/back/pipeline/contracts.py index df69d63a5..ef2c5b58a 100644 --- a/DashAI/back/pipeline/contracts.py +++ b/DashAI/back/pipeline/contracts.py @@ -57,7 +57,7 @@ class NodeContract(BaseModel): source=True, target=False, predecessors=[], - successors=["DataExploration", "Train", "SplitData"], + successors=["DataExploration", "Train", "SplitData", "RetrieveModel"], output="Dataset object", sourceHandles=2, maxOutputs=2, @@ -300,6 +300,8 @@ def _resolve_port( for port in ports: if port.name == port_name: return port + if len(ports) == 1: + return ports[0] raise ValueError(f"Unknown {role} port: {port_name}") if len(ports) == 1: return ports[0] diff --git a/DashAI/back/pipeline/retrieve_model_node.py b/DashAI/back/pipeline/retrieve_model_node.py index a49f31945..f1a93fb76 100644 --- a/DashAI/back/pipeline/retrieve_model_node.py +++ b/DashAI/back/pipeline/retrieve_model_node.py @@ -61,9 +61,14 @@ async def run( context["task_name"] = self.task dataset = context["dataset"] - input_columns_names = get_column_names_from_indexes( - dataset, self.input_columns - ) + if self.input_columns and all( + isinstance(col, str) for col in self.input_columns + ): + input_columns_names = self.input_columns + else: + input_columns_names = get_column_names_from_indexes( + dataset, self.input_columns + ) context["input_columns"] = input_columns_names model_class = component_registry[self.model]["class"] diff --git a/DashAI/front/src/components/pipelines/nodes/RetrieveModelNode.jsx b/DashAI/front/src/components/pipelines/nodes/RetrieveModelNode.jsx index 6092e68cb..794d0f081 100644 --- a/DashAI/front/src/components/pipelines/nodes/RetrieveModelNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/RetrieveModelNode.jsx @@ -11,6 +11,24 @@ import { filterModels } from "../../../api/pipeline"; import { useSnackbar } from "notistack"; import { validateNode } from "../../../api/pipeline"; import { useParams } from "react-router-dom"; + +const getModelEntry = (pipeline) => { + const pick = (obj) => { + if (!obj || typeof obj !== "object") return null; + if (obj.model_path) return obj; // flat (legacy) structure + const values = Object.values(obj); // aggregated: keyed by node id + return values.find((v) => v && v.model_path) || null; + }; + const entry = pick(pipeline?.train) || pick(pipeline?.task_and_model); + if (!entry) return null; + return { + model: entry.info || entry.model || "", + model_path: entry.model_path || "", + input_columns: entry.input_columns || [], + task: entry.task || "", + }; +}; + function RetrieveModelNode({ onClose, onSave, @@ -31,7 +49,7 @@ function RetrieveModelNode({ useEffect(() => { if (savedConfig?.model_path && pipelines.length > 0) { const matching = pipelines.find( - (p) => p.train?.model_path === savedConfig.model_path, + (p) => getModelEntry(p)?.model_path === savedConfig.model_path, ); if (matching) { setSelectedModelId(matching.id); @@ -67,12 +85,18 @@ function RetrieveModelNode({ const selected = pipelines.find((m) => m.id === selectedModelId); if (!selected) return; - const train = selected.train || {}; + const entry = getModelEntry(selected); + if (!entry) { + enqueueSnackbar("Selected pipeline has no trained model", { + variant: "warning", + }); + return; + } const nodeData = { - model: train.info || "", - model_path: train.model_path || "", - input_columns: train.input_columns || [], - task: train.task || "", + model: entry.model, + model_path: entry.model_path, + input_columns: entry.input_columns, + task: entry.task, }; try { @@ -97,7 +121,7 @@ function RetrieveModelNode({ id: "model", header: "Model", grow: true, - accessorFn: (row) => row.train?.info || "Unknown", + accessorFn: (row) => getModelEntry(row)?.model || "Unknown", }, ], [], From 658b57ddaff5663a9b597d6b96beb1080d27be16 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Thu, 2 Jul 2026 15:08:12 -0400 Subject: [PATCH 66/88] fix: repair pipeline UI regressions after merge (theming, canvas, results, default name) --- .../src/components/pipelines/CustomNode.jsx | 5 +- .../components/pipelines/PipelineDesigner.jsx | 12 +- .../components/pipelines/PipelineToolbar.jsx | 19 +- .../pipelines/results/ResultsGraphs.jsx | 195 ++++-------------- DashAI/front/src/hooks/usePipelineState.js | 12 +- 5 files changed, 63 insertions(+), 180 deletions(-) diff --git a/DashAI/front/src/components/pipelines/CustomNode.jsx b/DashAI/front/src/components/pipelines/CustomNode.jsx index ec52f1531..387ea2b07 100644 --- a/DashAI/front/src/components/pipelines/CustomNode.jsx +++ b/DashAI/front/src/components/pipelines/CustomNode.jsx @@ -191,10 +191,7 @@ const CustomNode = ({ data, isConnectable }) => { alignItems: "center", }} > - + {data.name || data.label} diff --git a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx index 0dab4a3fc..d5dfa0f20 100644 --- a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx +++ b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx @@ -1,6 +1,6 @@ import React, { useCallback, useEffect } from "react"; import { Box, Tooltip } from "@mui/material"; -import { useTheme } from "@mui/material/styles"; +import { useTheme, alpha } from "@mui/material/styles"; import ReactFlow, { addEdge, Background, @@ -43,7 +43,9 @@ function PipelineDesigner({ const canvasBorder = isLightCanvas ? "1px solid #e0e0e0" : `1px solid ${theme.palette.ui.borderLight}`; - const gridColor = isLightCanvas ? "#e0e0e0" : theme.palette.ui.border; + const gridColor = isLightCanvas + ? alpha("#9e9e9e", 0.7) + : alpha(theme.palette.text.secondary, 0.6); // Sync canvasMode into existing nodes so CustomNode can read it useEffect(() => { @@ -169,8 +171,8 @@ function PipelineDesigner({ width: "100%", height: "100%", borderRadius: 12, - background: theme.palette.background.default, - border: `1px solid ${theme.palette.ui.border}`, + background: canvasBg, + border: canvasBorder, boxShadow: "0 2px 6px rgba(0,0,0,0.1)", }} > @@ -199,7 +201,7 @@ function PipelineDesigner({ )} - + ); diff --git a/DashAI/front/src/components/pipelines/PipelineToolbar.jsx b/DashAI/front/src/components/pipelines/PipelineToolbar.jsx index 63333ca0b..c8813f479 100644 --- a/DashAI/front/src/components/pipelines/PipelineToolbar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineToolbar.jsx @@ -26,7 +26,11 @@ function PipelineToolbar({ display: "flex", justifyContent: "space-between", alignItems: "center", - mb: 4, + mb: 2, + p: 1.5, + borderRadius: 2, + backgroundColor: theme.palette.background.box, + border: `1px solid ${theme.palette.ui.borderLight}`, }} > - { - setSelectedChart(chartType); - }; - - const handleToggleParameter = (parameter) => { - const updatedParameters = selectedParameters.includes(parameter) - ? selectedParameters.filter((param) => param !== parameter) - : [...selectedParameters, parameter]; - setSelectedParameters(updatedParameters); - }; - - const handleToggleMetrics = () => { - setShowCustomMetrics(!showCustomMetrics); - setSelectedParameters([]); - }; - - useEffect(() => { - const processData = () => { - try { - const phases = Object.keys(metrics); - const metricNames = Object.keys(metrics[phases[0]]); - const tabularMetrics = []; - - phases.forEach((phase) => { - metricNames.forEach((metric) => { - tabularMetrics.push(`${phase} ${metric}`); - }); - }); - - const concatenatedMetrics = Array.from( - new Set(phases.concat(metricNames)), - ); - setConcatenatedMetrics(concatenatedMetrics); - setTabularMetrics(tabularMetrics); - - let parameterIndex = []; - let generalParameters = []; - - if (showCustomMetrics) { - parameterIndex = selectedParameters.map((param) => - tabularMetrics.indexOf(param), - ); - } else { - const criteria = {}; - concatenatedMetrics.forEach((item) => { - criteria[item] = item; - }); - - tabularMetrics.forEach((metric, index) => { - Object.entries(criteria).forEach(([metricName, substring]) => { - if ( - selectedGeneralMetric === metricName && - metric.includes(substring) - ) { - parameterIndex.push(index); - generalParameters.push(metric); - } - }); - }); - } - - const syntheticRun = { - status: 3, - train_metrics: metrics.train, - validation_metrics: metrics.validation, - test_metrics: metrics.test, - }; - - const newFilteredData = [syntheticRun]; - setFilteredDataProcess(newFilteredData); - - const graphsToView = {}; - let pieCounter = 0; - - const numericValues = []; - ["train", "validation", "test"].forEach((split) => { - metricNames.forEach((metric) => { - numericValues.push(metrics[split][metric]); - }); - }); - - const relevantNumericValues = parameterIndex.map( - (index) => numericValues[index], - ); - - graphsMaking( - graphsToView, - syntheticRun, - relevantNumericValues, - showCustomMetrics, - selectedParameters, - generalParameters, - pieCounter, - ); - - const { generalLayout, pieLayout } = layoutMaking( - selectedChart, - graphsToView, - ); - - const graphsToViewKeys = Object.keys(graphsToView); - const radarValues = graphsToView[graphsToViewKeys[0]]; - const barValues = graphsToView[graphsToViewKeys[1]]; - const pieValues = graphsToView[graphsToViewKeys[2]]; - - setChartData({ - generalLayout, - pieLayout, - radarValues, - barValues, - pieValues, - }); - } catch (err) { - enqueueSnackbar("Error processing metrics", { variant: "error" }); - console.error(err); - } - }; + const trainMetrics = normalizeSplitMetrics(metrics?.train); + const validationMetrics = normalizeSplitMetrics(metrics?.validation); + const testMetrics = normalizeSplitMetrics(metrics?.test); + + const hasAnyMetrics = + Object.keys(trainMetrics).length > 0 || + Object.keys(validationMetrics).length > 0 || + Object.keys(testMetrics).length > 0; + + const syntheticRuns = useMemo( + () => [ + { + status: 3, + train_metrics: trainMetrics, + validation_metrics: validationMetrics, + test_metrics: testMetrics, + }, + ], + [trainMetrics, validationMetrics, testMetrics], + ); - processData(); - }, [metrics, selectedParameters, selectedChart]); + if (!hasAnyMetrics) { + return ( + + No metric data available + Please provide valid metrics to render the graphs. + + ); + } - return ( - <> - {filteredDataProcess.length === 0 ? ( - - No metric data available - Please provide valid metrics to render the graphs. - - ) : ( - - )} - - ); + return ; } +PipelineResultsGraphs.propTypes = { + metrics: PropTypes.shape({ + train: PropTypes.object, + validation: PropTypes.object, + test: PropTypes.object, + }), +}; + +PipelineResultsGraphs.defaultProps = { + metrics: {}, +}; + export default PipelineResultsGraphs; diff --git a/DashAI/front/src/hooks/usePipelineState.js b/DashAI/front/src/hooks/usePipelineState.js index 6d97de050..e221806b5 100644 --- a/DashAI/front/src/hooks/usePipelineState.js +++ b/DashAI/front/src/hooks/usePipelineState.js @@ -113,14 +113,9 @@ export function usePipelineState(pipelineId, location, navigate) { useEffect(() => { if (defaultName && !userHasModifiedName && !pipelineId) { - if ( - pipelineName === "undefined" || - pipelineName.startsWith("Pipeline_") - ) { - setPipelineName(defaultName); - setNameError(false); - setNameErrorMessage(""); - } + setPipelineName(defaultName); + setNameError(false); + setNameErrorMessage(""); } else if (pipelineName === "" && userHasModifiedName) { setNameError(true); setNameErrorMessage("Name is required"); @@ -205,7 +200,6 @@ export function usePipelineState(pipelineId, location, navigate) { setNodeHelp({}); setNodeIdCounter(0); setUserHasModifiedName(false); - setPipelineName("undefined"); setActiveTab("flow"); } }, [pipelineId]); From 6645ad30cdca883afe8c5976d049224b7e48f4b1 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Thu, 2 Jul 2026 15:08:22 -0400 Subject: [PATCH 67/88] style: improve Data Exploration node config layout --- .../Steps/ConfigureExplorersStep.jsx | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx index 13781d44f..42f980a7a 100644 --- a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx +++ b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx @@ -318,19 +318,8 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { - - - - setExplorerData({ ...explorerData, name: e.target.value }) - } - fullWidth - /> - - - + + {} }) { /> + + + setExplorerData({ ...explorerData, name: e.target.value }) + } + fullWidth + /> + + diff --git a/DashAI/front/src/hooks/usePipelineState.js b/DashAI/front/src/hooks/usePipelineState.js index e221806b5..e1411ef5a 100644 --- a/DashAI/front/src/hooks/usePipelineState.js +++ b/DashAI/front/src/hooks/usePipelineState.js @@ -1,4 +1,4 @@ -import { useState, useEffect, useMemo } from "react"; +import { useState, useEffect, useMemo, useRef, useCallback } from "react"; import { addEdge, useEdgesState, useNodesState } from "reactflow"; import { useSnackbar } from "notistack"; import { @@ -8,7 +8,12 @@ import { getNodeTypesMap, buildNodeHelp, } from "../components/pipelines"; -import { getPipelineById, getPipelines, validateEdge } from "../api/pipeline"; +import { + getPipelineById, + getPipelines, + validateEdge, + getLatestPipelineRun, +} from "../api/pipeline"; import { generateSequentialName } from "../utils/nameGenerator"; import RunPipeline from "../components/pipelines/Run"; @@ -61,8 +66,110 @@ export function usePipelineState(pipelineId, location, navigate) { const [userHasModifiedName, setUserHasModifiedName] = useState(false); const [existingPipelines, setExistingPipelines] = useState([]); const [nodeIdCounter, setNodeIdCounter] = useState(0); + const [isRunning, setIsRunning] = useState(false); + const pollRef = useRef(null); + const finishTimerRef = useRef(null); const { enqueueSnackbar } = useSnackbar(); + const stopPolling = useCallback(() => { + if (pollRef.current) { + clearInterval(pollRef.current); + pollRef.current = null; + } + if (finishTimerRef.current) { + clearTimeout(finishTimerRef.current); + finishTimerRef.current = null; + } + }, []); + + const applyNodeStatuses = useCallback( + (nodeRuns) => { + const statusById = {}; + (nodeRuns || []).forEach((nr) => { + if (nr?.node_id) statusById[nr.node_id] = nr.status; + }); + setNodes((nds) => + nds.map((n) => ({ + ...n, + data: { ...n.data, status: statusById[n.id] ?? "NOT_STARTED" }, + })), + ); + }, + [setNodes], + ); + + const resetNodeStatuses = useCallback(() => { + setNodes((nds) => + nds.map((n) => ({ ...n, data: { ...n.data, status: "NOT_STARTED" } })), + ); + }, [setNodes]); + + const markAllNodesFinished = useCallback(() => { + setNodes((nds) => + nds.map((n) => ({ ...n, data: { ...n.data, status: "FINISHED" } })), + ); + }, [setNodes]); + + const startRunPolling = useCallback( + (pid, sinceRunId) => { + stopPolling(); + setIsRunning(true); + const poll = async () => { + try { + const run = await getLatestPipelineRun(pid); + if (!run || run.id == null) return; + if (sinceRunId != null && run.id <= sinceRunId) return; + applyNodeStatuses(run.node_runs); + if (run.status === "FINISHED") { + stopPolling(); + setIsRunning(false); + markAllNodesFinished(); + enqueueSnackbar("Pipeline execution finished", { + variant: "success", + }); + finishTimerRef.current = setTimeout(() => { + setActiveTab("results"); + }, 1000); + } else if (run.status === "ERROR") { + stopPolling(); + setIsRunning(false); + enqueueSnackbar(run.error_message || "Pipeline execution failed", { + variant: "error", + }); + } + } catch (e) { + // transient error + } + }; + poll(); + pollRef.current = setInterval(poll, 800); + }, + [ + applyNodeStatuses, + markAllNodesFinished, + enqueueSnackbar, + stopPolling, + setActiveTab, + ], + ); + + useEffect(() => stopPolling, [stopPolling]); + + useEffect(() => { + const since = location.state?.runningSinceRunId; + if (since === undefined || since === null) return; + if (!pipelineId || nodes.length === 0) return; + startRunPolling(Number(pipelineId), since); + navigate(location.pathname, { replace: true, state: {} }); + }, [ + pipelineId, + location.state, + location.pathname, + nodes.length, + startRunPolling, + navigate, + ]); + const { defaultName } = useMemo(() => { if (pipelineId) return { defaultName: null }; @@ -292,6 +399,20 @@ export function usePipelineState(pipelineId, location, navigate) { enqueueSnackbar("Error in pipeline", { variant: "error" }); return; } + + stopPolling(); + resetNodeStatuses(); + + let sinceRunId = 0; + if (pipelineId) { + try { + const prevRun = await getLatestPipelineRun(Number(pipelineId)); + sinceRunId = prevRun?.id ?? 0; + } catch (e) { + sinceRunId = 0; + } + } + const newId = await RunPipeline( sortedNodes, nodeData, @@ -302,8 +423,14 @@ export function usePipelineState(pipelineId, location, navigate) { ); if (newId) { setResultId(newId); - setActiveTab("results"); - navigate(`/app/pipelines/${newId}`); + setActiveTab("flow"); + if (String(newId) !== String(pipelineId)) { + navigate(`/app/pipelines/${newId}`, { + state: { runningSinceRunId: sinceRunId }, + }); + } else { + startRunPolling(Number(newId), sinceRunId); + } } }; @@ -457,6 +584,7 @@ export function usePipelineState(pipelineId, location, navigate) { nodeIdCounter, nameError, nameErrorMessage, + isRunning, // Setters setNodes, diff --git a/DashAI/front/src/pages/pipelines/NewPipeline.jsx b/DashAI/front/src/pages/pipelines/NewPipeline.jsx index 7d1b6afd1..fbe7e9f37 100644 --- a/DashAI/front/src/pages/pipelines/NewPipeline.jsx +++ b/DashAI/front/src/pages/pipelines/NewPipeline.jsx @@ -102,6 +102,7 @@ function NewPipeline() { nodeIdCounter, nameError, nameErrorMessage, + isRunning, // Setters setNodes, @@ -474,6 +475,7 @@ function NewPipeline() { handlePipelineNameChange={handlePipelineNameChange} canvasMode={canvasMode} setCanvasMode={setCanvasMode} + isRunning={isRunning} /> Date: Wed, 8 Jul 2026 17:01:30 -0400 Subject: [PATCH 73/88] feat: add Pipelines to the navigation bar --- DashAI/front/src/components/ResponsiveAppBar.jsx | 1 + 1 file changed, 1 insertion(+) diff --git a/DashAI/front/src/components/ResponsiveAppBar.jsx b/DashAI/front/src/components/ResponsiveAppBar.jsx index 47a16d296..b809d5808 100644 --- a/DashAI/front/src/components/ResponsiveAppBar.jsx +++ b/DashAI/front/src/components/ResponsiveAppBar.jsx @@ -30,6 +30,7 @@ function ResponsiveAppBar() { const pages = [ { name: t("common:datasets"), to: "/app/data" }, { name: t("common:models"), to: "/app/models" }, + { name: t("common:pipelines"), to: "/app/pipelines" }, { name: t("common:generative"), to: "/app/generative" }, { name: t("common:plugins"), to: "/app/plugins/browse" }, ]; From 079ccd62e272352d59d5f2174359fb56d860904e Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Wed, 8 Jul 2026 17:26:03 -0400 Subject: [PATCH 74/88] style: apply pinned ruff and prettier formatting --- DashAI/back/job/pipeline_job.py | 8 ++------ DashAI/front/src/components/pipelines/CustomNode.jsx | 3 +-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index 8929654f8..04683fca5 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -201,9 +201,7 @@ async def _schedule(node_id: str) -> None: NodeJob._commit_with_retry(db, readd=[pipeline_run]) raise - def _persist_node_run( - self, db, pipeline_run_id, node_id, node_type, node_config - ): + def _persist_node_run(self, db, pipeline_run_id, node_id, node_type, node_config): """Create a NodeRun (STARTED) on the orchestrator's session. Returns the NodeRun, or None if it could not be persisted (the node @@ -232,9 +230,7 @@ def _set_node_run_status(self, db, node_run, status, error_message=None): if status == "finished": node_run.set_status_as_finished() elif status == "error": - node_run.set_status_as_error( - error_message or "Node execution failed." - ) + node_run.set_status_as_error(error_message or "Node execution failed.") else: node_run.set_status_as_started() NodeJob._commit_with_retry(db, readd=[node_run]) diff --git a/DashAI/front/src/components/pipelines/CustomNode.jsx b/DashAI/front/src/components/pipelines/CustomNode.jsx index 2a294bcf7..4530e0d64 100644 --- a/DashAI/front/src/components/pipelines/CustomNode.jsx +++ b/DashAI/front/src/components/pipelines/CustomNode.jsx @@ -40,8 +40,7 @@ const CustomNode = ({ data, isConnectable }) => { ERROR: theme.palette.status.error, }; const runColor = runStatusColors[data.status]; - const isRunning = - data.status === "STARTED" || data.status === "DELIVERED"; + const isRunning = data.status === "STARTED" || data.status === "DELIVERED"; const statusBg = data.status === "FINISHED" ? alpha(theme.palette.status.finished, isLightCanvas ? 0.14 : 0.2) From b378c9582d6f4393517ee2483cbf67176219db70 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 10 Jul 2026 18:47:51 -0400 Subject: [PATCH 75/88] feat: show outputs of all nodes in the pipeline results view Add Data Selector, Split Data and Retrieve Model sections to the pipeline results, alongside the existing Exploration/Train/Prediction: - Data Selector: dataset name, row/column counts, task and a column preview (reuses DatasetSummaryTable / the dataset sample endpoint). - Split Data: input/output columns and per-partition ratios and sizes. - Retrieve Model: model, task, input columns and model path. Partial results now render progressively with a "running" indicator. All data is derived from what getPipelineById already returns (steps config and split_data); no backend or schema changes required. Co-Authored-By: Claude Opus 4.8 --- .../components/pipelines/results/Results.jsx | 148 +++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 9e66b57de..1e793bbce 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -18,6 +18,7 @@ import PipelineResultsGraphs from "./ResultsGraphs"; import PipelineResultsPrediction from "./ResultsPrediction"; import ResultsTabParameters from "../../../pages/results/components/ResultsTabParameters"; import ResultsExploration from "./ResultsExploration"; +import DatasetSummaryTable from "../../datasets/DatasetSummaryTable"; const POLL_INTERVAL_MS = 3000; @@ -206,7 +207,30 @@ function PipelineResults({ pipelineId, onClose }) { const hasPrediction = branchResults.filter((branch) => !!branch.prediction).length > 0; - if (!hasExploration && !hasTrain && !hasPrediction) { + const steps = Array.isArray(results.steps) ? results.steps : []; + const dataSelectorSteps = steps.filter((s) => s.type === "DataSelector"); + const splitDataSteps = steps.filter((s) => s.type === "SplitData"); + const retrieveModelSteps = steps.filter((s) => s.type === "RetrieveModel"); + const totalRows = dataSelectorSteps[0]?.config?.total_rows ?? null; + + const hasDataSelector = dataSelectorSteps.length > 0; + const hasSplitData = splitDataSteps.length > 0; + const hasRetrieveModel = retrieveModelSteps.length > 0; + + const splitSize = (ratio) => + totalRows != null && ratio != null ? Math.round(ratio * totalRows) : null; + + const formatColumns = (cols) => + Array.isArray(cols) && cols.length > 0 ? cols.join(", ") : "-"; + + if ( + !hasExploration && + !hasTrain && + !hasPrediction && + !hasDataSelector && + !hasSplitData && + !hasRetrieveModel + ) { return ( {onClose && ( @@ -230,6 +254,128 @@ function PipelineResults({ pipelineId, onClose }) { return ( + {polling && ( + + + + Pipeline is running. Remaining results will appear automatically… + + + )} + + {hasDataSelector && ( + + }> + Data Selector + + + + {dataSelectorSteps.map((step) => { + const cfg = step.config || {}; + return ( + + + Dataset: {cfg.name || "-"} + + + {cfg.total_rows ?? "?"} rows × {cfg.total_columns ?? "?"}{" "} + columns + {cfg.task ? ` | ${cfg.task}` : ""} + + {cfg.id != null && ( + + )} + + ); + })} + + + + )} + + {hasSplitData && ( + + }> + Split Data + + + + {splitDataSteps.map((step) => { + const cfg = step.config || {}; + const splits = cfg.splits || {}; + return ( + + + Input columns:{" "} + {formatColumns(cfg.input_columns)} + + + Output columns:{" "} + {formatColumns(cfg.output_columns)} + + {["train", "validation", "test"].map((key) => { + const ratio = splits[key]; + const size = splitSize(ratio); + return ( + + {key.charAt(0).toUpperCase() + key.slice(1)}:{" "} + {ratio != null ? `${Math.round(ratio * 100)}%` : "-"} + {size != null ? ` (~${size} rows)` : ""} + + ); + })} + + ); + })} + + + + )} + + {hasRetrieveModel && ( + + }> + Retrieve Model + + + + {retrieveModelSteps.map((step) => { + const cfg = step.config || {}; + return ( + + + {cfg.model || "-"} + + + {cfg.task ? `Task: ${cfg.task}` : ""} + + + Input columns:{" "} + {formatColumns(cfg.input_columns)} + + {cfg.model_path && ( + + {cfg.model_path} + + )} + + ); + })} + + + + )} + {hasExploration && ( }> From 45a822da2b06e40c13537663ef9b35079bbc085c Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 10 Jul 2026 19:14:41 -0400 Subject: [PATCH 76/88] refactor: remove legacy Train Model node from back --- DashAI/back/initial_components.py | 2 - DashAI/back/job/pipeline_job.py | 18 +- DashAI/back/pipeline/contracts.py | 28 --- DashAI/back/pipeline/train_node.py | 246 -------------------- DashAI/back/pipeline/validator/__init__.py | 2 - DashAI/back/pipeline/validator/validator.py | 42 ---- 6 files changed, 1 insertion(+), 337 deletions(-) delete mode 100644 DashAI/back/pipeline/train_node.py diff --git a/DashAI/back/initial_components.py b/DashAI/back/initial_components.py index 703d78581..21575d52c 100644 --- a/DashAI/back/initial_components.py +++ b/DashAI/back/initial_components.py @@ -278,7 +278,6 @@ from DashAI.back.pipeline.retrieve_model_node import RetrieveModel from DashAI.back.pipeline.split_data_node import SplitData from DashAI.back.pipeline.task_and_model_node import TaskAndModel -from DashAI.back.pipeline.train_node import Train # Plugins from DashAI.back.plugins.utils import get_available_plugins @@ -492,7 +491,6 @@ def get_initial_components(): Nystroem, DataSelector, DataExploration, - Train, RetrieveModel, Prediction, SplitData, diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index 04683fca5..6c4cbb60c 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -103,7 +103,7 @@ async def _execute_graph() -> None: task_map: Dict[asyncio.Task, str] = {} update_lock = asyncio.Lock() exclusive_lock = asyncio.Lock() - exclusive_types = {"TaskAndModel", "Train"} + exclusive_types = {"TaskAndModel"} async def _run_node(node_id: str) -> None: step = step_map.get(node_id, {}) @@ -348,22 +348,6 @@ def _update_context( "explorations": current_node_exploration, } pipeline.exploration = aggregated_exploration - elif node_type == "Train": - current_train = output.get("train") or {} - context["train"] = current_train - context["model_node_id"] = node_id - - aggregated_train = ( - dict(pipeline.train) if isinstance(pipeline.train, dict) else {} - ) - train_key = node_id or f"Train-{len(aggregated_train)}" - aggregated_train[train_key] = { - "node_id": node_id, - "model_node_id": node_id, - "dataset_name": context.get("dataset_name"), - **current_train, - } - pipeline.train = aggregated_train elif node_type == "SplitData": context["split_data"] = output.get("split_data") pipeline.split_data = context["split_data"] diff --git a/DashAI/back/pipeline/contracts.py b/DashAI/back/pipeline/contracts.py index b40806db1..8a6dcaabd 100644 --- a/DashAI/back/pipeline/contracts.py +++ b/DashAI/back/pipeline/contracts.py @@ -185,34 +185,6 @@ class NodeContract(BaseModel): ) ], ), - NodeContract( - type="Train", - name="Train Model", - icon="SettingsIcon", - description=( - "Trains a model using a selected dataset, with configurable columns, " - "splits, task, model, and metrics." - ), - requiresConfiguration=True, - source=True, - target=True, - input="Dataset object", - output="Trained model object and evaluation results", - inputs=[ - PortSpec( - name="dataset", - kind="DatasetRef", - description="Dataset used for training.", - ) - ], - outputs=[ - PortSpec( - name="model", - kind="ModelArtifact", - description="Trained model artifact.", - ) - ], - ), NodeContract( type="Prediction", name="Prediction", diff --git a/DashAI/back/pipeline/train_node.py b/DashAI/back/pipeline/train_node.py deleted file mode 100644 index 4de50d5a3..000000000 --- a/DashAI/back/pipeline/train_node.py +++ /dev/null @@ -1,246 +0,0 @@ -import logging -from typing import TYPE_CHECKING, Any, Dict, List - -from kink import di, inject - -from DashAI.back.job.base_job import BaseJob, JobError -from DashAI.back.metrics.base_metric import BaseMetric -from DashAI.back.models.base_model import BaseModel -from DashAI.back.models.model_factory import ModelFactory -from DashAI.back.tasks.base_task import BaseTask - -if TYPE_CHECKING: - from DashAI.back.dependencies.registry.component_registry import ComponentRegistry - - -log = logging.getLogger(__name__) - - -class Train(BaseJob): - """ - Train node for training machine learning models in pipelines. - - Parameters - ---------- - input_columns : List[int] - List of column indices to use as input features - output_columns : List[int] - List of column indices to use as output targets - splits : Dict[str, float] - Dictionary with train/validation/test split ratios - task : str - Name of the task to perform (e.g., "TabularClassificationTask") - model : str - Name of the model to train - metrics : List[str] - List of metric names to evaluate - parameters : Dict[str, Any], optional - Model-specific parameters for training - """ - - def __init__( - self, - input_columns: List[int], - output_columns: List[int], - splits: Dict[str, float], - task: str, - model: str, - metrics: List[str], - parameters: Dict[str, Any] = None, - ) -> None: - super().__init__( - kwargs={ - "input_columns": input_columns, - "output_columns": output_columns, - "splits": splits, - "task": task, - "model": model, - "metrics": metrics, - "parameters": parameters, - } - ) - self.input_columns = input_columns - self.output_columns = output_columns - self.splits = splits - self.task = task - self.model = model - self.metrics = metrics - self.parameters = parameters - - def set_status_as_delivered(self) -> None: - log.debug("Train executed successfully.") - - def set_status_as_error(self) -> None: - log.error("Train encountered an error.") - - def get_job_name(self) -> str: - return f"Train: {self.model}" - - @inject - async def run( - self, - context: Dict[str, Any], - config: Dict[str, Any] = lambda: di["config"], - component_registry: "ComponentRegistry" = lambda di: di["component_registry"], - ) -> Dict[str, Any]: - import os - - from DashAI.back.dataloaders.classes.dashai_dataset import ( - get_column_names_from_indexes, - prepare_for_model_session, - select_columns, - split_dataset, - ) - - context["task_name"] = self.task - context["model_name"] = self.model - pipeline_id = context["pipeline_id"] - dataset = context["dataset"] - task: BaseTask = component_registry[self.task]["class"] - task_instance = task() - - input_columns_names = get_column_names_from_indexes(dataset, self.input_columns) - context["input_columns"] = input_columns_names - output_columns_names = get_column_names_from_indexes( - dataset, self.output_columns - ) - - all_metrics = { - component_dict["name"]: component_dict - for component_dict in component_registry.get_components_by_types( - select="Metric" - ) - } - metrics: List[BaseMetric] = [] - for metric_name in self.metrics: - metric_class = all_metrics.get(metric_name)["class"] - if metric_class: - metrics.append(metric_class) - else: - log.warning(f"Metric '{metric_name}' not found in registry.") - - try: - prepared_dataset = task_instance.prepare_for_task( - dataset=dataset, - input_columns=input_columns_names, - output_columns=output_columns_names, - ) - n_labels = None - if self.task in [ - "TextClassificationTask", - "TabularClassificationTask", - "ImageClassificationTask", - ]: - all_classes = prepared_dataset.unique(output_columns_names[0]) - n_labels = len(all_classes) - - splits = self.splits - prepared_dataset, splits = prepare_for_model_session( - dataset=prepared_dataset, - splits=splits, - output_columns=output_columns_names, - ) - - x, y = select_columns( - prepared_dataset, input_columns_names, output_columns_names - ) - x_splits = split_dataset(x) - y_splits = split_dataset(y) - - except Exception as e: - log.exception(e) - raise JobError( - f"Can not prepare dataset for task {self.task}", - ) from e - - try: - model_class = component_registry[self.model]["class"] - context["model_class"] = model_class - except Exception as e: - log.exception(e) - raise JobError( - f"Unable to find Model with name {self.model} in registry.", - ) from e - - try: - parameters = self.parameters - factory = ModelFactory(model_class, parameters, n_labels=n_labels) - model: BaseModel = factory.model - except Exception as e: - log.exception(e) - raise JobError( - f"Unable to instantiate model {self.model}", - ) from e - - try: - # Check if model uses .train() (text models) or .fit() (sklearn models) - if hasattr(model, "train") and callable(model.train): - # Text models expect Dataset objects, not numpy arrays - model.train( - x_splits["train"], - y_splits["train"], - x_splits.get("validation"), - y_splits.get("validation"), - ) - # Keep splits as DashAIDataset for evaluation - x = x_splits - y = y_splits - elif hasattr(model, "fit") and callable(model.fit): - # Sklearn models expect numpy arrays - # Convert DashAIDataset to numpy arrays - x = {} - y = {} - for split in ["train", "validation", "test"]: - if split in x_splits: - x[split] = x_splits[split].to_pandas().values - if split in y_splits: - y[split] = y_splits[split].to_pandas().values.ravel() - - model.fit(x["train"], y["train"]) - else: - raise AttributeError( - f"Model {model_class} has neither 'train' nor 'fit' method" - ) - except Exception as e: - log.exception(e) - raise JobError( - "Model training failed", - ) from e - - try: - model_metrics = factory.evaluate(x, y, metrics) - except Exception as e: - log.exception(e) - raise JobError( - "Metrics calculation failed", - ) from e - - try: - sqlite_local = config["LOCAL_PATH"] - path = os.path.join(sqlite_local, "pipelines", "train") - os.makedirs(path, exist_ok=True) - - node_id = context.get("_node_id", "Train") - pipeline_path = os.path.join(path, str(pipeline_id)) - os.makedirs(pipeline_path, exist_ok=True) - model_path = os.path.join(pipeline_path, str(node_id), "model") - - os.makedirs(os.path.dirname(model_path), exist_ok=True) - model.save(model_path) - context["model_path"] = model_path - except Exception as e: - log.exception(e) - raise JobError( - "Model saving failed", - ) from e - - return { - "train": { - "info": self.model, - "parameters": self.parameters, - "metrics": model_metrics, - "model_path": model_path, - "input_columns": self.input_columns, - "task": self.task, - } - } diff --git a/DashAI/back/pipeline/validator/__init__.py b/DashAI/back/pipeline/validator/__init__.py index 76785faa9..1d5b896e7 100644 --- a/DashAI/back/pipeline/validator/__init__.py +++ b/DashAI/back/pipeline/validator/__init__.py @@ -6,14 +6,12 @@ RetrieveModelValidator, SplitDataValidator, TaskAndModelValidator, - TrainValidator, ) __all__ = [ "PipelineValidator", "DataSelectorValidator", "DataExplorationValidator", - "TrainValidator", "RetrieveModelValidator", "SplitDataValidator", "TaskAndModelValidator", diff --git a/DashAI/back/pipeline/validator/validator.py b/DashAI/back/pipeline/validator/validator.py index aeb4ed406..d9e07855a 100644 --- a/DashAI/back/pipeline/validator/validator.py +++ b/DashAI/back/pipeline/validator/validator.py @@ -54,47 +54,6 @@ def validate(self) -> Dict[str, str]: return {"status": "ok"} -class TrainValidator(BaseNodeValidator): - def validate(self) -> Dict[str, str]: - input_cols = self.data.get("input_columns") - output_cols = self.data.get("output_columns") - task = self.data.get("task") - splits = self.data.get("splits", {}) - metrics = self.data.get("metrics") - model = self.data.get("model") - - if not input_cols or not output_cols: - return { - "status": "error", - "message": "Input and output columns are required", - } - - if task is None: - return {"status": "error", "message": "Task is required"} - - if model is None: - return {"status": "error", "message": "Model is required"} - - if not metrics: - return { - "status": "error", - "message": "At least one metric must be selected", - } - - train, val, test = ( - splits.get("train", 0), - splits.get("validation", 0), - splits.get("test", 0), - ) - if round(train + val + test, 5) != 1.0: - return { - "status": "error", - "message": "Train, validation, and test splits must sum to 1", - } - - return {"status": "ok"} - - class RetrieveModelValidator(BaseNodeValidator): def validate(self) -> Dict[str, str]: model = self.data.get("model") @@ -173,7 +132,6 @@ def validate(self) -> Dict[str, str]: VALIDATOR_MAP = { "DataSelector": DataSelectorValidator, "DataExploration": DataExplorationValidator, - "Train": TrainValidator, "RetrieveModel": RetrieveModelValidator, "SplitData": SplitDataValidator, "TaskAndModel": TaskAndModelValidator, From e86df3b6350c7bb35d284e603a32ece5e3fea67d Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Fri, 10 Jul 2026 19:15:02 -0400 Subject: [PATCH 77/88] refactor: remove Train Model node from the palette of nodes --- .../front/src/components/pipelines/index.jsx | 1 - .../src/components/pipelines/nodeRegistry.jsx | 2 - .../components/pipelines/nodes/TrainNode.jsx | 571 ------------------ 3 files changed, 574 deletions(-) delete mode 100644 DashAI/front/src/components/pipelines/nodes/TrainNode.jsx diff --git a/DashAI/front/src/components/pipelines/index.jsx b/DashAI/front/src/components/pipelines/index.jsx index f6daed8e3..e4b54e45f 100644 --- a/DashAI/front/src/components/pipelines/index.jsx +++ b/DashAI/front/src/components/pipelines/index.jsx @@ -14,7 +14,6 @@ export { default as Run } from "./Run"; // Node components export { default as DataSelectorNode } from "./nodes/DataSelectorNode"; export { default as DataExplorationNode } from "./nodes/DataExplorationNode"; -export { default as TrainNode } from "./nodes/TrainNode"; export { default as RetrieveModelNode } from "./nodes/RetrieveModelNode"; export { default as SplitDataNode } from "./nodes/SplitDataNode"; export { default as TaskAndModelNode } from "./nodes/TaskAndModelNode"; diff --git a/DashAI/front/src/components/pipelines/nodeRegistry.jsx b/DashAI/front/src/components/pipelines/nodeRegistry.jsx index 66cf6b7f6..d488b5a90 100644 --- a/DashAI/front/src/components/pipelines/nodeRegistry.jsx +++ b/DashAI/front/src/components/pipelines/nodeRegistry.jsx @@ -1,6 +1,5 @@ import DataSelectorNode from "./nodes/DataSelectorNode"; import DataExplorationNode from "./nodes/DataExplorationNode"; -import TrainNode from "./nodes/TrainNode"; import RetrieveModelNode from "./nodes/RetrieveModelNode"; import ConfigurableNode from "./nodes/ConfigurableNode"; import SplitDataNode from "./nodes/SplitDataNode"; @@ -10,7 +9,6 @@ import MetricsEvalNode from "./nodes/MetricsEvalNode"; const nodeRegistry = { DataSelector: DataSelectorNode, DataExploration: DataExplorationNode, - Train: TrainNode, RetrieveModel: RetrieveModelNode, Configurable: ConfigurableNode, SplitData: SplitDataNode, diff --git a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx b/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx deleted file mode 100644 index d16fc72f7..000000000 --- a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx +++ /dev/null @@ -1,571 +0,0 @@ -import React, { useState, useEffect, useRef } from "react"; -import PropTypes from "prop-types"; -import { - Box, - Button, - DialogContent, - Typography, - TextField, - MenuItem, - Grid, - IconButton, - FormControlLabel, - Checkbox, -} from "@mui/material"; -import SettingsIcon from "@mui/icons-material/Settings"; -import useSchema from "../../../hooks/useSchema"; -import ParamsSettings from "../ParamsSettings"; -import { getComponents as getComponentsRequest } from "../../../api/component"; -import { validateColumns as validateColumnsRequest } from "../../../api/modelSession"; -import { useSnackbar } from "notistack"; -import { getDatasetInfo as getDatasetInfoRequest } from "../../../api/datasets"; -import { parseRangeToIndex } from "../../../utils/parseRange"; -import { validateNode } from "../../../api/pipeline"; - -const Train = ({ - open, - onClose, - onSave, - savedConfig = null, - prevNodes = [], -}) => { - const [inputColumns, setInputColumns] = useState( - savedConfig?.input_columns || "", - ); - const [outputColumns, setOutputColumns] = useState( - savedConfig?.output_columns || "", - ); - const [splits, setSplits] = useState( - savedConfig?.splits || { - train: 60, - validation: 20, - test: 20, - shuffle: true, - stratify: false, - splitType: "random", - }, - ); - const [task, setTask] = useState(savedConfig?.task || ""); - const [model, setModel] = useState(savedConfig?.model || ""); - const [metrics, setMetrics] = useState(savedConfig?.metrics || []); - const [openSettings, setOpenSettings] = useState(false); - const [modelParams, setModelParams] = useState(savedConfig?.parameters || {}); - const [initialized, setInitialized] = useState(false); - const [availableTasks, setAvailableTasks] = useState([]); - const [availableModels, setAvailableModels] = useState([]); - const [availableMetrics, setAvailableMetrics] = useState([]); - const [validationErrors, setValidationErrors] = useState({}); - const { enqueueSnackbar } = useSnackbar(); - const datasetNode = prevNodes?.find((node) => node?.file_path && node?.id); - const datasetId = datasetNode?.id ?? null; - const [validTasks, setValidTasks] = useState([]); - const [datasetInfo, setDatasetInfo] = useState({}); - const [infoLoading, setInfoLoading] = useState(true); - const [inputError, setInputError] = useState(false); - const [outputError, setOutputError] = useState(false); - const [inputErrorMessage, setInputErrorMessage] = useState(""); - const [outputErrorMessage, setOutputErrorMessage] = useState(""); - const [validationStatus, setValidationStatus] = useState(""); - - useEffect(() => { - const fetchComponents = async () => { - try { - const tasks = await getComponentsRequest({ selectTypes: ["Task"] }); - const models = await getComponentsRequest({ - selectTypes: ["Model"], - relatedComponent: task, - }); - const metrics = await getComponentsRequest({ - selectTypes: ["Metric"], - relatedComponent: task, - }); - - setAvailableTasks(tasks); - setAvailableModels(models); - setAvailableMetrics(metrics); - } catch (error) { - console.error("Error fetching components:", error); - } - }; - - fetchComponents(); - }, [task]); - - const { defaultValues, modelSchema, yupSchema, loading } = useSchema({ - modelName: model, - }); - - useEffect(() => { - const total = datasetInfo?.total_columns; - setInputColumns( - savedConfig?.input_columns?.join(",") || (total ? `1-${total - 1}` : ""), - ); - setOutputColumns( - savedConfig?.output_columns?.join(",") || (total ? `${total}` : ""), - ); - setSplits( - savedConfig?.splits || { - train: 0.6, - validation: 0.2, - test: 0.2, - shuffle: true, - stratify: false, - splitType: "random", - }, - ); - setTask(savedConfig?.task || ""); - setModel(savedConfig?.model || ""); - setMetrics(savedConfig?.metrics || []); - setModelParams(savedConfig?.parameters || {}); - }, [savedConfig, datasetInfo]); - - useEffect(() => { - setInitialized(false); - }, [model]); - - useEffect(() => { - if (loading || initialized) return; - - if ( - savedConfig?.model === model && - savedConfig?.parameters && - Object.keys(savedConfig.parameters).length > 0 - ) { - setModelParams(savedConfig.parameters); - } else if (defaultValues && Object.keys(defaultValues).length > 0) { - setModelParams(defaultValues); - } else { - setModelParams({}); - } - - setInitialized(true); - }, [loading, model, defaultValues, savedConfig, initialized]); - - const handleChange = (newValues) => { - setModelParams(newValues); - }; - - const handleSave = async () => { - const maxValue = datasetInfo?.total_columns; - const columnNames = datasetInfo?.column_names || []; - - const parsedInputIndices = inputColumns - ? parseRangeToIndex(inputColumns, maxValue) - : []; - const parsedOutputIndices = outputColumns - ? parseRangeToIndex(outputColumns, maxValue) - : []; - - // Convert indices to column names (1-based to 0-based and then get names) - const inputColumnNames = parsedInputIndices.map( - (idx) => columnNames[idx - 1] || `Column_${idx}`, - ); - const outputColumnNames = parsedOutputIndices.map( - (idx) => columnNames[idx - 1] || `Column_${idx}`, - ); - - const payload = { - input_columns: parsedInputIndices, - output_columns: parsedOutputIndices, - splits: { - train: splits.train, - validation: splits.validation, - test: splits.test, - shuffle: splits.shuffle, - stratify: splits.stratify, - splitType: splits.splitType, - }, - task: task, - model: model, - metrics: metrics, - parameters: modelParams, - }; - - try { - const response = await validateNode("Train", payload); - if (response.status === "ok") { - setValidationStatus("ok"); - onSave(payload); - onClose(); - } else { - setValidationStatus("error"); - enqueueSnackbar("Validation failed", { variant: "error" }); - } - } catch (e) { - setValidationStatus("error"); - enqueueSnackbar("Error validating node", { variant: "error" }); - console.error(e); - } - }; - - const getDatasetInfo = async () => { - setInfoLoading(true); - try { - const datasetInfo = await getDatasetInfoRequest(datasetId); - setDatasetInfo(datasetInfo); - } catch (error) { - enqueueSnackbar("Error while trying to obtain the dataset info."); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - } finally { - setInfoLoading(false); - } - }; - - useEffect(() => { - if (!datasetId) { - setInfoLoading(false); - return; - } - getDatasetInfo(); - }, [datasetId]); - - const hasWarnedRef = useRef(false); - - useEffect(() => { - const validateAvailableTasks = async () => { - if (!datasetId) { - if (!hasWarnedRef.current) { - enqueueSnackbar("Missing dataset", { variant: "warning" }); - hasWarnedRef.current = true; - } - setValidTasks([]); - return; - } - - const maxValue = datasetInfo?.total_columns; - const columnNames = datasetInfo?.column_names || []; - let input = []; - let output = []; - let parseError = false; - - try { - input = inputColumns ? parseRangeToIndex(inputColumns, maxValue) : []; - setInputError(false); - setInputErrorMessage(""); - } catch (err) { - setInputError(true); - setInputErrorMessage(err.message); - parseError = true; - } - - try { - output = outputColumns - ? parseRangeToIndex(outputColumns, maxValue) - : []; - setOutputError(false); - setOutputErrorMessage(""); - } catch (err) { - setOutputError(true); - setOutputErrorMessage(err.message); - parseError = true; - } - - if (parseError) { - setValidTasks([]); - return; - } - - // Convert indices to column names - const inputColumnNames = input.map( - (idx) => columnNames[idx - 1] || `Column_${idx}`, - ); - const outputColumnNames = output.map( - (idx) => columnNames[idx - 1] || `Column_${idx}`, - ); - - const results = await Promise.all( - availableTasks.map(async (taskObj) => { - try { - const validation = await validateColumnsRequest( - taskObj.name, - datasetId, - inputColumnNames, - outputColumnNames, - ); - if (validation.dataset_status === "valid") { - return { name: taskObj.name, valid: true }; - } else { - return { - name: taskObj.name, - valid: false, - error: validation.error, - }; - } - } catch (error) { - console.warn(`Error validating task ${taskObj.name}`, error); - return { - name: taskObj.name, - valid: false, - error: "Request failed", - }; - } - }), - ); - - // Extract valid tasks and errors - const validTasksList = results.filter((r) => r.valid).map((r) => r.name); - const errorsMap = {}; - results.forEach((r) => { - if (!r.valid) { - errorsMap[r.name] = r.error; - } - }); - - setValidationErrors(errorsMap); - const valid = validTasksList; - setValidTasks(valid); - }; - - validateAvailableTasks(); - }, [availableTasks, inputColumns, outputColumns, datasetId, datasetInfo]); - - return ( - <> - - - - Split Data: - - - - setInputColumns(e.target.value)} - margin="normal" - error={inputError} - helperText={inputError ? inputErrorMessage : ""} - /> - setOutputColumns(e.target.value)} - margin="normal" - error={outputError} - helperText={outputError ? outputErrorMessage : ""} - /> - - - - setSplits({ ...splits, shuffle: e.target.checked }) - } - /> - } - label="Shuffle" - sx={{ ml: 2 }} - /> - - setSplits({ ...splits, stratify: e.target.checked }) - } - /> - } - label="Stratify" - /> - - - - - - setSplits({ ...splits, train: parseFloat(e.target.value, 10) }) - } - margin="normal" - /> - - setSplits({ - ...splits, - validation: parseFloat(e.target.value, 10), - }) - } - margin="normal" - /> - - setSplits({ ...splits, test: parseFloat(e.target.value, 10) }) - } - margin="normal" - /> - - - - Task and Model: - - - {validTasks.length === 0 && - Object.keys(validationErrors).length > 0 && ( - - - ⚠️ No compatible tasks available. Validation errors: - - {Object.entries(validationErrors).map(([taskName, error]) => ( - - {taskName}: {error} - - ))} - - )} - - - { - setTask(e.target.value); - setModel(""); - setMetrics([]); - setModelParams({}); - }} - margin="normal" - disabled={validTasks.length === 0} - > - {availableTasks.map((taskObj) => ( - - {taskObj.name} - - ))} - - - - - - - { - setModel(e.target.value); - }} - margin="normal" - disabled={!task} - > - {availableModels.map((model) => ( - - {model.name} - - ))} - - - - setOpenSettings(true)} - disabled={!model} - aria-label="model settings" - sx={{ mt: "8px" }} - > - - - - - - - - Metrics: - - - - setMetrics(e.target.value)} - margin="normal" - disabled={!task} - slotProps={{ - select: { multiple: true }, - }} - > - {availableMetrics.map((metric) => ( - - {metric.name} - - ))} - - - - - - - - - - setOpenSettings(false)} - /> - - ); -}; - -Train.propTypes = { - open: PropTypes.bool.isRequired, - onClose: PropTypes.func.isRequired, - onSave: PropTypes.func.isRequired, - savedConfig: PropTypes.object, - prevNodes: PropTypes.arrayOf(PropTypes.object), -}; - -export default Train; From abb66ff47f3ac9861ff92794f081b88e0f0a5fcb Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Wed, 22 Jul 2026 22:46:00 -0400 Subject: [PATCH 78/88] feat: add contextual node-palette highlighting to guide pipeline construction --- .../src/components/pipelines/NodeSidebar.jsx | 100 +++++++++++++++--- .../front/src/pages/pipelines/NewPipeline.jsx | 37 ++++++- 2 files changed, 123 insertions(+), 14 deletions(-) diff --git a/DashAI/front/src/components/pipelines/NodeSidebar.jsx b/DashAI/front/src/components/pipelines/NodeSidebar.jsx index b0fa2d8ca..ded7b830d 100644 --- a/DashAI/front/src/components/pipelines/NodeSidebar.jsx +++ b/DashAI/front/src/components/pipelines/NodeSidebar.jsx @@ -1,11 +1,40 @@ import React from "react"; import { Box, Typography } from "@mui/material"; -import { useTheme } from "@mui/material/styles"; +import { useTheme, alpha } from "@mui/material/styles"; import HelpOutlineIcon from "@mui/icons-material/HelpOutline"; +import PlayArrowIcon from "@mui/icons-material/PlayArrow"; +import ArrowForwardIcon from "@mui/icons-material/ArrowForward"; import { getNodeHelp } from "./nodeHelp"; -function NodeSidebar({ availableNodes, onDragStart, nodeHelp }) { +/** + * NodeSidebar – draggable node palette + contextual help. + * + * Highlighting props make pipeline construction more intuitive without ever + * restricting what can be dragged: + * - highlightMode "start": the canvas is empty; root nodes (the ones that can + * open a flow) are emphasized as the suggested starting point. + * - highlightMode "successors": a node is focused on the canvas; the nodes + * that can legally come after it are emphasized and the rest are dimmed. + * The set of emphasized node types is passed in `highlightedTypes`. Non-matching + * nodes stay fully draggable — they are only visually de-emphasized. + */ +function NodeSidebar({ + availableNodes, + onDragStart, + nodeHelp, + highlightMode = null, + highlightedTypes = [], + highlightLabel = "", +}) { const theme = useTheme(); + + const highlightActive = Boolean(highlightMode) && highlightedTypes.length > 0; + const highlightSet = new Set(highlightedTypes); + const accentColor = + highlightMode === "start" + ? theme.palette.success.main + : theme.palette.primary.main; + return ( Nodes - {availableNodes.map((node) => ( + + {highlightActive && highlightLabel && ( onDragStart(e, node.type)} - draggable sx={{ + display: "flex", + alignItems: "center", + gap: 1, mb: 2, - p: 2, - backgroundColor: theme.palette.ui.border, - color: theme.palette.text.primary, + px: 1.5, + py: 1, borderRadius: 1, - textAlign: "center", - cursor: "grab", + backgroundColor: alpha(accentColor, 0.12), + border: `1px solid ${alpha(accentColor, 0.5)}`, }} > - {node.name || node.type} + {highlightMode === "start" ? ( + + ) : ( + + )} + + {highlightLabel} + - ))} + )} + + {availableNodes.map((node) => { + const isHighlighted = highlightActive && highlightSet.has(node.type); + const isDimmed = highlightActive && !isHighlighted; + + return ( + onDragStart(e, node.type)} + draggable + sx={{ + mb: 2, + p: 2, + backgroundColor: isHighlighted + ? alpha(accentColor, 0.16) + : theme.palette.ui.border, + color: theme.palette.text.primary, + borderRadius: 1, + textAlign: "center", + cursor: "grab", + border: isHighlighted + ? `2px solid ${accentColor}` + : "2px solid transparent", + boxShadow: isHighlighted + ? `0 0 8px ${alpha(accentColor, 0.5)}` + : "none", + opacity: isDimmed ? 0.35 : 1, + transition: + "opacity 0.15s ease, box-shadow 0.15s ease, border-color 0.15s ease", + }} + > + {node.name || node.type} + + ); + })} { + const focusedType = nodeHelp?.type || nodeHelp?.data?.type; + const focusedContract = focusedType + ? availableNodes.find((n) => n.type === focusedType) + : null; + + if (focusedContract) { + return { + highlightMode: "successors", + highlightedTypes: focusedContract.successors || [], + highlightLabel: `Puede ir después de: ${ + focusedContract.name || focusedType + }`, + }; + } + + if (nodes.length === 0 && availableNodes.length > 0) { + const roots = availableNodes.filter((n) => n.source && !n.target); + return { + highlightMode: "start", + highlightedTypes: roots.map((n) => n.type), + highlightLabel: "Selecciona y arrastra para comenzar tu pipeline", + }; + } + + return { highlightMode: null, highlightedTypes: [], highlightLabel: "" }; + }, [nodeHelp, availableNodes, nodes.length]); + const handleApplyTemplate = (templateId) => { const selectedTemplate = PIPELINE_TEMPLATES.find( (t) => t.id === templateId, @@ -453,6 +485,9 @@ function NewPipeline() { availableNodes={availableNodes} onDragStart={onDragStart} nodeHelp={nodeHelp} + highlightMode={nodeHighlight.highlightMode} + highlightedTypes={nodeHighlight.highlightedTypes} + highlightLabel={nodeHighlight.highlightLabel} /> Date: Wed, 22 Jul 2026 22:46:17 -0400 Subject: [PATCH 79/88] feat: display comparison metrics per branch in pipeline results --- .../components/pipelines/results/Results.jsx | 75 ++++++++++++++----- .../pipelines/results/ResultsMetrics.jsx | 12 ++- 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 1e793bbce..00b6ed124 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -116,6 +116,7 @@ function buildBranchResults({ function PipelineResults({ pipelineId, onClose }) { const [results, setResults] = useState(null); const [trainTab, setTrainTab] = useState(0); + const [metricsTab, setMetricsTab] = useState(0); const [polling, setPolling] = useState(false); const pollingRef = useRef(null); @@ -180,6 +181,10 @@ function PipelineResults({ pipelineId, onClose }) { setTrainTab(newValue); }; + const handleMetricsTabChange = (event, newValue) => { + setMetricsTab(newValue); + }; + if (!results) { return Loading results...; } @@ -202,8 +207,9 @@ function PipelineResults({ pipelineId, onClose }) { const hasExploration = results.exploration && results.exploration !== "No exploration data"; const hasTrain = - branchResults.filter((branch) => branch.parameters || branch.metrics) - .length > 0; + branchResults.filter((branch) => branch.parameters).length > 0; + const hasMetrics = + branchResults.filter((branch) => !!branch.metrics).length > 0; const hasPrediction = branchResults.filter((branch) => !!branch.prediction).length > 0; @@ -226,6 +232,7 @@ function PipelineResults({ pipelineId, onClose }) { if ( !hasExploration && !hasTrain && + !hasMetrics && !hasPrediction && !hasDataSelector && !hasSplitData && @@ -403,7 +410,7 @@ function PipelineResults({ pipelineId, onClose }) { {branchResults - .filter((branch) => branch.parameters || branch.metrics) + .filter((branch) => branch.parameters) .map((branch, branchIndex) => { const paramData = { parameters: branch.parameters }; @@ -425,8 +432,6 @@ function PipelineResults({ pipelineId, onClose }) { > - - @@ -445,18 +450,6 @@ function PipelineResults({ pipelineId, onClose }) { )} - {trainTab === 2 && ( - - - - )} - {trainTab === 3 && ( - - - - )} ); @@ -466,6 +459,54 @@ function PipelineResults({ pipelineId, onClose }) { )} + {hasMetrics && ( + + }> + Metrics + + + + {branchResults + .filter((branch) => !!branch.metrics) + .map((branch, branchIndex) => ( + + + + Path {branchIndex + 1} | Dataset: {branch.datasetName} + + + {branch.taskName} | {branch.modelName} + + + + + + + + + + {metricsTab === 0 && ( + + + + )} + {metricsTab === 1 && ( + + + + )} + + + ))} + + + + )} + {hasPrediction && ( }> diff --git a/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx b/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx index 916da0c94..5bac4fd4a 100644 --- a/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx +++ b/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx @@ -7,6 +7,10 @@ import ResultsTabMetricsToggle from "../../../pages/results/components/ResultsTa function PipelineResultsMetrics({ metricsData = {} }) { const [displaySet, setDisplaySet] = useState("test_metrics"); + // The default parameter only covers `undefined`; guard against an explicit + // `null` so the component never crashes if it is ever rendered without data. + const safeMetricsData = metricsData || {}; + const displaySetMap = { train_metrics: "train", test_metrics: "test", @@ -14,11 +18,11 @@ function PipelineResultsMetrics({ metricsData = {} }) { }; const currentKey = displaySetMap[displaySet]; - const currentMetrics = metricsData[currentKey] || {}; + const currentMetrics = safeMetricsData[currentKey] || {}; - const hasTrainData = !!metricsData.train; - const hasTestData = !!metricsData.test; - const hasValidationData = !!metricsData.validation; + const hasTrainData = !!safeMetricsData.train; + const hasTestData = !!safeMetricsData.test; + const hasValidationData = !!safeMetricsData.validation; return ( From 621973c05c936e221d9a7662faae9ce228e80039 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Wed, 22 Jul 2026 22:46:25 -0400 Subject: [PATCH 80/88] fix: reshape multi-column ECDF plots to long-form for mixed-dtype overlay --- .../back/exploration/explorers/ecdf_plot.py | 55 ++++++++++++++----- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/DashAI/back/exploration/explorers/ecdf_plot.py b/DashAI/back/exploration/explorers/ecdf_plot.py index ad03bb6f4..902e6e1ac 100644 --- a/DashAI/back/exploration/explorers/ecdf_plot.py +++ b/DashAI/back/exploration/explorers/ecdf_plot.py @@ -288,22 +288,49 @@ def launch_exploration(self, dataset: "DashAIDataset", explorer_info: Explorer): _df = dataset.to_pandas() columns = [col["columnName"] for col in explorer_info.columns] - fig = px.ecdf( - _df, - x=columns[0] if len(columns) == 1 else columns, - color=self.color_column, - facet_col=self.facet_col, - facet_row=self.facet_row, - ecdfnorm=( - self.ecdf_norm.value if self.ecdf_norm is not ECDFNorm.NONE else None - ), - title=( - f"ECDF Plot of {len(columns)} columns" - if len(columns) > 1 - else f"ECDF Plot of {columns[0]}" - ), + ecdfnorm = ( + self.ecdf_norm.value if self.ecdf_norm is not ECDFNorm.NONE else None ) + if len(columns) == 1: + fig = px.ecdf( + _df, + x=columns[0], + color=self.color_column, + facet_col=self.facet_col, + facet_row=self.facet_row, + ecdfnorm=ecdfnorm, + title=f"ECDF Plot of {columns[0]}", + ) + else: + # Reshape wide -> long (tidy data) so that several columns can be + # overlaid in a single ECDF even when their dtypes differ. Plotly + # Express cannot process wide-form data whose columns are of + # different type, and its own documentation recommends long-form for + # this case (Wickham, 2014, "Tidy Data"). Each selected column + # becomes a colored series; an explicit color column, if provided, + # takes precedence over this per-column grouping. + id_vars = [ + col + for col in {self.color_column, self.facet_col, self.facet_row} + if col and col not in columns + ] + long_df = _df.melt( + id_vars=id_vars, + value_vars=columns, + var_name="variable", + value_name="value", + ) + fig = px.ecdf( + long_df, + x="value", + color=self.color_column or "variable", + facet_col=self.facet_col, + facet_row=self.facet_row, + ecdfnorm=ecdfnorm, + title=f"ECDF Plot of {len(columns)} columns", + ) + if explorer_info.name is not None and explorer_info.name != "": fig.update_layout(title=f"{explorer_info.name}") From 3b97074506ab6e59d7d8b2c9e6e8ce72d98336cb Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Wed, 22 Jul 2026 22:57:28 -0400 Subject: [PATCH 81/88] fix: apply pre-commit formatting (ruff-format, prettier) --- DashAI/back/exploration/explorers/ecdf_plot.py | 4 +--- DashAI/front/src/components/pipelines/results/Results.jsx | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DashAI/back/exploration/explorers/ecdf_plot.py b/DashAI/back/exploration/explorers/ecdf_plot.py index 902e6e1ac..e447e7f36 100644 --- a/DashAI/back/exploration/explorers/ecdf_plot.py +++ b/DashAI/back/exploration/explorers/ecdf_plot.py @@ -288,9 +288,7 @@ def launch_exploration(self, dataset: "DashAIDataset", explorer_info: Explorer): _df = dataset.to_pandas() columns = [col["columnName"] for col in explorer_info.columns] - ecdfnorm = ( - self.ecdf_norm.value if self.ecdf_norm is not ECDFNorm.NONE else None - ) + ecdfnorm = self.ecdf_norm.value if self.ecdf_norm is not ECDFNorm.NONE else None if len(columns) == 1: fig = px.ecdf( diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 00b6ed124..7753edbd0 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -491,7 +491,9 @@ function PipelineResults({ pipelineId, onClose }) { {metricsTab === 0 && ( - + )} {metricsTab === 1 && ( From 0c9ba09315fa4dd24f90d2da9f0f279afe4acca2 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 21 Sep 2026 14:01:12 -0300 Subject: [PATCH 82/88] fix: point /app/pipelines to NewPipelineWrapper after develop merge --- DashAI/front/src/App.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DashAI/front/src/App.jsx b/DashAI/front/src/App.jsx index 3288eeb0d..e31462721 100644 --- a/DashAI/front/src/App.jsx +++ b/DashAI/front/src/App.jsx @@ -160,7 +160,7 @@ function App() { path="/app/generative/sessions/:id" element={} /> - } /> + } /> } From 4d93a35088cabb05f06425fa5d241d4d3ebd7a86 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 21 Sep 2026 14:16:25 -0300 Subject: [PATCH 83/88] style: ruff check for model_factory.py --- DashAI/back/models/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DashAI/back/models/model_factory.py b/DashAI/back/models/model_factory.py index 1fb2fcbdf..7829ba66a 100644 --- a/DashAI/back/models/model_factory.py +++ b/DashAI/back/models/model_factory.py @@ -1,6 +1,6 @@ from collections import namedtuple -import numpy as np +import numpy as np from kink import di from DashAI.back.core.schema_fields.search_space import SEARCH_DTYPE_KEY From 12c26e706196273ab947af1831fb1ee0b465450c Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 21 Sep 2026 19:40:21 -0300 Subject: [PATCH 84/88] fix: merge migration heads --- ...7_merge_develop_and_dag_pipelines_heads.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 DashAI/alembic/versions/50e54749a557_merge_develop_and_dag_pipelines_heads.py diff --git a/DashAI/alembic/versions/50e54749a557_merge_develop_and_dag_pipelines_heads.py b/DashAI/alembic/versions/50e54749a557_merge_develop_and_dag_pipelines_heads.py new file mode 100644 index 000000000..1ee8231f7 --- /dev/null +++ b/DashAI/alembic/versions/50e54749a557_merge_develop_and_dag_pipelines_heads.py @@ -0,0 +1,23 @@ +"""merge develop and dag-pipelines heads + +Revision ID: 50e54749a557 +Revises: a5f2c71e9d40, b1cf2532fbbf +Create Date: 2026-09-21 14:00:00.000000 + +""" + +from typing import Sequence, Union + +# revision identifiers, used by Alembic. +revision: str = "50e54749a557" +down_revision: Union[str, None] = ("a5f2c71e9d40", "b1cf2532fbbf") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass From d3c6881bd5f231116b80e56da52ad355bb8f02da Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 28 Sep 2026 18:17:33 -0300 Subject: [PATCH 85/88] fix: update pipeline history grid to x-data-grid v8 --- .../src/components/pipelines/PipelineHistorySidebar.jsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx index 43f92b848..9b25eca6d 100644 --- a/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx @@ -67,7 +67,10 @@ function PipelineHistorySidebar({ currentPipelineId }) { const selectedPipeline = useMemo(() => { const parsedId = Number(currentPipelineId); - return Number.isFinite(parsedId) ? [parsedId] : []; + return { + type: "include", + ids: new Set(Number.isFinite(parsedId) ? [parsedId] : []), + }; }, [currentPipelineId]); const handleOpenRowMenu = (event, pipeline) => { From 19eba5cdc18a944a27167c1d0670c868a40d43cf Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Mon, 28 Sep 2026 18:18:03 -0300 Subject: [PATCH 86/88] chore: remove pipelines from home and navbar --- .../front/src/components/ResponsiveAppBar.jsx | 1 - DashAI/front/src/pages/home/Home.jsx | 19 ------------------- 2 files changed, 20 deletions(-) diff --git a/DashAI/front/src/components/ResponsiveAppBar.jsx b/DashAI/front/src/components/ResponsiveAppBar.jsx index 8ef14ab26..10c06ec2d 100644 --- a/DashAI/front/src/components/ResponsiveAppBar.jsx +++ b/DashAI/front/src/components/ResponsiveAppBar.jsx @@ -31,7 +31,6 @@ function ResponsiveAppBar() { const pages = [ { name: t("common:datasets"), to: "/app/data" }, { name: t("common:models"), to: "/app/models" }, - { name: t("common:pipelines"), to: "/app/pipelines" }, { name: t("common:generative"), to: "/app/generative" }, { name: t("common:plugins"), to: "/app/plugins/browse" }, ]; diff --git a/DashAI/front/src/pages/home/Home.jsx b/DashAI/front/src/pages/home/Home.jsx index ac42b1623..0ccc03a26 100644 --- a/DashAI/front/src/pages/home/Home.jsx +++ b/DashAI/front/src/pages/home/Home.jsx @@ -3,7 +3,6 @@ import { FileUpload as FileUploadIcon, Science as ScienceIcon, Extension as ExtensionIcon, - AccountTree as AccountTreeIcon, AutoAwesome as AutoAwesomeIcon, DescriptionOutlined as DocsIcon, SchoolOutlined as TutorialsIcon, @@ -168,24 +167,6 @@ function Home() { ], tourAttr: "models-button", }, - { - title: t("common:pipelines"), - description: t("home:description.pipelines"), - to: "/app/pipelines", - Icon: AccountTreeIcon, - accent: theme.palette.accent.green, - accentDim: theme.palette.accent.greenDim, - accentBorder: theme.palette.accent.greenBorder, - accentGlow: theme.palette.accent.greenGlow, - tag: t("home:tag.advanced"), - chips: [ - t("home:chip.exploreConvertData"), - t("home:chip.modelTraining"), - t("home:chip.modelComparison"), - t("home:chip.predictions"), - ], - tourAttr: "pipelines-button", - }, { title: t("common:generative"), description: t("home:description.generative"), From ecc678d5396b3c7a1810242dac3c087ce2c3f71d Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Tue, 29 Sep 2026 19:57:19 -0300 Subject: [PATCH 87/88] fix: merge alembic heads after develop merge --- ...9_merge_develop_and_dag_pipelines_heads.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 DashAI/alembic/versions/66a67b766b49_merge_develop_and_dag_pipelines_heads.py diff --git a/DashAI/alembic/versions/66a67b766b49_merge_develop_and_dag_pipelines_heads.py b/DashAI/alembic/versions/66a67b766b49_merge_develop_and_dag_pipelines_heads.py new file mode 100644 index 000000000..f0b8398fd --- /dev/null +++ b/DashAI/alembic/versions/66a67b766b49_merge_develop_and_dag_pipelines_heads.py @@ -0,0 +1,23 @@ +"""merge develop and dag-pipelines heads + +Revision ID: 66a67b766b49 +Revises: 50e54749a557, f4a91c62d8e7 +Create Date: 2026-09-29 12:00:00.000000 + +""" + +from typing import Sequence, Union + +# revision identifiers, used by Alembic. +revision: str = "66a67b766b49" +down_revision: Union[str, None] = ("50e54749a557", "f4a91c62d8e7") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass From 812d02b878998c87c072da14d063d02c9c1e3d96 Mon Sep 17 00:00:00 2001 From: pip3alfar0 Date: Tue, 29 Sep 2026 20:36:43 -0300 Subject: [PATCH 88/88] fix: make preprocessing migration test downgrade unambiguous --- .../migrations/test_add_preprocessing_to_model_session.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/back/migrations/test_add_preprocessing_to_model_session.py b/tests/back/migrations/test_add_preprocessing_to_model_session.py index 6dd1ddf2b..ce2c352c4 100644 --- a/tests/back/migrations/test_add_preprocessing_to_model_session.py +++ b/tests/back/migrations/test_add_preprocessing_to_model_session.py @@ -8,7 +8,12 @@ def test_migration_adds_preprocessing_columns(tmp_path): alembic_cfg = Config("alembic.ini") alembic_cfg.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}") - command.upgrade(alembic_cfg, "head") + # Upgrade to the migration under test rather than to ``head``: once this + # branch is merged into develop the head becomes a merge revision (two + # parents), which makes the relative ``downgrade("-1")`` below ambiguous. + # Targeting the revision keeps the downgrade unambiguous and tests exactly + # this migration's up/down behavior. + command.upgrade(alembic_cfg, "f4a91c62d8e7") engine = sa.create_engine(f"sqlite:///{db_path}") inspector = sa.inspect(engine)