diff --git a/DashAI/alembic/versions/0dc6fd4637a3_merge_dag_pipelines_and_develop_heads.py b/DashAI/alembic/versions/0dc6fd4637a3_merge_dag_pipelines_and_develop_heads.py new file mode 100644 index 000000000..1e34682f2 --- /dev/null +++ b/DashAI/alembic/versions/0dc6fd4637a3_merge_dag_pipelines_and_develop_heads.py @@ -0,0 +1,26 @@ +"""merge dag-pipelines and develop heads + +Revision ID: 0dc6fd4637a3 +Revises: b4f9e70098e7, c4f7a8b2d1e3 +Create Date: 2026-04-02 19:32:55.182155 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '0dc6fd4637a3' +down_revision: Union[str, None] = ('b4f9e70098e7', 'c4f7a8b2d1e3') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/DashAI/alembic/versions/50e54749a557_merge_develop_and_dag_pipelines_heads.py b/DashAI/alembic/versions/50e54749a557_merge_develop_and_dag_pipelines_heads.py new file mode 100644 index 000000000..1ee8231f7 --- /dev/null +++ b/DashAI/alembic/versions/50e54749a557_merge_develop_and_dag_pipelines_heads.py @@ -0,0 +1,23 @@ +"""merge develop and dag-pipelines heads + +Revision ID: 50e54749a557 +Revises: a5f2c71e9d40, b1cf2532fbbf +Create Date: 2026-09-21 14:00:00.000000 + +""" + +from typing import Sequence, Union + +# revision identifiers, used by Alembic. +revision: str = "50e54749a557" +down_revision: Union[str, None] = ("a5f2c71e9d40", "b1cf2532fbbf") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/DashAI/alembic/versions/66a67b766b49_merge_develop_and_dag_pipelines_heads.py b/DashAI/alembic/versions/66a67b766b49_merge_develop_and_dag_pipelines_heads.py new file mode 100644 index 000000000..f0b8398fd --- /dev/null +++ b/DashAI/alembic/versions/66a67b766b49_merge_develop_and_dag_pipelines_heads.py @@ -0,0 +1,23 @@ +"""merge develop and dag-pipelines heads + +Revision ID: 66a67b766b49 +Revises: 50e54749a557, f4a91c62d8e7 +Create Date: 2026-09-29 12:00:00.000000 + +""" + +from typing import Sequence, Union + +# revision identifiers, used by Alembic. +revision: str = "66a67b766b49" +down_revision: Union[str, None] = ("50e54749a557", "f4a91c62d8e7") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/DashAI/alembic/versions/b1cf2532fbbf_merge_pipeline_tracking_and_folder_.py b/DashAI/alembic/versions/b1cf2532fbbf_merge_pipeline_tracking_and_folder_.py new file mode 100644 index 000000000..171729071 --- /dev/null +++ b/DashAI/alembic/versions/b1cf2532fbbf_merge_pipeline_tracking_and_folder_.py @@ -0,0 +1,25 @@ +"""merge pipeline tracking and folder system heads + +Revision ID: b1cf2532fbbf +Revises: d9c8f2a1b7e4, f1a2b3c4d5e6 +Create Date: 2026-06-25 00:13:45.379060 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = 'b1cf2532fbbf' +down_revision: Union[str, None] = ('d9c8f2a1b7e4', 'f1a2b3c4d5e6') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py b/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py new file mode 100644 index 000000000..426580b6d --- /dev/null +++ b/DashAI/alembic/versions/c4f7a8b2d1e3_atomize_train_node_into_split_task_metrics.py @@ -0,0 +1,107 @@ +"""Atomize train node into split/task/metrics tables. + +Revision ID: c4f7a8b2d1e3 +Revises: 98791b9df4f0 +Create Date: 2026-03-25 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c4f7a8b2d1e3" +down_revision: Union[str, None] = "98791b9df4f0" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _table_exists(table_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return table_name in inspector.get_table_names() + + +def _column_exists(table_name: str, column_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return any( + column["name"] == column_name + for column in inspector.get_columns(table_name) + ) + + +def upgrade() -> None: + if _table_exists("pipeline"): + if not _column_exists("pipeline", "split_data"): + op.add_column("pipeline", sa.Column("split_data", sa.JSON(), nullable=True)) + if not _column_exists("pipeline", "task_and_model"): + op.add_column( + "pipeline", sa.Column("task_and_model", sa.JSON(), nullable=True) + ) + if not _column_exists("pipeline", "metrics_result"): + op.add_column( + "pipeline", sa.Column("metrics_result", sa.JSON(), nullable=True) + ) + + if not _table_exists("split_data_node"): + op.create_table( + "split_data_node", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.Column("input_columns", sa.JSON(), nullable=False), + sa.Column("output_columns", sa.JSON(), nullable=False), + sa.Column("splits", sa.JSON(), nullable=False), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + + if not _table_exists("task_and_model_node"): + op.create_table( + "task_and_model_node", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.Column("task_name", sa.String(), nullable=False), + sa.Column("model_name", sa.String(), nullable=False), + sa.Column("parameters", sa.JSON(), nullable=True), + sa.Column("model_path", sa.String(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + + if not _table_exists("metrics_node"): + op.create_table( + "metrics_node", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.Column("metric_names", sa.JSON(), nullable=False), + sa.Column("results", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + if _table_exists("metrics_node"): + op.drop_table("metrics_node") + + if _table_exists("task_and_model_node"): + op.drop_table("task_and_model_node") + + if _table_exists("split_data_node"): + op.drop_table("split_data_node") + + if _table_exists("pipeline"): + with op.batch_alter_table("pipeline", schema=None) as batch_op: + if _column_exists("pipeline", "metrics_result"): + batch_op.drop_column("metrics_result") + if _column_exists("pipeline", "task_and_model"): + batch_op.drop_column("task_and_model") + if _column_exists("pipeline", "split_data"): + batch_op.drop_column("split_data") diff --git a/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py b/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py new file mode 100644 index 000000000..d99ac3ca0 --- /dev/null +++ b/DashAI/alembic/versions/d9c8f2a1b7e4_add_pipeline_run_tracking.py @@ -0,0 +1,108 @@ +"""Add pipeline run tracking tables. + +Revision ID: d9c8f2a1b7e4 +Revises: 0dc6fd4637a3 +Create Date: 2026-04-10 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "d9c8f2a1b7e4" +down_revision: Union[str, None] = "0dc6fd4637a3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +def _table_exists(table_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return table_name in inspector.get_table_names() + +def upgrade() -> None: + if not _table_exists("pipeline_run"): + op.create_table( + "pipeline_run", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.Column("steps", sa.JSON(), nullable=True), + sa.Column("edges", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), + sa.Column("delivery_time", sa.DateTime(), nullable=True), + sa.Column("start_time", sa.DateTime(), nullable=True), + sa.Column("end_time", sa.DateTime(), nullable=True), + sa.Column( + "status", + sa.Enum( + "NOT_STARTED", + "DELIVERED", + "STARTED", + "FINISHED", + "ERROR", + name="pipelinerunstatus", + ), + nullable=False, + ), + sa.Column("error_message", sa.String(), nullable=True), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipeline.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + + if not _table_exists("pipeline_node_run"): + op.create_table( + "pipeline_node_run", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_run_id", sa.Integer(), nullable=False), + sa.Column("node_id", sa.String(), nullable=False), + sa.Column("node_type", sa.String(), nullable=False), + sa.Column("config", sa.JSON(), nullable=True), + sa.Column("input", sa.JSON(), nullable=True), + sa.Column("output", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("last_modified", sa.DateTime(), nullable=False), + sa.Column("delivery_time", sa.DateTime(), nullable=True), + sa.Column("start_time", sa.DateTime(), nullable=True), + sa.Column("end_time", sa.DateTime(), nullable=True), + sa.Column( + "status", + sa.Enum( + "NOT_STARTED", + "DELIVERED", + "STARTED", + "FINISHED", + "ERROR", + name="noderunstatus", + ), + nullable=False, + ), + sa.Column("error_message", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["pipeline_run_id"], ["pipeline_run.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + if not _table_exists("pipeline_node_artifact"): + op.create_table( + "pipeline_node_artifact", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("node_run_id", sa.Integer(), nullable=False), + sa.Column("key", sa.String(), nullable=False), + sa.Column("value", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["node_run_id"], ["pipeline_node_run.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + +def downgrade() -> None: + if _table_exists("pipeline_node_artifact"): + op.drop_table("pipeline_node_artifact") + + if _table_exists("pipeline_node_run"): + op.drop_table("pipeline_node_run") + + if _table_exists("pipeline_run"): + op.drop_table("pipeline_run") diff --git a/DashAI/back/api/api_v1/endpoints/datasets.py b/DashAI/back/api/api_v1/endpoints/datasets.py index 028011069..4df570ce2 100644 --- a/DashAI/back/api/api_v1/endpoints/datasets.py +++ b/DashAI/back/api/api_v1/endpoints/datasets.py @@ -1954,6 +1954,11 @@ async def preview_with_types( detail=f"Dataloader {dataloader_name} not found in registry.", ) + if dataloader_name == "CSVDataLoader" and "separator" not in parsed_params: + parsed_params["separator"] = "," + if dataloader_name == "JSONDataLoader" and "data_key" not in parsed_params: + parsed_params["data_key"] = None + dataloader_cls = component_registry[dataloader_name]["class"] dataloader = dataloader_cls() @@ -2179,3 +2184,269 @@ async def validate_type_changes( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to validate type changes: {str(e)}", ) from e + + +@router.post("/load_preview") +@inject +async def load_preview( + file: UploadFile = File(...), + params: str = Form(None), + component_registry: Dict = Depends(lambda: di["component_registry"]), +): + """ + Alias for preview_with_types - Load preview of a dataset file. + This endpoint is for backward compatibility with frontend API calls. + + Parameters + ---------- + file : UploadFile + The file uploaded by the user. + params : str + JSON string with parameters for the dataloader. + component_registry : Dict + Registry of available dataloaders. + + Returns + ------- + Dict + A dictionary containing the dataset sample and the Arrow schema. + """ + import contextlib + import json + import os + import shutil + import tempfile + import zipfile + + import pyarrow as pa + + from DashAI.back.types.utils import arrow_to_dashai_schema + + try: + parsed_params = json.loads(params) if params else {} + + with tempfile.NamedTemporaryFile( + delete=False, suffix=file.filename + ) as tmp_file: + content = await file.read() + tmp_file.write(content) + tmp_file_path = tmp_file.name + + try: + inference_rows = parsed_params.get("inference_rows", 1000) + if file.filename.endswith(".zip"): + extract_dir = tempfile.mkdtemp() + try: + with zipfile.ZipFile(tmp_file_path, "r") as zf: + zf.extractall(extract_dir) + + supported_map = { + ".csv": "CSVDataLoader", + ".json": "JSONDataLoader", + ".xlsx": "ExcelDataLoader", + ".xls": "ExcelDataLoader", + } + dataloader_name = None + matched_file = None + for root, _, files in os.walk(extract_dir): + for f in files: + ext = os.path.splitext(f)[1].lower() + if ext in supported_map: + dataloader_name = supported_map[ext] + matched_file = os.path.join(root, f) + break + if dataloader_name: + break + + if dataloader_name is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "ZIP file does not contain any supported dataset files." + "Supported inner files: .csv, .json, .xlsx, .xls" + ), + ) + + if dataloader_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=( + f"Dataloader {dataloader_name} not found in registry." + ), + ) + + dataloader = component_registry[dataloader_name]["class"]() + + if ( + dataloader_name == "CSVDataLoader" + and "separator" not in parsed_params + ): + parsed_params["separator"] = "," + if ( + dataloader_name == "JSONDataLoader" + and "data_key" not in parsed_params + ): + parsed_params["data_key"] = None + + loaded_dataset = dataloader.load_preview( + filepath_or_buffer=matched_file, + params=parsed_params, + n_rows=inference_rows, + ) + + finally: + with contextlib.suppress(Exception): + shutil.rmtree(extract_dir, ignore_errors=True) + + else: + if file.filename.endswith(".csv"): + dataloader_name = "CSVDataLoader" + if "separator" not in parsed_params: + parsed_params["separator"] = "," + + elif file.filename.endswith(".xlsx") or file.filename.endswith(".xls"): + dataloader_name = "ExcelDataLoader" + + elif file.filename.endswith(".json"): + dataloader_name = "JSONDataLoader" + if "data_key" not in parsed_params: + parsed_params["data_key"] = None + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file type. Only CSV, Excel and JSON files are " + "supported." + ), + ) + + if dataloader_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Dataloader {dataloader_name} not found in registry.", + ) + + dataloader = component_registry[dataloader_name]["class"]() + + loaded_dataset = dataloader.load_preview( + filepath_or_buffer=tmp_file_path, + params=parsed_params, + n_rows=inference_rows, + ) + + sample_df = loaded_dataset.head(100) + + table = pa.Table.from_pandas(loaded_dataset) + arrow_schema = arrow_to_dashai_schema(table) + + return { + "sample": sample_df.to_dict(orient="records"), + "schema": arrow_schema, + } + + finally: + if os.path.exists(tmp_file_path): + os.unlink(tmp_file_path) + + except Exception as e: + logger.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load preview: {str(e)}", + ) from e + + +@router.post("/infer_datatypes") +@inject +async def infer_datatypes( + file: UploadFile = File(...), + params: str = Form(None), + component_registry: Dict = Depends(lambda: di["component_registry"]), +): + """ + Infer data types for dataset columns. + This endpoint is for backward compatibility with frontend API calls. + + Parameters + ---------- + file : UploadFile + The file uploaded by the user. + params : str + JSON string with parameters for the dataloader. + component_registry : Dict + Registry of available dataloaders. + + Returns + ------- + Dict + A dictionary containing inferred types for each column. + """ + import json + import os + import tempfile + + from DashAI.back.types.inf.type_inference import infer_types + + try: + parsed_params = json.loads(params) if params else {} + + with tempfile.NamedTemporaryFile( + delete=False, suffix=file.filename + ) as tmp_file: + content = await file.read() + tmp_file.write(content) + tmp_file_path = tmp_file.name + + try: + inference_rows = parsed_params.get("inference_rows", 1000) + methods = parsed_params.get("methods", ["DashAIPtype"]) + + if file.filename.endswith(".csv"): + dataloader_name = "CSVDataLoader" + if "separator" not in parsed_params: + parsed_params["separator"] = "," + + elif file.filename.endswith(".xlsx") or file.filename.endswith(".xls"): + dataloader_name = "ExcelDataLoader" + + elif file.filename.endswith(".json"): + dataloader_name = "JSONDataLoader" + if "data_key" not in parsed_params: + parsed_params["data_key"] = None + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Unsupported file type", + ) + + if dataloader_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Dataloader {dataloader_name} not found", + ) + + dataloader = component_registry[dataloader_name]["class"]() + + loaded_dataset = dataloader.load_preview( + filepath_or_buffer=tmp_file_path, + params=parsed_params, + n_rows=inference_rows, + ) + + inferred_types = {} + for method in methods: + method_types = infer_types(loaded_dataset, method=method) + inferred_types.update(method_types) + + return inferred_types + + finally: + if os.path.exists(tmp_file_path): + os.unlink(tmp_file_path) + + except Exception as e: + logger.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to infer datatypes: {str(e)}", + ) from e diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index aad2cc72b..bfa16dcf5 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -230,6 +230,11 @@ async def validate_columns( except (TypeError, ValueError) as e: validation_response["dataset_status"] = "invalid" validation_response["error"] = str(e) + except Exception as e: + log.exception(f"[VALIDATION] Unexpected error in task {params.task_name}") + validation_response["dataset_status"] = "invalid" + validation_response["error"] = f"Unexpected error: {str(e)}" + return validation_response diff --git a/DashAI/back/api/api_v1/endpoints/pipelines.py b/DashAI/back/api/api_v1/endpoints/pipelines.py index b0671fdbf..ce4e617da 100644 --- a/DashAI/back/api/api_v1/endpoints/pipelines.py +++ b/DashAI/back/api/api_v1/endpoints/pipelines.py @@ -9,11 +9,18 @@ DatasetFilterParams, PipelineCreateParams, PipelineUpdateParams, + ValidateEdgeParams, ValidateNodeParams, ValidatePipelineParams, ) from DashAI.back.core.artifacts import normalize_artifacts -from DashAI.back.dependencies.database.models import Dataset, Pipeline +from DashAI.back.dependencies.database.models import ( + Dataset, + NodeRun, + Pipeline, + PipelineRun, +) +from DashAI.back.pipeline.contracts import get_contracts_payload, validate_edge from DashAI.back.pipeline.validator.nodes_definitions import NODES from DashAI.back.pipeline.validator.pipeline_validator import PipelineValidator from DashAI.back.pipeline.validator.validator import VALIDATOR_MAP @@ -77,6 +84,18 @@ async def get_nodes() -> List[Dict[str, Any]]: ) from e +@router.get("/contracts") +async def get_node_contracts() -> List[Dict[str, Any]]: + """Retrieve pipeline node contracts.""" + try: + return get_contracts_payload() + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to load node contracts", + ) from e + + @router.get("/predict_summary") @inject async def pipeline_predict_summary( @@ -124,28 +143,52 @@ class distribution (if applicable), and sample data. if isinstance(data[0], str): summary["data_type"] = "string" else: - summary["data_type"] = "numeric" - class_set = set(data) - classes = [str(item) for item in class_set] - summary["Unique_classes"] = len(classes) - class_distribution = [] - id = 1 - for class_name in classes: - try: - occurrences = data.count(int(class_name)) - except ValueError as e: - raise HTTPException( - status_code=400, detail=f"Invalid class value: {class_name}" - ) from e - distribution = { - "id": id, - "Class": class_name, - "Ocurrences": occurrences, - "Percentage": round(occurrences / len(data) * 100, 2), + # Check if data looks like integers or floats + is_classification = all( + isinstance(x, int) or (isinstance(x, float) and x.is_integer()) + for x in data + ) + unique_values = len(set(data)) + + # If there are many unique values relative to total -> regression + if not is_classification or unique_values > len(data) * 0.5: + # Regression: provide statistics + summary["data_type"] = "regression" + import numpy as np + + data_array = np.array(data) + summary["statistics"] = { + "mean": float(np.mean(data_array)), + "median": float(np.median(data_array)), + "std": float(np.std(data_array)), + "min": float(np.min(data_array)), + "max": float(np.max(data_array)), } - id += 1 - class_distribution.append(distribution) - summary["class_distribution"] = class_distribution + else: + # Classification: count class occurrences + summary["data_type"] = "classification" + class_set = set(data) + classes = [str(item) for item in class_set] + summary["Unique_classes"] = len(classes) + class_distribution = [] + id = 1 + for class_name in classes: + # Count occurrences of the actual value (int or float) + original_value = ( + int(float(class_name)) + if "." not in class_name + else float(class_name) + ) + occurrences = data.count(original_value) + distribution = { + "id": id, + "Class": class_name, + "Ocurrences": occurrences, + "Percentage": round(occurrences / len(data) * 100, 2), + } + id += 1 + class_distribution.append(distribution) + summary["class_distribution"] = class_distribution sample_data = [ {"id": idx, "value": value} for idx, value in enumerate(data[:50], 1) @@ -204,6 +247,59 @@ async def get_pipeline( return pipeline +@router.get("/{pipeline_id}/runs/latest") +@inject +async def get_latest_pipeline_run( + pipeline_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Return the latest execution run of a pipeline with per-node statuses. + + Used by the frontend to color the pipeline nodes live as they execute. + + Returns + ------- + dict + The latest run id, its status name and a list of node runs, each with + its node_id, node_type and status name (NOT_STARTED, DELIVERED, + STARTED, FINISHED, ERROR). + """ + with session_factory() as db: + try: + run = ( + db.query(PipelineRun) + .filter(PipelineRun.pipeline_id == pipeline_id) + .order_by(PipelineRun.id.desc()) + .first() + ) + if run is None: + return {"id": None, "status": None, "node_runs": []} + + node_runs = ( + db.query(NodeRun).filter(NodeRun.pipeline_run_id == run.id).all() + ) + return { + "id": run.id, + "status": run.status.name if run.status is not None else None, + "error_message": run.error_message, + "node_runs": [ + { + "node_id": nr.node_id, + "node_type": nr.node_type, + "status": nr.status.name if nr.status is not None else None, + "error_message": nr.error_message, + } + for nr in node_runs + ], + } + except exc.SQLAlchemyError as e: + logger.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + @router.get("/{pipeline_id}/dataexploration/results/") @inject async def get_pipeline_dataexploration_results( @@ -257,9 +353,46 @@ async def get_pipeline_dataexploration_results( detail="Pipeline has no valid Data Exploration step", ) - results = {} - - for exploration_id, exploration_info in dataexploration.items(): + grouped_results = {} + steps = pipeline.steps or [] + edges = pipeline.edges or [] + + step_map = { + step.get("id"): step + for step in steps + if isinstance(step, dict) and step.get("id") + } + predecessor_map = {} + for edge in edges: + source = edge.get("source") if isinstance(edge, dict) else None + target = edge.get("target") if isinstance(edge, dict) else None + if source and target: + predecessor_map.setdefault(target, []).append(source) + + def resolve_dataset_name_from_graph(node_id: str | None) -> str | None: + if not node_id: + return None + + current = node_id + visited = set() + while current and current not in visited: + visited.add(current) + step = step_map.get(current) + if isinstance(step, dict) and step.get("type") == "DataSelector": + config = step.get("config") or {} + return config.get("name") + + predecessors = predecessor_map.get(current) or [] + current = predecessors[0] if predecessors else None + + return None + + def append_result( + dataset_name: str, + node_id: str, + exploration_id: str, + exploration_info: dict, + ) -> None: exploration_type = exploration_info["exploration_type"] exploration_path = exploration_info["path"] parameters = exploration_info.get("parameters", {}) @@ -279,11 +412,15 @@ async def get_pipeline_dataexploration_results( exploration_path=exploration_path, options={}, ) - results[exploration_id] = { - "exploration_type": exploration_type, - "results": normalize_artifacts(result), - "name": name, - } + grouped_results.setdefault(dataset_name, []).append( + { + "node_id": node_id, + "exploration_id": exploration_id, + "exploration_type": exploration_type, + "results": normalize_artifacts(result), + "name": name, + } + ) except Exception as e: logger.exception(e) raise HTTPException( @@ -291,7 +428,54 @@ async def get_pipeline_dataexploration_results( detail=(f"Error while getting results for '{exploration_type}'"), ) from e - return results + for node_or_exploration_id, node_or_exploration_info in dataexploration.items(): + if not isinstance(node_or_exploration_info, dict): + continue + + if "explorations" in node_or_exploration_info: + inferred_dataset_name = resolve_dataset_name_from_graph( + node_or_exploration_id + ) + dataset_name = ( + node_or_exploration_info.get("dataset_name") + or inferred_dataset_name + or "Unknown Dataset" + ) + node_explorations = node_or_exploration_info.get("explorations") or {} + if not isinstance(node_explorations, dict): + continue + + for exploration_id, exploration_info in node_explorations.items(): + if not isinstance(exploration_info, dict): + continue + append_result( + dataset_name=dataset_name, + node_id=node_or_exploration_id, + exploration_id=exploration_id, + exploration_info=exploration_info, + ) + continue + + dataexploration_nodes = [ + step + for step in steps + if isinstance(step, dict) and step.get("type") == "DataExploration" + ] + inferred_legacy_node_id = ( + dataexploration_nodes[-1].get("id") if dataexploration_nodes else "legacy" + ) + inferred_legacy_dataset_name = ( + resolve_dataset_name_from_graph(inferred_legacy_node_id) + or "Unknown Dataset" + ) + append_result( + dataset_name=inferred_legacy_dataset_name, + node_id=inferred_legacy_node_id or "legacy", + exploration_id=node_or_exploration_id, + exploration_info=node_or_exploration_info, + ) + + return grouped_results @router.post("/", status_code=status.HTTP_201_CREATED) @@ -341,6 +525,9 @@ async def create_pipeline( exploration=None, train=None, prediction=None, + split_data=None, + task_and_model=None, + metrics_result=None, ) db.add(new_pipeline) db.commit() @@ -403,6 +590,9 @@ async def update_pipeline( pipeline.exploration = None pipeline.train = None pipeline.prediction = None + pipeline.split_data = None + pipeline.task_and_model = None + pipeline.metrics_result = None steps_dict = [ step.model_dump() if hasattr(step, "model_dump") else step @@ -530,6 +720,20 @@ async def validate_pipeline( return validator.validate() +@router.post("/validate_edge") +async def validate_edge_connection(params: ValidateEdgeParams) -> Dict[str, Any]: + """Validate a single edge connection between two node ports.""" + result = validate_edge( + source_type=params.source.type, + target_type=params.target.type, + source_port=params.source.port, + target_port=params.target.port, + source_current_outputs=params.source.currentConnections, + target_current_inputs=params.target.currentConnections, + ) + return result + + @router.post("/filter_models") async def filter_models_endpoint( params: DatasetFilterParams, @@ -578,7 +782,7 @@ async def filter_models_endpoint( if params.pipeline_id and pipeline.id == params.pipeline_id: continue - if not pipeline.train: + if not (pipeline.train or pipeline.task_and_model): continue steps = pipeline.steps diff --git a/DashAI/back/api/api_v1/schemas/pipelines_params.py b/DashAI/back/api/api_v1/schemas/pipelines_params.py index 993754754..fdeadd9c5 100644 --- a/DashAI/back/api/api_v1/schemas/pipelines_params.py +++ b/DashAI/back/api/api_v1/schemas/pipelines_params.py @@ -35,3 +35,15 @@ class ValidateNodeParams(BaseModel): class ValidatePipelineParams(BaseModel): nodes: List[Dict[str, Any]] edges: List[Dict[str, Any]] + + +class EdgeEndpointRef(BaseModel): + nodeId: Optional[str] = None # noqa: N815 + type: str + port: Optional[str] = None + currentConnections: Optional[int] = None # noqa: N815 + + +class ValidateEdgeParams(BaseModel): + source: EdgeEndpointRef + target: EdgeEndpointRef diff --git a/DashAI/back/core/enums/status.py b/DashAI/back/core/enums/status.py index 20ec6f22e..b18db52a2 100644 --- a/DashAI/back/core/enums/status.py +++ b/DashAI/back/core/enums/status.py @@ -67,3 +67,19 @@ class DatafileStatus(Enum): DOWNLOADING = "downloading" READY = "ready" ERROR = "error" + + +class PipelineRunStatus(Enum): + NOT_STARTED = 0 + DELIVERED = 1 + STARTED = 2 + FINISHED = 3 + ERROR = 4 + + +class NodeRunStatus(Enum): + NOT_STARTED = 0 + DELIVERED = 1 + STARTED = 2 + FINISHED = 3 + ERROR = 4 diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index 2b808f20f..7eb6b3aba 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -28,6 +28,8 @@ DatasetStatus, ExplainerStatus, ExplorerStatus, + NodeRunStatus, + PipelineRunStatus, PluginStatus, PredictionStatus, ReportStatus, @@ -138,8 +140,6 @@ class ModelSession(Base): task_name: Mapped[str] = mapped_column(String, nullable=False) input_columns: Mapped[str] = mapped_column(JSON, nullable=False) output_columns: Mapped[str] = mapped_column(JSON, nullable=False) - - # Metrics per split train_metrics: Mapped[list[str]] = mapped_column(JSON, nullable=True) validation_metrics: Mapped[list[str]] = mapped_column(JSON, nullable=True) test_metrics: Mapped[list[str]] = mapped_column(JSON, nullable=True) @@ -188,11 +188,9 @@ class Run(Base): default=datetime.now, onupdate=datetime.now, ) - # model and parameters model_name: Mapped[str] = mapped_column(String) parameters: Mapped[JSON] = mapped_column(JSON) split_indexes: Mapped[str] = mapped_column(JSON, nullable=True) - # optimizer optimizer_name: Mapped[str] = mapped_column(String) optimizer_parameters: Mapped[JSON] = mapped_column(JSON) nested: Mapped[JSON] = mapped_column(JSON, nullable=True) @@ -200,11 +198,8 @@ class Run(Base): plot_slice_path: Mapped[str] = mapped_column(String, nullable=True) plot_contour_path: Mapped[str] = mapped_column(String, nullable=True) plot_importance_path: Mapped[str] = mapped_column(String, nullable=True) - # goal metrics goal_metric: Mapped[str] = mapped_column(String) - # artifacts artifacts: Mapped[str] = mapped_column(JSON, nullable=True) - # metadata name: Mapped[str] = mapped_column(String) description: Mapped[str] = mapped_column(String, nullable=True) run_path: Mapped[str] = mapped_column(String, nullable=True) @@ -263,8 +258,6 @@ class Prediction(Base): delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) - - # Relationships run: Mapped["Run"] = relationship("Run", back_populates="predictions") dataset: Mapped["Dataset"] = relationship("Dataset", back_populates="predictions") @@ -499,7 +492,6 @@ class GenerativeProcess(Base): default=datetime.now, onupdate=datetime.now, ) - # metadata session_id: Mapped[int] = mapped_column( ForeignKey("generative_session.id", ondelete="CASCADE") ) @@ -589,12 +581,9 @@ class GenerativeSession(Base): default=datetime.now, onupdate=datetime.now, ) - # task name task_name: Mapped[str] = mapped_column(String, nullable=False) - # model and parameters model_name: Mapped[str] = mapped_column(String) parameters: Mapped[JSON] = mapped_column(JSON) - # metadata name: Mapped[str] = mapped_column(String, unique=True, nullable=False) description: Mapped[str] = mapped_column(String, nullable=True) # Huey id of the RAG indexing job most recently started for this session. @@ -602,7 +591,6 @@ class GenerativeSession(Base): # that job is still alive, so a stale id simply resolves to nothing. index_job_id: Mapped[str] = mapped_column(String, nullable=True) - # Relationship with GenerativeSessionParameterHistory parameters_history: Mapped[List["GenerativeSessionParameterHistory"]] = ( relationship( "GenerativeSessionParameterHistory", @@ -611,7 +599,6 @@ class GenerativeSession(Base): ) ) - # Relationship with GenerativeProcess processes: Mapped[List["GenerativeProcess"]] = relationship( "GenerativeProcess", cascade="all, delete-orphan", back_populates="session" ) @@ -636,6 +623,230 @@ class Pipeline(Base): exploration: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) train: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) prediction: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + split_data: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + task_and_model: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + metrics_result: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + + split_data_nodes: Mapped[List["SplitDataNode"]] = relationship( + "SplitDataNode", cascade="all, delete-orphan", back_populates="pipeline" + ) + task_and_model_nodes: Mapped[List["TaskAndModelNode"]] = relationship( + "TaskAndModelNode", cascade="all, delete-orphan", back_populates="pipeline" + ) + metrics_nodes: Mapped[List["MetricsNode"]] = relationship( + "MetricsNode", cascade="all, delete-orphan", back_populates="pipeline" + ) + pipeline_runs: Mapped[List["PipelineRun"]] = relationship( + "PipelineRun", cascade="all, delete-orphan", back_populates="pipeline" + ) + + +class SplitDataNode(Base): + __tablename__ = "split_data_node" + """ + Table to store configuration and results for the Split Data step + of a pipeline. Configures how the dataset is partitioned into + train / validation / test splits. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_id: Mapped[int] = mapped_column( + ForeignKey("pipeline.id", ondelete="CASCADE"), nullable=False + ) + input_columns: Mapped[List[int]] = mapped_column(JSON, nullable=False) + output_columns: Mapped[List[int]] = mapped_column(JSON, nullable=False) + splits: Mapped[Dict[str, float]] = mapped_column(JSON, nullable=False) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + + # Relationships + pipeline: Mapped["Pipeline"] = relationship( + "Pipeline", back_populates="split_data_nodes" + ) + + +class TaskAndModelNode(Base): + __tablename__ = "task_and_model_node" + """ + Table to store configuration and results for the Task & Model step + of a pipeline. Selects the ML task and model, stores training + parameters, and keeps a reference to the persisted model artifact. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_id: Mapped[int] = mapped_column( + ForeignKey("pipeline.id", ondelete="CASCADE"), nullable=False + ) + task_name: Mapped[str] = mapped_column(String, nullable=False) + model_name: Mapped[str] = mapped_column(String, nullable=False) + parameters: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + model_path: Mapped[str] = mapped_column(String, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + + # Relationships + pipeline: Mapped["Pipeline"] = relationship( + "Pipeline", back_populates="task_and_model_nodes" + ) + + +class MetricsNode(Base): + __tablename__ = "metrics_node" + """ + Table to store configuration and results for the Metrics step + of a pipeline. Holds the list of metric names to compute and, + after execution, the evaluation results. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_id: Mapped[int] = mapped_column( + ForeignKey("pipeline.id", ondelete="CASCADE"), nullable=False + ) + metric_names: Mapped[List[str]] = mapped_column(JSON, nullable=False) + results: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + + # Relationships + pipeline: Mapped["Pipeline"] = relationship( + "Pipeline", back_populates="metrics_nodes" + ) + + +class PipelineRun(Base): + __tablename__ = "pipeline_run" + """ + Table to store execution runs for a pipeline. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_id: Mapped[int] = mapped_column( + ForeignKey("pipeline.id", ondelete="CASCADE"), nullable=False + ) + steps: Mapped[List[Dict[str, Any]]] = mapped_column(JSON, nullable=True) + edges: Mapped[List[Dict[str, Any]]] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + status: Mapped[Enum] = mapped_column( + Enum(PipelineRunStatus), + nullable=False, + default=PipelineRunStatus.NOT_STARTED, + ) + error_message: Mapped[str] = mapped_column(String, nullable=True) + + pipeline: Mapped["Pipeline"] = relationship( + "Pipeline", back_populates="pipeline_runs" + ) + node_runs: Mapped[List["NodeRun"]] = relationship( + "NodeRun", cascade="all, delete-orphan", back_populates="pipeline_run" + ) + + def set_status_as_delivered(self) -> None: + """ + Update the status of the pipeline run to delivered and set delivery_time to now. + """ + self.status = PipelineRunStatus.DELIVERED + self.delivery_time = datetime.now() + + def set_status_as_started(self) -> None: + """ + Update the status of the pipeline run to started and set start_time to now. + """ + self.status = PipelineRunStatus.STARTED + self.start_time = datetime.now() + + def set_status_as_finished(self) -> None: + """Update the status of the pipeline run to finished and set end_time to now.""" + self.status = PipelineRunStatus.FINISHED + self.end_time = datetime.now() + + def set_status_as_error(self, error_message: str | None = None) -> None: + """Update the status of the pipeline run to error.""" + self.status = PipelineRunStatus.ERROR + self.error_message = error_message + self.end_time = datetime.now() + + +class NodeRun(Base): + __tablename__ = "pipeline_node_run" + """ + Table to store execution runs for individual pipeline nodes. + """ + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_run_id: Mapped[int] = mapped_column( + ForeignKey("pipeline_run.id", ondelete="CASCADE"), nullable=False + ) + node_id: Mapped[str] = mapped_column(String, nullable=False) + node_type: Mapped[str] = mapped_column(String, nullable=False) + config: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + input: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + output: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + last_modified: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, onupdate=datetime.now + ) + delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) + status: Mapped[Enum] = mapped_column( + Enum(NodeRunStatus), + nullable=False, + default=NodeRunStatus.NOT_STARTED, + ) + error_message: Mapped[str] = mapped_column(String, nullable=True) + + pipeline_run: Mapped["PipelineRun"] = relationship( + "PipelineRun", back_populates="node_runs" + ) + artifacts: Mapped[List["NodeArtifact"]] = relationship( + "NodeArtifact", cascade="all, delete-orphan", back_populates="node_run" + ) + + def set_status_as_delivered(self) -> None: + """ + Update the status of the node run to delivered and set delivery_time to now. + """ + self.status = NodeRunStatus.DELIVERED + self.delivery_time = datetime.now() + + def set_status_as_started(self) -> None: + """Update the status of the node run to started and set start_time to now.""" + self.status = NodeRunStatus.STARTED + self.start_time = datetime.now() + + def set_status_as_finished(self) -> None: + """Update the status of the node run to finished and set end_time to now.""" + self.status = NodeRunStatus.FINISHED + self.end_time = datetime.now() + + def set_status_as_error(self, error_message: str | None = None) -> None: + """Update the status of the node run to error.""" + self.status = NodeRunStatus.ERROR + self.error_message = error_message + self.end_time = datetime.now() + + +class NodeArtifact(Base): + __tablename__ = "pipeline_node_artifact" + """ + Table to store artifacts emitted by a node run. + """ + id: Mapped[int] = mapped_column(primary_key=True) + node_run_id: Mapped[int] = mapped_column( + ForeignKey("pipeline_node_run.id", ondelete="CASCADE"), nullable=False + ) + key: Mapped[str] = mapped_column(String, nullable=False) + value: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) + + node_run: Mapped["NodeRun"] = relationship("NodeRun", back_populates="artifacts") class Converter(Base): @@ -664,8 +875,6 @@ class Converter(Base): delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) - - # Relationships notebook: Mapped["Notebook"] = relationship(back_populates="converters") def set_status_as_delivered(self) -> None: @@ -712,7 +921,6 @@ class Notebook(Base): file_path: Mapped[str] = mapped_column(String, nullable=False) name: Mapped[str] = mapped_column(String, nullable=True) description: Mapped[str] = mapped_column(String, nullable=True) - # Relationships explorers: Mapped[List["Explorer"]] = relationship( back_populates="notebook", cascade="all, delete-orphan" ) @@ -742,8 +950,6 @@ class GenerativeSessionParameterHistory(Base): DateTime, default=datetime.now, ) - - # Relationship with GenerativeSession session = relationship( "GenerativeSession", back_populates="parameters_history", @@ -767,7 +973,6 @@ class Explorer(Base): default=datetime.now, onupdate=datetime.now, ) - # explorer columns: Mapped[JSON] = mapped_column(JSON, nullable=False) exploration_type: Mapped[str] = mapped_column(String, nullable=False) parameters: Mapped[JSON] = mapped_column(JSON, nullable=False) @@ -781,14 +986,12 @@ class Explorer(Base): plot_overrides: Mapped[JSON] = mapped_column(JSON, nullable=True) # Metadata name: Mapped[str] = mapped_column(String, nullable=True) - delivery_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) start_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) end_time: Mapped[DateTime] = mapped_column(DateTime, nullable=True) status: Mapped[Enum] = mapped_column( Enum(ExplorerStatus), nullable=False, default=ExplorerStatus.NOT_STARTED ) - # Relationships notebook: Mapped["Notebook"] = relationship(back_populates="explorers") def set_status_as_delivered(self) -> None: diff --git a/DashAI/back/dependencies/database/sqlite_database.py b/DashAI/back/dependencies/database/sqlite_database.py index e450a184f..6ce857681 100644 --- a/DashAI/back/dependencies/database/sqlite_database.py +++ b/DashAI/back/dependencies/database/sqlite_database.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Tuple -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker @@ -41,9 +41,18 @@ def setup_sqlite_db(config: Dict[str, str]) -> Tuple[Engine, sessionmaker]: engine: Engine = create_engine( db_url, echo=config["LOGGING_LEVEL"] == logging.DEBUG, - connect_args={"check_same_thread": False}, + connect_args={"check_same_thread": False, "timeout": 30}, + pool_pre_ping=True, ) + @event.listens_for(engine, "connect") + def _set_sqlite_pragma(dbapi_connection, connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL;") + cursor.execute("PRAGMA synchronous=NORMAL;") + cursor.execute("PRAGMA busy_timeout=30000;") + cursor.close() + session_factory = sessionmaker( autocommit=False, autoflush=True, diff --git a/DashAI/back/exploration/base_explorer.py b/DashAI/back/exploration/base_explorer.py index a71379ee9..3ad17e629 100644 --- a/DashAI/back/exploration/base_explorer.py +++ b/DashAI/back/exploration/base_explorer.py @@ -14,6 +14,23 @@ from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset +try: + from _plotly_utils.utils import PlotlyJSONEncoder as _PlotlyJSONEncoder + + if not getattr(_PlotlyJSONEncoder, "_dashai_bytes_patch", False): + _plotly_orig_default = _PlotlyJSONEncoder.default + + def _plotly_default_with_bytes(self, obj): + if isinstance(obj, (bytes, bytearray)): + return obj.decode("utf-8", "replace") + return _plotly_orig_default(self, obj) + + _PlotlyJSONEncoder.default = _plotly_default_with_bytes + _PlotlyJSONEncoder._dashai_bytes_patch = True +except Exception: + pass + + class BaseExplorerSchema(BaseSchema): """ Base schema for explorers, it defines the parameters to be used in each explorer. diff --git a/DashAI/back/exploration/explorers/ecdf_plot.py b/DashAI/back/exploration/explorers/ecdf_plot.py index 68c1c6ee0..90be0d837 100644 --- a/DashAI/back/exploration/explorers/ecdf_plot.py +++ b/DashAI/back/exploration/explorers/ecdf_plot.py @@ -292,21 +292,46 @@ def launch_exploration(self, dataset: "DashAIDataset", explorer_info: Explorer): _df = dataset.to_pandas() columns = [col["columnName"] for col in explorer_info.columns] - fig = px.ecdf( - _df, - x=columns[0] if len(columns) == 1 else columns, - color=self.color_column, - facet_col=self.facet_col, - facet_row=self.facet_row, - ecdfnorm=( - self.ecdf_norm.value if self.ecdf_norm is not ECDFNorm.NONE else None - ), - title=( - f"ECDF Plot of {len(columns)} columns" - if len(columns) > 1 - else f"ECDF Plot of {columns[0]}" - ), - ) + ecdfnorm = self.ecdf_norm.value if self.ecdf_norm is not ECDFNorm.NONE else None + + if len(columns) == 1: + fig = px.ecdf( + _df, + x=columns[0], + color=self.color_column, + facet_col=self.facet_col, + facet_row=self.facet_row, + ecdfnorm=ecdfnorm, + title=f"ECDF Plot of {columns[0]}", + ) + else: + # Reshape wide -> long (tidy data) so that several columns can be + # overlaid in a single ECDF even when their dtypes differ. Plotly + # Express cannot process wide-form data whose columns are of + # different type, and its own documentation recommends long-form for + # this case (Wickham, 2014, "Tidy Data"). Each selected column + # becomes a colored series; an explicit color column, if provided, + # takes precedence over this per-column grouping. + id_vars = [ + col + for col in {self.color_column, self.facet_col, self.facet_row} + if col and col not in columns + ] + long_df = _df.melt( + id_vars=id_vars, + value_vars=columns, + var_name="variable", + value_name="value", + ) + fig = px.ecdf( + long_df, + x="value", + color=self.color_column or "variable", + facet_col=self.facet_col, + facet_row=self.facet_row, + ecdfnorm=ecdfnorm, + title=f"ECDF Plot of {len(columns)} columns", + ) if explorer_info.name is not None and explorer_info.name != "": fig.update_layout(title=f"{explorer_info.name}") diff --git a/DashAI/back/exploration/explorers/multibox_plot.py b/DashAI/back/exploration/explorers/multibox_plot.py index 8a65dc804..27381c0f4 100644 --- a/DashAI/back/exploration/explorers/multibox_plot.py +++ b/DashAI/back/exploration/explorers/multibox_plot.py @@ -268,16 +268,25 @@ def launch_exploration(self, dataset: "DashAIDataset", explorer_info: Explorer): _df = dataset.to_pandas() cols = [col["columnName"] for col in explorer_info.columns] + def _decode(values): + return [ + v.decode("utf-8", "replace") if isinstance(v, (bytes, bytearray)) else v + for v in values + ] + opposite_axis = ( - _df[self.opposite_axis] if self.opposite_axis is not None else None + _decode(_df[self.opposite_axis].tolist()) + if self.opposite_axis is not None + else None ) fig = go.Figure() for col in cols: + col_values = _decode(_df[col].tolist()) fig.add_trace( go.Box( - x=_df[col] if self.horizontal else opposite_axis, - y=opposite_axis if self.horizontal else _df[col], + x=col_values if self.horizontal else opposite_axis, + y=opposite_axis if self.horizontal else col_values, name=col, boxpoints=self.points, ) @@ -326,13 +335,28 @@ def save_notebook( str The path of the saved file as a POSIX string. """ + import json import os from pathlib import Path + import numpy as np + filename = f"{explorer_info.id}.pickle" path = Path(os.path.join(save_path, filename)) - result.write_json(path.as_posix()) + def _default(o): + if isinstance(o, (bytes, bytearray)): + return o.decode("utf-8", "replace") + if isinstance(o, np.ndarray): + return o.tolist() + if isinstance(o, np.generic): + return o.item() + raise TypeError( + f"Object of type {type(o).__name__} is not JSON serializable" + ) + + with open(path.as_posix(), "w", encoding="utf-8") as f: + json.dump(result.to_dict(), f, default=_default) return path.as_posix() diff --git a/DashAI/back/initial_components.py b/DashAI/back/initial_components.py index 67f490b07..67a702407 100644 --- a/DashAI/back/initial_components.py +++ b/DashAI/back/initial_components.py @@ -419,9 +419,11 @@ # Pipeline nodes from DashAI.back.pipeline.data_selector_node import DataSelector from DashAI.back.pipeline.exploration_node import DataExploration +from DashAI.back.pipeline.metrics_node import MetricsEval from DashAI.back.pipeline.prediction_node import Prediction from DashAI.back.pipeline.retrieve_model_node import RetrieveModel -from DashAI.back.pipeline.train_node import Train +from DashAI.back.pipeline.split_data_node import SplitData +from DashAI.back.pipeline.task_and_model_node import TaskAndModel # Plugins from DashAI.back.plugins.utils import get_available_plugins @@ -749,9 +751,11 @@ def get_initial_components(): Nystroem, DataSelector, DataExploration, - Train, RetrieveModel, Prediction, + SplitData, + TaskAndModel, + MetricsEval, SMOTEConverter, SMOTEENNConverter, RandomUnderSamplerConverter, diff --git a/DashAI/back/job/base_job.py b/DashAI/back/job/base_job.py index b69ef9e2d..1b845f21a 100644 --- a/DashAI/back/job/base_job.py +++ b/DashAI/back/job/base_job.py @@ -78,7 +78,7 @@ def get_job_name(self) -> str: raise NotImplementedError @abstractmethod - def run() -> None: + def run(self) -> None: """Run the job.""" raise NotImplementedError diff --git a/DashAI/back/job/dataset_job.py b/DashAI/back/job/dataset_job.py index f6e6d3d54..4fa85257f 100644 --- a/DashAI/back/job/dataset_job.py +++ b/DashAI/back/job/dataset_job.py @@ -292,7 +292,16 @@ def run( n_sample=None, ) else: - # --- File / URL upload path (unchanged) --- + if ( + params.get("dataloader") == "CSVDataLoader" + and "separator" not in params + ): + params["separator"] = "," + if ( + params.get("dataloader") == "JSONDataLoader" + and "data_key" not in params + ): + params["data_key"] = None parsed_params = parse_params(DatasetParams, json.dumps(params)) dataloader = component_registry[parsed_params.dataloader][ "class" diff --git a/DashAI/back/job/node_job.py b/DashAI/back/job/node_job.py new file mode 100644 index 000000000..ed1159da6 --- /dev/null +++ b/DashAI/back/job/node_job.py @@ -0,0 +1,224 @@ +import asyncio +import json +import logging +import threading +import time +from typing import TYPE_CHECKING, Any, Dict + +from kink import inject +from sqlalchemy import exc + +from DashAI.back.dependencies.database.models import NodeRun +from DashAI.back.job.base_job import BaseJob, JobError + +if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + + from DashAI.back.dependencies.registry import ComponentRegistry + + +log = logging.getLogger(__name__) + +_DB_WRITE_LOCK = threading.Lock() + + +class NodeJob(BaseJob): + """Execute a single pipeline node and persist run metadata.""" + + @inject + def set_status_as_delivered( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + node_run_id = self.kwargs.get("node_run_id") + if not node_run_id: + return + + with session_factory() as db: + node_run: NodeRun | None = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError(f"NodeRun with id {node_run_id} not found.") + + try: + node_run.set_status_as_delivered() + self._commit_with_retry(db) + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError( + "Error while setting node run status as delivered." + ) from e + + @inject + def set_status_as_error( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + node_run_id = self.kwargs.get("node_run_id") + if not node_run_id: + return + + with session_factory() as db: + node_run: NodeRun | None = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError(f"NodeRun with id {node_run_id} not found.") + + try: + node_run.set_status_as_error("Node job failed.") + self._commit_with_retry(db) + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Error while setting node run status as error.") from e + + def get_job_name(self) -> str: + node_type = self.kwargs.get("node_type", "Node") + node_id = self.kwargs.get("node_id") + if node_id: + return f"{node_type} ({node_id})" + return str(node_type) + + @inject + def run( + self, + session_factory: "sessionmaker" = lambda di: di["session_factory"], + component_registry: "ComponentRegistry" = lambda di: di["component_registry"], + ) -> Dict[str, Any]: + node_type = self.kwargs.get("node_type") + node_config = self.kwargs.get("node_config", {}) + context = self.kwargs.get("context", {}) + node_run_id = self.kwargs.get("node_run_id") + + if not node_type: + raise JobError("Node type is required to execute a node job.") + if not node_run_id: + try: + node_class = component_registry[node_type]["class"] + except KeyError as e: + raise JobError( + f"Component type {node_type} not found in registry" + ) from e + + try: + node_instance = node_class(**node_config) + except Exception as e: + raise JobError(f"Error instantiating node {node_type}: {str(e)}") from e + + return self._run_node_instance(node_instance, context) + + with session_factory() as db: + node_run: NodeRun | None = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError(f"NodeRun with id {node_run_id} not found.") + + try: + node_run.set_status_as_started() + node_run.input = self._safe_payload(context) + self._commit_with_retry(db) + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Error while setting node run status as started.") from e + + try: + node_class = component_registry[node_type]["class"] + except KeyError as e: + raise JobError(f"Component type {node_type} not found in registry") from e + + try: + node_instance = node_class(**node_config) + except Exception as e: + raise JobError(f"Error instantiating node {node_type}: {str(e)}") from e + + try: + output = self._run_node_instance(node_instance, context) + except Exception as e: + with session_factory() as db: + node_run = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError( + f"NodeRun with id {node_run_id} not found after failure." + ) from e + + try: + node_run.set_status_as_error(str(e)) + self._commit_with_retry(db) + except exc.SQLAlchemyError as db_error: + log.exception(db_error) + raise JobError( + "Error while setting node run status as error." + ) from db_error + + raise + + with session_factory() as db: + node_run = db.get(NodeRun, node_run_id) + if node_run is None: + raise JobError(f"NodeRun with id {node_run_id} not found.") + + try: + node_run.output = self._safe_payload(output) + node_run.set_status_as_finished() + self._commit_with_retry(db) + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError( + "Error while setting node run status as finished." + ) from e + + return output + + @staticmethod + def _run_node_instance( + node_instance: Any, context: Dict[str, Any] + ) -> Dict[str, Any]: + result = node_instance.run(context=context) + if asyncio.iscoroutine(result): + return asyncio.run(result) + return result + + @staticmethod + def _commit_with_retry( + db, + attempts: int = 5, + base_delay: float = 0.1, + readd: list[Any] | None = None, + ) -> None: + readd = readd or [] + with _DB_WRITE_LOCK: + for attempt in range(attempts): + try: + for obj in readd: + if obj is not None: + db.add(obj) + db.commit() + return + except exc.OperationalError as error: + if "database is locked" not in str(error).lower(): + raise + db.rollback() + if attempt >= attempts - 1: + raise + time.sleep(base_delay * (2**attempt)) + + @staticmethod + def _safe_payload(value: Any) -> Any: + if isinstance(value, dict): + sanitized: Dict[str, Any] = {} + for key, item in value.items(): + if key == "dataset": + continue + sanitized_item = NodeJob._safe_payload(item) + if sanitized_item is not None: + sanitized[key] = sanitized_item + return sanitized + if isinstance(value, list): + sanitized_list = [] + for item in value: + sanitized_item = NodeJob._safe_payload(item) + if sanitized_item is not None: + sanitized_list.append(sanitized_item) + return sanitized_list + if isinstance(value, (str, int, float, bool)) or value is None: + return value + + try: + json.dumps(value) + return value + except TypeError: + return str(value) diff --git a/DashAI/back/job/pipeline_job.py b/DashAI/back/job/pipeline_job.py index 1e679b4f5..0b20fa698 100644 --- a/DashAI/back/job/pipeline_job.py +++ b/DashAI/back/job/pipeline_job.py @@ -1,15 +1,15 @@ +import asyncio import logging from typing import TYPE_CHECKING, Any, Dict, List -from kink import di +from kink import di, inject -from DashAI.back.dependencies.database.models import Pipeline +from DashAI.back.dependencies.database.models import NodeRun, Pipeline, PipelineRun from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.job.node_job import NodeJob if TYPE_CHECKING: - from sqlalchemy.orm import Session - - from DashAI.back.dependencies.registry import ComponentRegistry + from sqlalchemy.orm import sessionmaker log = logging.getLogger(__name__) @@ -20,68 +20,295 @@ class PipelineJob(BaseJob): ) def set_status_as_delivered(self) -> None: - pass + """Pipeline job delivered. No persistent status to update.""" + log.debug("PipelineJob marked as delivered.") + + def set_status_as_error(self) -> None: + """Pipeline job errored. No persistent status to update.""" + log.debug("PipelineJob marked as error.") + + def get_job_name(self) -> str: + """Return a descriptive name for the pipeline job.""" + pipeline_id = self.kwargs.get("id") + if not pipeline_id: + return "Pipeline" + + try: + session_factory: "sessionmaker" = di["session_factory"] + with session_factory() as db: + pipeline: Pipeline | None = db.get(Pipeline, pipeline_id) + if pipeline and getattr(pipeline, "name", None): + return f"Pipeline: {pipeline.name}" + except Exception: + pass - async def run( + return f"Pipeline ({pipeline_id})" + + @inject + def run( self, - component_registry: "ComponentRegistry" = lambda di: di["component_registry"], + session_factory: "sessionmaker" = lambda di: di["session_factory"], ) -> None: - db: "Session" = self.kwargs["db"] - id: int = self.kwargs.get("id", None) - pipeline: Pipeline = db.get(Pipeline, id) - steps: List[Dict[str, Any]] = self.kwargs.get("steps", []) or pipeline.steps - - if not id: + pipeline_id: int | None = self.kwargs.get("id") + if not pipeline_id: raise JobError("No id provided to execute the pipeline.") - if not steps: - raise JobError("No steps provided to execute the pipeline.") - - if not steps: - raise JobError("Pipeline has no steps to execute") - log.info(f"Starting pipeline execution for pipeline {id}...") + with session_factory() as db: + pipeline: Pipeline | None = db.get(Pipeline, pipeline_id) + if pipeline is None: + raise JobError(f"Pipeline with id {pipeline_id} not found.") - context: Dict[str, Any] = {"pipeline_id": id} + steps: List[Dict[str, Any]] = self.kwargs.get("steps", []) or ( + pipeline.steps or [] + ) + edges: List[Dict[str, Any]] = self.kwargs.get("edges", []) or ( + pipeline.edges or [] + ) + if not steps: + raise JobError("No steps provided to execute the pipeline.") - for idx, step in enumerate(steps): - node_id = step.get("id") - node_type = step.get("type") - node_config = step.get("config", {}) + pipeline.exploration = None + pipeline.train = None + pipeline.prediction = None + pipeline.split_data = None + pipeline.task_and_model = None + pipeline.metrics_result = None - log.debug( - f"Pipeline {id}: Executing step {idx + 1}/{len(steps)} - " - f"{node_type} ({node_id})" + pipeline_run = PipelineRun( + pipeline_id=pipeline_id, + steps=steps, + edges=edges, ) + pipeline_run.set_status_as_started() + db.add(pipeline_run) + NodeJob._commit_with_retry(db, readd=[pipeline_run]) + db.refresh(pipeline_run) - try: - node_class = component_registry(di)[node_type]["class"] - except KeyError as e: - error_msg = f"Component type {node_type} not found in registry" - raise JobError( - f"Error in node {node_id} ({node_type}): {error_msg}" - ) from e + log.info(f"Starting pipeline execution for pipeline {pipeline_id}...") + base_context: Dict[str, Any] = {"pipeline_id": pipeline_id} + node_contexts: Dict[str, Dict[str, Any]] = {} + predecessor_map = self._build_predecessor_map(steps, edges) + successor_map = self._build_successor_map(steps, edges) + step_map = {step.get("id"): step for step in steps if step.get("id")} + + async def _execute_graph() -> None: + pending = set(step_map.keys()) + if not pending: + raise JobError("Pipeline steps are missing ids.") + + ready = [ + node_id for node_id in pending if not predecessor_map.get(node_id) + ] + if not ready: + raise JobError( + "No starting nodes found. Check for cycles or missing roots." + ) + + task_map: Dict[asyncio.Task, str] = {} + update_lock = asyncio.Lock() + exclusive_lock = asyncio.Lock() + exclusive_types = {"TaskAndModel"} + + async def _run_node(node_id: str) -> None: + step = step_map.get(node_id, {}) + node_type = step.get("type") + node_config = step.get("config", {}) + node_context = self._build_node_context( + node_id=node_id, + predecessor_map=predecessor_map, + node_contexts=node_contexts, + base_context=base_context, + ) + node_context["_node_id"] = node_id + + async with update_lock: + node_run = self._persist_node_run( + db, pipeline_run.id, node_id, node_type, node_config + ) + + node_job = NodeJob( + node_run_id=None, + node_id=node_id, + node_type=node_type, + node_config=node_config, + context=node_context, + ) + + try: + if node_type in exclusive_types: + async with exclusive_lock: + output = await asyncio.to_thread(node_job.run) + else: + output = await asyncio.to_thread(node_job.run) + except Exception: + async with update_lock: + self._set_node_run_status( + db, node_run, "error", "Node execution failed." + ) + raise + + async with update_lock: + self._update_context( + node_context, pipeline, node_type, node_id, output + ) + node_contexts[node_id] = node_context + self._set_node_run_status(db, node_run, "finished") + + async def _schedule(node_id: str) -> None: + task = asyncio.create_task(_run_node(node_id)) + task_map[task] = node_id + + for node_id in ready: + await _schedule(node_id) + + while task_map: + done, _ = await asyncio.wait( + task_map.keys(), return_when=asyncio.FIRST_COMPLETED + ) + + for task in done: + node_id = task_map.pop(task) + pending.discard(node_id) + + try: + task.result() + except Exception as e: + for pending_task in task_map: + pending_task.cancel() + raise JobError( + f"Error executing node {node_id}: {str(e)}" + ) from e + + for successor in successor_map.get(node_id, []): + if successor not in pending: + continue + predecessors = predecessor_map.get(successor, []) + if all(pred in node_contexts for pred in predecessors): + await _schedule(successor) + + if pending: + raise JobError( + "Pipeline execution stalled: unresolved dependencies remain." + ) try: - node_instance = node_class(**node_config) + asyncio.run(_execute_graph()) + pipeline_run.set_status_as_finished() + db.add(pipeline) + db.add(pipeline_run) + NodeJob._commit_with_retry(db, readd=[pipeline_run, pipeline]) + self.set_status_as_delivered() + log.info(f"Pipeline {pipeline_id} execution completed successfully.") except Exception as e: - error_msg = f"Error in node {node_id} ({node_type}): {str(e)}" - log.exception(error_msg) - raise JobError(error_msg) from e + pipeline_run.set_status_as_error(str(e)) + db.add(pipeline_run) + NodeJob._commit_with_retry(db, readd=[pipeline_run]) + raise - try: - output = await node_instance.run(context=context) - self._update_context(context, pipeline, node_type, node_id, output) - log.debug(f"Node {node_id} executed successfully.") + def _persist_node_run(self, db, pipeline_run_id, node_id, node_type, node_config): + """Create a NodeRun (STARTED) on the orchestrator's session. - except Exception as e: - error_msg = f"Error in node {node_id} ({node_type}): {str(e)}" - log.exception(error_msg) - raise JobError(error_msg) from e + Returns the NodeRun, or None if it could not be persisted (the node + still runs, just without a status to color). + """ + try: + node_run = NodeRun( + pipeline_run_id=pipeline_run_id, + node_id=node_id, + node_type=node_type or "Unknown", + config=node_config, + ) + node_run.set_status_as_started() + db.add(node_run) + NodeJob._commit_with_retry(db, readd=[node_run]) + return node_run + except Exception as e: # noqa: BLE001 + log.warning("Could not persist NodeRun for %s: %s", node_id, e) + return None + + def _set_node_run_status(self, db, node_run, status, error_message=None): + """Update a NodeRun's status on the orchestrator's session.""" + if node_run is None: + return + try: + if status == "finished": + node_run.set_status_as_finished() + elif status == "error": + node_run.set_status_as_error(error_message or "Node execution failed.") + else: + node_run.set_status_as_started() + NodeJob._commit_with_retry(db, readd=[node_run]) + except Exception as e: # noqa: BLE001 + log.warning( + "Could not update NodeRun status (%s) for %s: %s", + status, + getattr(node_run, "node_id", "?"), + e, + ) + + def _build_predecessor_map( + self, + steps: List[Dict[str, Any]], + edges: List[Dict[str, Any]], + ) -> Dict[str, List[str]]: + """Build a mapping target_node_id -> [source_node_ids].""" + step_ids = {step.get("id") for step in steps if step.get("id")} + predecessor_map: Dict[str, List[str]] = {step_id: [] for step_id in step_ids} + + for edge in edges: + source = edge.get("source") + target = edge.get("target") + if source in step_ids and target in step_ids: + predecessor_map[target].append(source) + + return predecessor_map + + def _build_successor_map( + self, + steps: List[Dict[str, Any]], + edges: List[Dict[str, Any]], + ) -> Dict[str, List[str]]: + """Build a mapping source_node_id -> [target_node_ids].""" + step_ids = {step.get("id") for step in steps if step.get("id")} + successor_map: Dict[str, List[str]] = {step_id: [] for step_id in step_ids} + + for edge in edges: + source = edge.get("source") + target = edge.get("target") + if source in step_ids and target in step_ids: + successor_map[source].append(target) + + return successor_map + + def _build_node_context( + self, + node_id: str, + predecessor_map: Dict[str, List[str]], + node_contexts: Dict[str, Dict[str, Any]], + base_context: Dict[str, Any], + ) -> Dict[str, Any]: + """Build execution context for a node from all predecessor branches.""" + if not node_id: + raise JobError("Pipeline step is missing an id.") + + predecessors = predecessor_map.get(node_id, []) + if not predecessors: + return dict(base_context) + + merged_context: Dict[str, Any] = dict(base_context) + branches: List[Dict[str, Any]] = [] + for predecessor_id in predecessors: + predecessor_context = node_contexts.get(predecessor_id) + if predecessor_context is None: + raise JobError( + f"Cannot resolve predecessor context for node {node_id} " + f"from source {predecessor_id}." + ) + branches.append(dict(predecessor_context)) + merged_context.update(predecessor_context) - log.info(f"Pipeline {id} execution completed successfully.") - db.add(pipeline) - db.commit() - self.set_status_as_delivered() + merged_context["_branches"] = branches + return merged_context def _update_context( self, @@ -103,16 +330,104 @@ def _update_context( """ if node_type == "DataSelector": context["dataset"] = output.get("dataset") + context["dataset_name"] = output.get( + "dataset_name", context.get("dataset_name") + ) elif node_type == "DataExploration": - context["exploration"] = output.get("exploration") - pipeline.exploration = context["exploration"] - elif node_type == "Train": - context["train"] = output.get("train") - pipeline.train = context["train"] + current_node_exploration = output.get("exploration") or {} + context["exploration"] = current_node_exploration + + aggregated_exploration = ( + dict(pipeline.exploration) + if isinstance(pipeline.exploration, dict) + else {} + ) + exploration_key = ( + node_id or f"DataExploration-{len(aggregated_exploration)}" + ) + aggregated_exploration[exploration_key] = { + "dataset_name": context.get("dataset_name") + or output.get("dataset_name") + or "Unknown Dataset", + "explorations": current_node_exploration, + } + pipeline.exploration = aggregated_exploration + elif node_type == "SplitData": + context["split_data"] = output.get("split_data") + pipeline.split_data = context["split_data"] + elif node_type == "TaskAndModel": + current_task_and_model = output.get("task_and_model") or {} + context["task_and_model"] = current_task_and_model + context["model_node_id"] = node_id + + aggregated_task_and_model = ( + dict(pipeline.task_and_model) + if isinstance(pipeline.task_and_model, dict) + else {} + ) + task_model_key = node_id or f"TaskAndModel-{len(aggregated_task_and_model)}" + aggregated_task_and_model[task_model_key] = { + "node_id": node_id, + "model_node_id": node_id, + "dataset_name": context.get("dataset_name"), + **current_task_and_model, + } + pipeline.task_and_model = aggregated_task_and_model + elif node_type == "MetricsEval": + current_metrics_result = output.get("metrics_result") or {} + context["metrics_result"] = current_metrics_result + + aggregated_metrics_result = ( + dict(pipeline.metrics_result) + if isinstance(pipeline.metrics_result, dict) + else {} + ) + if isinstance(current_metrics_result, list): + for idx, entry in enumerate(current_metrics_result): + metrics_key = entry.get("model_node_id") or f"{node_id}-{idx}" + aggregated_metrics_result[metrics_key] = { + "node_id": node_id, + **entry, + } + else: + metrics_key = node_id or f"MetricsEval-{len(aggregated_metrics_result)}" + aggregated_metrics_result[metrics_key] = { + "node_id": node_id, + "model_node_id": context.get("model_node_id"), + "dataset_name": context.get("dataset_name"), + "model_name": context.get("model_name"), + "task_name": context.get("task_name"), + **current_metrics_result, + } + pipeline.metrics_result = aggregated_metrics_result elif node_type == "RetrieveModel": context["retrieve"] = output.get("retrieve") elif node_type == "Prediction": - context["prediction"] = output.get("prediction") - pipeline.prediction = context["prediction"] + current_prediction = output.get("prediction") + context["prediction"] = current_prediction + + aggregated_prediction = ( + dict(pipeline.prediction) + if isinstance(pipeline.prediction, dict) + else {} + ) + if isinstance(current_prediction, list): + for idx, entry in enumerate(current_prediction): + prediction_key = entry.get("model_node_id") or f"{node_id}-{idx}" + aggregated_prediction[prediction_key] = { + "node_id": node_id, + **entry, + } + else: + prediction_key = node_id or f"Prediction-{len(aggregated_prediction)}" + aggregated_prediction[prediction_key] = { + "node_id": node_id, + "model_node_id": context.get("model_node_id"), + "dataset_name": context.get("dataset_name"), + "model_name": context.get("model_name"), + "task_name": context.get("task_name"), + "prediction": current_prediction, + } + pipeline.prediction = aggregated_prediction else: context[node_id] = output diff --git a/DashAI/back/models/model_factory.py b/DashAI/back/models/model_factory.py index 8385d9eff..7829ba66a 100644 --- a/DashAI/back/models/model_factory.py +++ b/DashAI/back/models/model_factory.py @@ -1,5 +1,6 @@ from collections import namedtuple +import numpy as np from kink import di from DashAI.back.core.schema_fields.search_space import SEARCH_DTYPE_KEY @@ -329,10 +330,14 @@ def evaluate(self, x, y, metrics): results[split] = split_results continue predictions = self.model.predict(x[split]) - if hasattr(self.model, "prepare_output"): - transformed_y = self.model.prepare_output(y[split]) + # Handle both numpy arrays and DashAIDataset objects + if isinstance(y[split], np.ndarray): + transformed_y = y[split] else: - transformed_y = self.model.prepare_dataset(y[split]) + if hasattr(self.model, "prepare_output"): + transformed_y = self.model.prepare_output(y[split]) + else: + transformed_y = self.model.prepare_dataset(y[split]) for metric in metrics: if ( isinstance(metric, type) diff --git a/DashAI/back/pipeline/contracts.py b/DashAI/back/pipeline/contracts.py new file mode 100644 index 000000000..8a6dcaabd --- /dev/null +++ b/DashAI/back/pipeline/contracts.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, List, Literal, Optional + +from pydantic import BaseModel, Field + +if TYPE_CHECKING: + from DashAI.back.pipeline.validator.nodes_definitions import NodeDefinition + +DataKind = Literal[ + "DatasetRef", + "DatasetSplit", + "ModelArtifact", + "MetricsResult", + "PredictionArtifact", + "ExplorationResult", + "Any", +] + + +class PortSpec(BaseModel): + name: str + kind: DataKind + accepts: Optional[List[DataKind]] = None + required: bool = True + max_connections: Optional[int] = 1 + description: Optional[str] = None + + +class NodeContract(BaseModel): + type: str + name: str + icon: str + description: str + requiresConfiguration: bool # noqa: N815 + source: bool + target: bool + predecessors: List[str] = Field(default_factory=list) + successors: List[str] = Field(default_factory=list) + input: Optional[str] = None + output: Optional[str] = None + sourceHandles: int = 1 # noqa: N815 + maxInputs: Optional[int] = 1 # noqa: N815 + maxOutputs: Optional[int] = None # noqa: N815 + configType: str = "custom" # noqa: N815 + inputs: List[PortSpec] + outputs: List[PortSpec] + + +NODE_CONTRACTS: List[NodeContract] = [ + NodeContract( + type="DataSelector", + name="Data Selector", + icon="FolderIcon", + description="Loads a dataset stored in the system to be used in the pipeline.", + requiresConfiguration=True, + source=True, + target=False, + output="Dataset object", + sourceHandles=2, + maxOutputs=2, + inputs=[], + outputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + max_connections=2, + description="Reference to the dataset loaded from storage.", + ) + ], + ), + NodeContract( + type="DataExploration", + name="Data Exploration", + icon="InsertChartIcon", + description=( + "Visualize and analyze your dataset to gain insights into its structure " + "and content." + ), + requiresConfiguration=True, + source=False, + target=True, + input="Dataset object", + inputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + description="Dataset used for exploration.", + ) + ], + outputs=[ + PortSpec( + name="exploration", + kind="ExplorationResult", + description="Exploration outputs and charts.", + ) + ], + ), + NodeContract( + type="SplitData", + name="Split Data", + icon="CallSplitIcon", + description=( + "Configures the partition of the dataset into train / validation / test " + "splits." + ), + requiresConfiguration=True, + source=True, + target=True, + input="Dataset object", + output="Split dataset subsets", + sourceHandles=2, + maxOutputs=2, + inputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + description="Dataset to split.", + ) + ], + outputs=[ + PortSpec( + name="split", + kind="DatasetSplit", + max_connections=2, + description="Train/validation/test splits.", + ) + ], + ), + NodeContract( + type="TaskAndModel", + name="Task & Model", + icon="ModelTrainingIcon", + description=( + "Selects the ML task and model, configures hyper-parameters, and trains " + "the model on the split data." + ), + requiresConfiguration=True, + source=True, + target=True, + input="Split dataset subsets", + output="Trained model object", + sourceHandles=2, + maxOutputs=2, + inputs=[ + PortSpec( + name="split", + kind="DatasetSplit", + description="Split dataset used for training.", + ) + ], + outputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + max_connections=2, + description="Trained model artifact.", + ) + ], + ), + NodeContract( + type="MetricsEval", + name="Metrics", + icon="AssessmentIcon", + description="Evaluates the trained model using the selected metrics.", + requiresConfiguration=True, + source=True, + target=True, + input="Trained model and split data", + output="Model artifact (metrics stored in pipeline results)", + maxInputs=2, + inputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + max_connections=2, + description="Model to evaluate.", + ) + ], + outputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + description="Model artifact passthrough for downstream nodes.", + ) + ], + ), + NodeContract( + type="Prediction", + name="Prediction", + icon="EmojiObjectsIcon", + description="Generates predictions using a trained model.", + requiresConfiguration=False, + source=False, + target=True, + input="Trained model and dataset object", + inputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + description="Model used for predictions.", + ) + ], + outputs=[ + PortSpec( + name="prediction", + kind="PredictionArtifact", + description="Prediction output artifact.", + ) + ], + ), + NodeContract( + type="RetrieveModel", + name="Retrieve Model", + icon="ManageHistoryIcon", + description="Loads a trained model from saved pipelines.", + requiresConfiguration=True, + source=True, + target=True, + input="Dataset object", + output="Trained model object", + inputs=[ + PortSpec( + name="dataset", + kind="DatasetRef", + description="Dataset used for compatibility checks.", + ) + ], + outputs=[ + PortSpec( + name="model", + kind="ModelArtifact", + description="Loaded model artifact.", + ) + ], + ), +] + +NODE_CONTRACTS_MAP: Dict[str, NodeContract] = { + contract.type: contract for contract in NODE_CONTRACTS +} + + +def _derive_compatibility_from_ports() -> None: + """Populate predecessors/successors from the typed ports. + + Compatibility is driven purely by the port data kinds: node B is a valid + successor of node A when any output kind of A is accepted by some input of + B. This way, adding a new node only requires declaring its input/output + kinds -- there is no need to edit every other node's contract. + """ + outputs_by_type = {c.type: {p.kind for p in c.outputs} for c in NODE_CONTRACTS} + accepts_by_type = { + c.type: {kind for port in c.inputs for kind in (port.accepts or [port.kind])} + for c in NODE_CONTRACTS + } + for src in NODE_CONTRACTS: + for tgt in NODE_CONTRACTS: + if src.type == tgt.type: + continue + if outputs_by_type[src.type] & accepts_by_type[tgt.type]: + if tgt.type not in src.successors: + src.successors.append(tgt.type) + if src.type not in tgt.predecessors: + tgt.predecessors.append(src.type) + + +_derive_compatibility_from_ports() + + +def get_contracts_payload() -> List[Dict[str, object]]: + payload = [contract.model_dump() for contract in NODE_CONTRACTS] + type_to_name = {contract.type: contract.name for contract in NODE_CONTRACTS} + for entry in payload: + entry["next"] = [type_to_name.get(t, t) for t in entry.get("successors", [])] + return payload + + +def _resolve_port( + ports: List[PortSpec], port_name: Optional[str], role: str +) -> PortSpec: + if port_name: + for port in ports: + if port.name == port_name: + return port + if len(ports) == 1: + return ports[0] + raise ValueError(f"Unknown {role} port: {port_name}") + if len(ports) == 1: + return ports[0] + raise ValueError(f"Ambiguous {role} port: specify a port name") + + +def validate_edge( + source_type: str, + target_type: str, + source_port: Optional[str] = None, + target_port: Optional[str] = None, + source_current_outputs: Optional[int] = None, + target_current_inputs: Optional[int] = None, +) -> Dict[str, object]: + source_contract = NODE_CONTRACTS_MAP.get(source_type) + target_contract = NODE_CONTRACTS_MAP.get(target_type) + + if source_contract is None: + return _error_response( + "Unknown source node type.", + reason="UNKNOWN_SOURCE", + ) + if target_contract is None: + return _error_response( + "Unknown target node type.", + reason="UNKNOWN_TARGET", + ) + + if target_contract.predecessors and source_type not in target_contract.predecessors: + return _error_response( + f"{target_contract.name} does not accept {source_contract.name} as input.", + reason="INVALID_PREDECESSOR", + ) + if source_contract.successors and target_type not in source_contract.successors: + return _error_response( + f"{source_contract.name} cannot connect to {target_contract.name}.", + reason="INVALID_SUCCESSOR", + ) + + try: + target_port_spec = _resolve_port( + target_contract.inputs, target_port, role="target" + ) + if source_port: + source_port_spec = _resolve_port( + source_contract.outputs, source_port, role="source" + ) + else: + accepts = target_port_spec.accepts or [target_port_spec.kind] + compatible = [ + port for port in source_contract.outputs if port.kind in accepts + ] + if len(compatible) == 1: + source_port_spec = compatible[0] + else: + source_port_spec = _resolve_port( + source_contract.outputs, source_port, role="source" + ) + except ValueError as exc: + return _error_response(str(exc), reason="INVALID_PORT") + + if ( + target_port_spec.max_connections is not None + and target_current_inputs is not None + and target_current_inputs >= target_port_spec.max_connections + ): + return _error_response( + f"{target_contract.name} does not allow more connections on this input.", + reason="MAX_INPUTS_EXCEEDED", + severity="warning", + edge_color="#F57C00", + edge_class="edge-warning", + ) + + if ( + source_current_outputs is not None + and source_port_spec.max_connections is not None + and source_current_outputs >= source_port_spec.max_connections + ): + return _error_response( + f"{source_contract.name} does not allow more outputs on this port.", + reason="MAX_OUTPUTS_EXCEEDED", + severity="warning", + edge_color="#F57C00", + edge_class="edge-warning", + ) + + accepts = target_port_spec.accepts or [target_port_spec.kind] + if source_port_spec.kind not in accepts: + return _error_response( + ( + f"{source_contract.name} outputs {source_port_spec.kind}, but " + f"{target_contract.name} expects {', '.join(accepts)}." + ), + reason="TYPE_MISMATCH", + ) + + return { + "ok": True, + "severity": "info", + "message": "Compatible connection.", + "reason": None, + "style": { + "edgeColor": "#4CAF50", + "edgeClass": "edge-ok", + }, + } + + +def _error_response( + message: str, + reason: str, + severity: str = "error", + edge_color: str = "#D32F2F", + edge_class: str = "edge-error", +) -> Dict[str, object]: + return { + "ok": False, + "severity": severity, + "message": message, + "reason": reason, + "style": { + "edgeColor": edge_color, + "edgeClass": edge_class, + }, + } + + +def contracts_to_node_definitions() -> List["NodeDefinition"]: + from DashAI.back.pipeline.validator.nodes_definitions import NodeDefinition + + definitions: List[NodeDefinition] = [] + for contract in NODE_CONTRACTS: + definitions.append( + NodeDefinition( + type=contract.type, + name=contract.name, + icon=contract.icon, + requiresConfiguration=contract.requiresConfiguration, + source=contract.source, + target=contract.target, + predecessors=contract.predecessors, + successors=contract.successors, + description=contract.description, + input=contract.input, + output=contract.output, + sourceHandles=contract.sourceHandles, + maxInputs=contract.maxInputs, + maxOutputs=contract.maxOutputs, + configType=contract.configType, + ) + ) + return definitions diff --git a/DashAI/back/pipeline/data_selector_node.py b/DashAI/back/pipeline/data_selector_node.py index e3e5d7e28..78750b257 100644 --- a/DashAI/back/pipeline/data_selector_node.py +++ b/DashAI/back/pipeline/data_selector_node.py @@ -34,13 +34,20 @@ def __init__(self, **kwargs) -> None: def set_status_as_delivered(self) -> None: log.debug("DataSelector executed successfully.") + def set_status_as_error(self) -> None: + log.error("DataSelector encountered an error.") + + def get_job_name(self) -> str: + return "DataSelector" + @inject async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: from pathlib import Path from DashAI.back.dataloaders.classes.dashai_dataset import load_dataset - context["dataset_name"] = self.kwargs["name"] + dataset_name = self.kwargs["name"] + context["dataset_name"] = dataset_name dataset_dir = Path(self.kwargs["file_path"]) data_path = dataset_dir / "dataset/data.arrow" @@ -57,4 +64,5 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: return { "dataset": loaded_dataset, + "dataset_name": dataset_name, } diff --git a/DashAI/back/pipeline/exploration_node.py b/DashAI/back/pipeline/exploration_node.py index a767a54bf..e8d6a14c3 100644 --- a/DashAI/back/pipeline/exploration_node.py +++ b/DashAI/back/pipeline/exploration_node.py @@ -1,10 +1,9 @@ import logging from typing import TYPE_CHECKING, Any, Dict, Type -from kink import di, inject +from kink import inject from DashAI.back.config import DefaultSettings -from DashAI.back.dependencies.database.models import Explorer from DashAI.back.exploration.base_explorer import BaseExplorer from DashAI.back.job.base_job import BaseJob, JobError @@ -44,6 +43,12 @@ def __init__( def set_status_as_delivered(self) -> None: log.debug("DataExploration executed successfully.") + def set_status_as_error(self) -> None: + log.error("DataExploration encountered an error.") + + def get_job_name(self) -> str: + return "DataExploration" + @inject async def run( self, @@ -51,16 +56,22 @@ async def run( component_registry: "ComponentRegistry" = lambda di: di["component_registry"], ) -> Dict[str, Any]: try: + import inspect import os from pathlib import Path pipeline_id = context.get("pipeline_id") + node_id = context.get("_node_id", "DataExploration") loaded_dataset = context.get("dataset") settings = DefaultSettings() sqlite_local = os.path.expanduser(settings.LOCAL_PATH) base_path = ( - Path(sqlite_local) / "pipelines" / "exploration" / str(pipeline_id) + Path(sqlite_local) + / "pipelines" + / "exploration" + / str(pipeline_id) + / str(node_id) ) base_path.mkdir(parents=True, exist_ok=True) @@ -71,12 +82,12 @@ async def run( exploration_type = option["exploration_type"] parameters = option.get("parameters", {}) columns = option.get("columns", []) - id = option.get("id", str(idx)) - name = option.get("name") + explorer_id = option.get("id", str(idx)) + name = option.get("name", "") - explorer_component_class: Type[BaseExplorer] = component_registry( - di - )[exploration_type]["class"] + explorer_component_class: Type[BaseExplorer] = component_registry[ + exploration_type + ]["class"] explorer_instance = explorer_component_class(**parameters) assert isinstance(explorer_instance, BaseExplorer) @@ -84,25 +95,66 @@ async def run( loaded_dataset, columns ) - explorer_info = Explorer( + # Create a simple explorer info object for pipeline execution + # This is not a database object, + # just a container for explorer metadata + class ExplorerInfo: + def __init__( + self, exploration_type, columns, parameters, id, name + ): + self.exploration_type = exploration_type + self.columns = columns + self.parameters = parameters + self.id = id + self.name = name + + explorer_info = ExplorerInfo( exploration_type=exploration_type, columns=columns, parameters=parameters, - id=id, + id=explorer_id, + name=name, ) result = explorer_instance.launch_exploration( prepared_dataset, explorer_info ) - save_path = explorer_instance.save_notebook( - __notebook_info__=None, - explorer_info=explorer_info, - save_path=base_path, - result=result, + save_notebook_sig = inspect.signature( + explorer_instance.save_notebook ) - - results[str(id)] = { + save_notebook_params = set(save_notebook_sig.parameters) + + notebook_info_arg_name = None + for candidate in ( + "__notebook_info__", + "__exploration_info__", + "notebook_info", + ): + if candidate in save_notebook_params: + notebook_info_arg_name = candidate + break + + save_notebook_kwargs = { + "explorer_info": explorer_info, + "save_path": base_path, + "result": result, + } + if notebook_info_arg_name is not None: + save_notebook_kwargs[notebook_info_arg_name] = None + save_path = explorer_instance.save_notebook( + **save_notebook_kwargs + ) + else: + # Fallback to positional first arg for legacy implementations. + save_path = explorer_instance.save_notebook( + None, + explorer_info=explorer_info, + save_path=base_path, + result=result, + ) + + results[str(explorer_id)] = { "exploration_type": exploration_type, "path": str(save_path), "parameters": parameters, diff --git a/DashAI/back/pipeline/metrics_node.py b/DashAI/back/pipeline/metrics_node.py new file mode 100644 index 000000000..a457b5157 --- /dev/null +++ b/DashAI/back/pipeline/metrics_node.py @@ -0,0 +1,132 @@ +import logging +from typing import Any, Dict, List + +from kink import di + +from DashAI.back.dependencies.registry import ComponentRegistry +from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.metrics.base_metric import BaseMetric + +log = logging.getLogger(__name__) + + +class MetricsEval(BaseJob): + """Pipeline node that evaluates the trained model with selected metrics. + + It expects both the split data **and** the trained model factory to + already be available in the pipeline context (populated by the + *SplitData* and *TaskAndModel* nodes respectively). + + Parameters + ---------- + metrics : List[str] + Names of the metrics to compute (must be registered in the + component registry). + """ + + def __init__(self, metrics: List[str]) -> None: + super().__init__(kwargs={"metrics": metrics}) + self.metrics = metrics + + def set_status_as_delivered(self) -> None: + log.debug("MetricsEval executed successfully.") + + def set_status_as_error(self) -> None: + log.error("MetricsEval encountered an error.") + + def get_job_name(self) -> str: + return "MetricsEval" + + async def run( + self, + context: Dict[str, Any], + component_registry: ComponentRegistry = lambda di: di["component_registry"], + ) -> Dict[str, Any]: + """Compute metrics for the trained model. + + Reads from ``context``: + * ``trained_x``, ``trained_y`` – from *TaskAndModel* + * ``model_factory`` – ``ModelFactory`` instance from *TaskAndModel* + + Returns + ------- + dict + ``{"metrics_result": { ... }}`` with per-metric scores. + """ + all_metrics = { + component_dict["name"]: component_dict + for component_dict in component_registry(di).get_components_by_types( + select="Metric" + ) + } + + metric_classes: List[BaseMetric] = [] + for metric_name in self.metrics: + entry = all_metrics.get(metric_name) + if entry and entry.get("class"): + metric_classes.append(entry["class"]) + else: + log.warning(f"Metric '{metric_name}' not found in registry.") + + if not metric_classes: + raise JobError("No valid metrics found to evaluate.") + + branches = context.get("_branches") + if branches: + entries: List[Dict[str, Any]] = [] + for branch in branches: + factory = branch.get("model_factory") + x = branch.get("trained_x") + y = branch.get("trained_y") + + if factory is None or x is None or y is None: + log.warning("Skipping metrics for branch without a trained model.") + continue + + try: + model_metrics = factory.evaluate(x, y, metric_classes) + except Exception as e: + log.exception(e) + raise JobError("Metrics calculation failed") from e + + entries.append( + { + "model_node_id": branch.get("model_node_id"), + "dataset_name": branch.get("dataset_name"), + "model_name": branch.get("model_name"), + "task_name": branch.get("task_name"), + "metric_names": self.metrics, + "results": model_metrics, + } + ) + + if not entries: + raise JobError( + "MetricsEval requires a trained model. " + "Make sure TaskAndModel runs before this node." + ) + + return {"metrics_result": entries} + + factory = context.get("model_factory") + x = context.get("trained_x") + y = context.get("trained_y") + + if factory is None or x is None or y is None: + raise JobError( + "MetricsEval requires a trained model. " + "Make sure TaskAndModel runs before this node." + ) + + try: + model_metrics = factory.evaluate(x, y, metric_classes) + except Exception as e: + log.exception(e) + raise JobError("Metrics calculation failed") from e + + return { + "metrics_result": { + "metric_names": self.metrics, + "results": model_metrics, + }, + } diff --git a/DashAI/back/pipeline/prediction_node.py b/DashAI/back/pipeline/prediction_node.py index 18d00a839..028ab8a46 100644 --- a/DashAI/back/pipeline/prediction_node.py +++ b/DashAI/back/pipeline/prediction_node.py @@ -28,6 +28,12 @@ def __init__(self, **kwargs) -> None: def set_status_as_delivered(self) -> None: log.debug("Prediction executed successfully.") + def set_status_as_error(self) -> None: + log.error("Prediction encountered an error.") + + def get_job_name(self) -> str: + return "Prediction" + @inject async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: import json @@ -35,34 +41,10 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: import numpy as np - loaded_dataset = context["dataset"] - model = context["model_class"] - model_path = context["model_path"] - trained_model: BaseModel = model.load(model_path) config = di["config"] + sqlite_local = config["LOCAL_PATH"] try: - prepared_dataset = loaded_dataset.select_columns(context["input_columns"]) - y_pred_proba = np.array(trained_model.predict(prepared_dataset)) - if isinstance(y_pred_proba[0], str): - y_pred = y_pred_proba - else: - y_pred = np.argmax(y_pred_proba, axis=1) - - except ValueError as ve: - log.error(f"Validation Error: {ve}") - raise HTTPException( - status_code=400, - detail=f"Invalid columns selected: {str(ve)}", - ) from ve - except Exception as e: - log.error(e) - raise JobError( - "Model prediction failed", - ) from e - - try: - sqlite_local = config["LOCAL_PATH"] path = os.path.join(sqlite_local, "pipelines", "predictions") os.makedirs(path, exist_ok=True) existing_ids = [] @@ -76,22 +58,94 @@ async def run(self, context: Dict[str, Any]) -> Dict[str, Any]: continue next_id = max(existing_ids, default=0) + 1 - json_name = f"prediction_{next_id}.json" - - json_data = { - "metadata": { - "id": next_id, - "model_name": context["model_name"], - "dataset_name": context["dataset_name"], - "task_name": context["task_name"], - }, - "prediction": y_pred.tolist(), - } - - with open(os.path.join(path, json_name), "w") as json_file: - json.dump(json_data, json_file, indent=4) - - return {"prediction": json_name} + + def _predict_for_context(pred_context: Dict[str, Any]) -> Dict[str, Any]: + nonlocal next_id + loaded_dataset = pred_context["dataset"] + model = pred_context["model_class"] + pipeline_id = pred_context.get("pipeline_id") + model_node_id = pred_context.get("model_node_id") + + model_path = pred_context["model_path"] + if pipeline_id is not None and model_node_id: + node_model_path = os.path.join( + sqlite_local, + "pipelines", + "train", + str(pipeline_id), + str(model_node_id), + "model", + ) + if os.path.exists(node_model_path): + model_path = node_model_path + + try: + trained_model: BaseModel = model.load(model_path) + except Exception as e: + log.exception(e) + raise JobError( + "Failed to load trained model for Prediction. " + f"model_name={pred_context.get('model_name')}, " + f"model_path={model_path}, " + f"model_node_id={model_node_id}, " + f"pipeline_id={pipeline_id}" + ) from e + + try: + prepared_dataset = loaded_dataset.select_columns( + pred_context["input_columns"] + ) + y_pred_proba = np.array(trained_model.predict(prepared_dataset)) + + if isinstance(y_pred_proba[0], str): + y_pred = y_pred_proba + elif y_pred_proba.ndim == 2: + y_pred = np.argmax(y_pred_proba, axis=1) + else: + y_pred = y_pred_proba + + except ValueError as ve: + log.error(f"Validation Error: {ve}") + raise HTTPException( + status_code=400, + detail=f"Invalid columns selected: {str(ve)}", + ) from ve + except Exception as e: + log.error(e) + raise JobError( + "Model prediction failed", + ) from e + + json_name = f"prediction_{next_id}.json" + json_data = { + "metadata": { + "id": next_id, + "model_name": pred_context["model_name"], + "dataset_name": pred_context["dataset_name"], + "task_name": pred_context["task_name"], + }, + "prediction": y_pred.tolist(), + } + + with open(os.path.join(path, json_name), "w") as json_file: + json.dump(json_data, json_file, indent=4) + + next_id += 1 + return { + "prediction": json_name, + "model_node_id": pred_context.get("model_node_id"), + "dataset_name": pred_context.get("dataset_name"), + "model_name": pred_context.get("model_name"), + "task_name": pred_context.get("task_name"), + } + + branches = context.get("_branches") + if branches: + entries = [_predict_for_context(branch) for branch in branches] + return {"prediction": entries} + + entry = _predict_for_context(context) + return {"prediction": entry["prediction"]} except Exception as e: log.exception(e) raise JobError("Can not save prediction to json file") from e diff --git a/DashAI/back/pipeline/retrieve_model_node.py b/DashAI/back/pipeline/retrieve_model_node.py index 0cf7b73bb..f1a93fb76 100644 --- a/DashAI/back/pipeline/retrieve_model_node.py +++ b/DashAI/back/pipeline/retrieve_model_node.py @@ -1,7 +1,7 @@ import logging from typing import TYPE_CHECKING, Any, Dict, List -from kink import di, inject +from kink import inject from DashAI.back.job.base_job import BaseJob, JobError @@ -39,6 +39,12 @@ def __init__( def set_status_as_delivered(self) -> None: log.debug("RetrieveModel executed successfully.") + def set_status_as_error(self) -> None: + log.error("RetrieveModel encountered an error.") + + def get_job_name(self) -> str: + return f"RetrieveModel: {self.model}" + @inject async def run( self, @@ -55,12 +61,17 @@ async def run( context["task_name"] = self.task dataset = context["dataset"] - input_columns_names = get_column_names_from_indexes( - dataset, self.input_columns - ) + if self.input_columns and all( + isinstance(col, str) for col in self.input_columns + ): + input_columns_names = self.input_columns + else: + input_columns_names = get_column_names_from_indexes( + dataset, self.input_columns + ) context["input_columns"] = input_columns_names - model_class = component_registry(di)[self.model]["class"] + model_class = component_registry[self.model]["class"] context["model_class"] = model_class return { diff --git a/DashAI/back/pipeline/split_data_node.py b/DashAI/back/pipeline/split_data_node.py new file mode 100644 index 000000000..31a707312 --- /dev/null +++ b/DashAI/back/pipeline/split_data_node.py @@ -0,0 +1,109 @@ +import logging +from typing import Any, Dict, List + +from DashAI.back.dataloaders.classes.dashai_dataset import ( + get_column_names_from_indexes, +) +from DashAI.back.job.base_job import BaseJob, JobError + +log = logging.getLogger(__name__) + + +class SplitData(BaseJob): + """Pipeline node that configures the data partition of the dataset. + + It resolves the input/output column names and stores the split ratios + in the pipeline context. The actual ``prepare_for_task`` and split + operations are deferred to the *TaskAndModel* node which knows the + chosen task. + + Parameters + ---------- + input_columns : List[int] + Column indices to use as input features. + output_columns : List[int] + Column indices to use as output targets. + splits : Dict[str, float] + Dictionary with ``train``, ``validation``, and ``test`` ratios + (must sum to 1). + """ + + def __init__( + self, + input_columns: List[int], + output_columns: List[int], + splits: Dict[str, float], + ) -> None: + super().__init__( + kwargs={ + "input_columns": input_columns, + "output_columns": output_columns, + "splits": splits, + }, + ) + self.input_columns = input_columns + self.output_columns = output_columns + self.splits = splits + + # -- BaseJob interface --------------------------------------------------- + + def set_status_as_delivered(self) -> None: + log.debug("SplitData executed successfully.") + + def set_status_as_error(self) -> None: + log.error("SplitData encountered an error.") + + def get_job_name(self) -> str: + return "SplitData" + + # -- Execution ----------------------------------------------------------- + + async def run( + self, + context: Dict[str, Any], + ) -> Dict[str, Any]: + """Resolve column names and store split config in *context*. + + The following keys are added to ``context``: + * ``input_columns`` – resolved column names + * ``output_columns`` – resolved column names + * ``input_columns_idx`` – original column indices + * ``output_columns_idx`` – original column indices + * ``split_ratios`` – the requested split ratios dict + """ + dataset = context["dataset"] + + try: + input_columns_names = get_column_names_from_indexes( + dataset, self.input_columns + ) + output_columns_names = get_column_names_from_indexes( + dataset, self.output_columns + ) + except Exception as e: + log.exception(e) + dataset_name = context.get("dataset_name", "") + try: + total_columns = len(list(dataset.features.keys())) + except Exception: + total_columns = "" + raise JobError( + "Cannot resolve column names from the dataset. " + f"dataset_name={dataset_name}, total_columns={total_columns}, " + f"input_columns={self.input_columns}, " + f"output_columns={self.output_columns}", + ) from e + + context["input_columns"] = input_columns_names + context["output_columns"] = output_columns_names + context["input_columns_idx"] = self.input_columns + context["output_columns_idx"] = self.output_columns + context["split_ratios"] = self.splits + + return { + "split_data": { + "input_columns": self.input_columns, + "output_columns": self.output_columns, + "splits": self.splits, + }, + } diff --git a/DashAI/back/pipeline/task_and_model_node.py b/DashAI/back/pipeline/task_and_model_node.py new file mode 100644 index 000000000..1b9627fe6 --- /dev/null +++ b/DashAI/back/pipeline/task_and_model_node.py @@ -0,0 +1,236 @@ +import logging +import os +import shutil +from typing import Any, Dict + +from kink import di + +from DashAI.back.config import DefaultSettings +from DashAI.back.dataloaders.classes.dashai_dataset import ( + prepare_for_model_session, + select_columns, + split_dataset, +) +from DashAI.back.dependencies.registry import ComponentRegistry +from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.models.base_model import BaseModel +from DashAI.back.models.model_factory import ModelFactory +from DashAI.back.tasks.base_task import BaseTask + +log = logging.getLogger(__name__) + + +class TaskAndModel(BaseJob): + """Pipeline node that selects the ML task and model, and trains it. + + It expects the split data to already be available in the pipeline + context (populated by the *SplitData* node). + + Parameters + ---------- + task : str + Name of the ML task (e.g. ``"TabularClassificationTask"``). + model : str + Name of the model to train. + parameters : Dict[str, Any] | None + Model-specific hyper-parameters. + """ + + def __init__( + self, + task: str, + model: str, + parameters: Dict[str, Any] = None, + ) -> None: + super().__init__( + kwargs={ + "task": task, + "model": model, + "parameters": parameters, + }, + ) + self.task = task + self.model = model + self.parameters = parameters + + # -- BaseJob interface --------------------------------------------------- + + def set_status_as_delivered(self) -> None: + log.debug("TaskAndModel executed successfully.") + + def set_status_as_error(self) -> None: + log.error("TaskAndModel encountered an error.") + + def get_job_name(self) -> str: + return f"TaskAndModel: {self.model}" + + # -- Execution ----------------------------------------------------------- + + async def run( + self, + context: Dict[str, Any], + component_registry: ComponentRegistry = lambda di: di["component_registry"], + ) -> Dict[str, Any]: + """Train a model and save it to disk. + + Reads from ``context``: + * ``input_columns``, ``output_columns`` – populated by *SplitData* + * ``split_ratios`` – populated by *SplitData* + * ``dataset`` – the raw dataset + * ``pipeline_id`` + + Writes to ``context``: + * ``model_class``, ``model_path``, ``model_name``, ``task_name`` + * ``trained_x``, ``trained_y`` – datasets used during training + (needed by the *Metrics* node) + * ``x_splits``, ``y_splits``, ``n_labels`` + """ + pipeline_id = context["pipeline_id"] + dataset = context["dataset"] + input_columns_names = context["input_columns"] + output_columns_names = context["output_columns"] + split_ratios = context["split_ratios"] + + context["task_name"] = self.task + context["model_name"] = self.model + + # --- Resolve task and prepare dataset --- + try: + task_class: BaseTask = component_registry(di)[self.task]["class"] + task_instance = task_class() + + prepared_dataset = task_instance.prepare_for_task( + dataset=dataset, + input_columns=input_columns_names, + output_columns=output_columns_names, + ) + + n_labels = None + if self.task in [ + "TextClassificationTask", + "TabularClassificationTask", + "ImageClassificationTask", + ]: + all_classes = prepared_dataset.unique(output_columns_names[0]) + n_labels = len(all_classes) + + prepared_dataset, splits = prepare_for_model_session( + dataset=prepared_dataset, + splits=split_ratios, + output_columns=output_columns_names, + ) + + x, y = select_columns( + prepared_dataset, input_columns_names, output_columns_names + ) + x_splits = split_dataset(x) + y_splits = split_dataset(y) + + context["x_splits"] = x_splits + context["y_splits"] = y_splits + context["splits"] = splits + context["n_labels"] = n_labels + except Exception as e: + log.exception(e) + raise JobError( + f"Cannot prepare / split dataset for task {self.task}", + ) from e + + # --- Resolve model class from registry --- + try: + model_class = component_registry(di)[self.model]["class"] + context["model_class"] = model_class + except Exception as e: + log.exception(e) + raise JobError( + f"Unable to find Model with name {self.model} in registry.", + ) from e + + # --- Instantiate model via factory --- + try: + factory = ModelFactory(model_class, self.parameters, n_labels=n_labels) + model: BaseModel = factory.model + except Exception as e: + log.exception(e) + raise JobError( + f"Unable to instantiate model {self.model}", + ) from e + + # --- Train --- + try: + if hasattr(model, "train") and callable(model.train): + model.train( + x_splits["train"], + y_splits["train"], + x_splits.get("validation"), + y_splits.get("validation"), + ) + x = x_splits + y = y_splits + elif hasattr(model, "fit") and callable(model.fit): + x = {} + y = {} + for split in ["train", "validation", "test"]: + if split in x_splits: + x[split] = x_splits[split].to_pandas().values + if split in y_splits: + y[split] = y_splits[split].to_pandas().values.ravel() + model.fit(x["train"], y["train"]) + else: + raise AttributeError( + f"Model {model_class} has neither 'train' nor 'fit' method" + ) + except Exception as e: + log.exception(e) + raise JobError("Model training failed") from e + + # --- Save trained model --- + try: + settings = DefaultSettings() + sqlite_local = os.path.expanduser(settings.LOCAL_PATH) + train_root = os.path.join(sqlite_local, "pipelines", "train") + os.makedirs(train_root, exist_ok=True) + + node_id = context.get("_node_id", "TaskAndModel") + + path = os.path.join(train_root, str(pipeline_id)) + + # Backward-compatibility: older runs may have saved a file in + # `train/`. This node now expects that location to be + # a directory, so replace legacy file with a directory. + if os.path.isfile(path): + log.warning("Replacing legacy model file path with directory: %s", path) + os.remove(path) + + os.makedirs(path, exist_ok=True) + + node_path = os.path.join(path, str(node_id)) + os.makedirs(node_path, exist_ok=True) + model_path = os.path.join(node_path, "model") + + # Ensure clean overwrite when retraining same pipeline id. + if os.path.isdir(model_path): + shutil.rmtree(model_path) + elif os.path.isfile(model_path): + os.remove(model_path) + + model.save(model_path) + context["model_path"] = model_path + except Exception as e: + log.exception(e) + raise JobError("Model saving failed") from e + + # Keep references for the Metrics node + context["trained_x"] = x + context["trained_y"] = y + context["model_factory"] = factory + + return { + "task_and_model": { + "task": self.task, + "model": self.model, + "parameters": self.parameters, + "model_path": model_path, + "input_columns": context.get("input_columns"), + }, + } diff --git a/DashAI/back/pipeline/train_node.py b/DashAI/back/pipeline/train_node.py deleted file mode 100644 index 1b90471f6..000000000 --- a/DashAI/back/pipeline/train_node.py +++ /dev/null @@ -1,205 +0,0 @@ -import logging -from typing import TYPE_CHECKING, Any, Dict, List - -from kink import di, inject - -from DashAI.back.job.base_job import BaseJob, JobError -from DashAI.back.metrics.base_metric import BaseMetric -from DashAI.back.models.base_model import BaseModel -from DashAI.back.models.model_factory import ModelFactory -from DashAI.back.tasks.base_task import BaseTask - -if TYPE_CHECKING: - from DashAI.back.dependencies.registry.component_registry import ComponentRegistry - - -log = logging.getLogger(__name__) - - -class Train(BaseJob): - """ - Train node for training machine learning models in pipelines. - - Parameters - ---------- - input_columns : List[int] - List of column indices to use as input features - output_columns : List[int] - List of column indices to use as output targets - splits : Dict[str, float] - Dictionary with train/validation/test split ratios - task : str - Name of the task to perform (e.g., "TabularClassificationTask") - model : str - Name of the model to train - metrics : List[str] - List of metric names to evaluate - parameters : Dict[str, Any], optional - Model-specific parameters for training - """ - - def __init__( - self, - input_columns: List[int], - output_columns: List[int], - splits: Dict[str, float], - task: str, - model: str, - metrics: List[str], - parameters: Dict[str, Any] = None, - ) -> None: - super().__init__( - kwargs={ - "input_columns": input_columns, - "output_columns": output_columns, - "splits": splits, - "task": task, - "model": model, - "metrics": metrics, - "parameters": parameters, - } - ) - self.input_columns = input_columns - self.output_columns = output_columns - self.splits = splits - self.task = task - self.model = model - self.metrics = metrics - self.parameters = parameters - - def set_status_as_delivered(self) -> None: - log.debug("Train executed successfully.") - - @inject - async def run( - self, - context: Dict[str, Any], - config: Dict[str, Any] = lambda: di["config"], - component_registry: "ComponentRegistry" = lambda di: di["component_registry"], - ) -> Dict[str, Any]: - import os - - from DashAI.back.dataloaders.classes.dashai_dataset import ( - get_column_names_from_indexes, - prepare_for_model_session, - select_columns, - split_dataset, - ) - - context["task_name"] = self.task - context["model_name"] = self.model - pipeline_id = context["pipeline_id"] - dataset = context["dataset"] - task: BaseTask = component_registry(di)[self.task]["class"] - task_instance = task() - - input_columns_names = get_column_names_from_indexes(dataset, self.input_columns) - context["input_columns"] = input_columns_names - output_columns_names = get_column_names_from_indexes( - dataset, self.output_columns - ) - - all_metrics = { - component_dict["name"]: component_dict - for component_dict in component_registry(di).get_components_by_types( - select="Metric" - ) - } - metrics: List[BaseMetric] = [] - for metric_name in self.metrics: - metric_class = all_metrics.get(metric_name)["class"] - if metric_class: - metrics.append(metric_class) - else: - log.warning(f"Metric '{metric_name}' not found in registry.") - - try: - prepared_dataset = task_instance.prepare_for_task( - dataset, output_columns_names - ) - n_labels = None - if self.task in [ - "TextClassificationTask", - "TabularClassificationTask", - "ImageClassificationTask", - ]: - all_classes = prepared_dataset.unique(output_columns_names[0]) - n_labels = len(all_classes) - - splits = self.splits - prepared_dataset, splits = prepare_for_model_session( - dataset=prepared_dataset, - splits=splits, - output_columns=output_columns_names, - ) - - x, y = select_columns( - prepared_dataset, input_columns_names, output_columns_names - ) - x = split_dataset(x) - y = split_dataset(y) - - except Exception as e: - log.exception(e) - raise JobError( - f"Can not prepare dataset for task {self.task}", - ) from e - - try: - model_class = component_registry(di)[self.model]["class"] - context["model_class"] = model_class - except Exception as e: - log.exception(e) - raise JobError( - f"Unable to find Model with name {self.model} in registry.", - ) from e - - try: - parameters = self.parameters - factory = ModelFactory(model_class, parameters, n_labels=n_labels) - model: BaseModel = factory.model - except Exception as e: - log.exception(e) - raise JobError( - f"Unable to instantiate model {self.model}", - ) from e - - try: - model.fit(x["train"], y["train"]) - except Exception as e: - log.exception(e) - raise JobError( - "Model training failed", - ) from e - - try: - model_metrics = factory.evaluate(x, y, metrics) - except Exception as e: - log.exception(e) - raise JobError( - "Metrics calculation failed", - ) from e - - try: - sqlite_local = config["LOCAL_PATH"] - path = os.path.join(sqlite_local, "pipelines", "train") - os.makedirs(path, exist_ok=True) - model_path = os.path.join(path, str(pipeline_id)) - model.save(model_path) - context["model_path"] = model_path - except Exception as e: - log.exception(e) - raise JobError( - "Model saving failed", - ) from e - - return { - "train": { - "info": self.model, - "parameters": self.parameters, - "metrics": model_metrics, - "model_path": model_path, - "input_columns": self.input_columns, - "task": self.task, - } - } diff --git a/DashAI/back/pipeline/validator/__init__.py b/DashAI/back/pipeline/validator/__init__.py index a3786066f..1d5b896e7 100644 --- a/DashAI/back/pipeline/validator/__init__.py +++ b/DashAI/back/pipeline/validator/__init__.py @@ -1,10 +1,19 @@ from .pipeline_validator import PipelineValidator -from .validator import DataSelectorValidator, DataExplorationValidator, TrainValidator, RetrieveModelValidator +from .validator import ( + DataExplorationValidator, + DataSelectorValidator, + MetricsEvalValidator, + RetrieveModelValidator, + SplitDataValidator, + TaskAndModelValidator, +) __all__ = [ "PipelineValidator", "DataSelectorValidator", "DataExplorationValidator", - "TrainValidator", "RetrieveModelValidator", + "SplitDataValidator", + "TaskAndModelValidator", + "MetricsEvalValidator", ] diff --git a/DashAI/back/pipeline/validator/nodes_definitions.py b/DashAI/back/pipeline/validator/nodes_definitions.py index 8ea712159..d64eff6d0 100644 --- a/DashAI/back/pipeline/validator/nodes_definitions.py +++ b/DashAI/back/pipeline/validator/nodes_definitions.py @@ -2,6 +2,8 @@ from pydantic import BaseModel +from DashAI.back.pipeline.contracts import contracts_to_node_definitions + class NodeDefinition(BaseModel): type: str @@ -15,85 +17,12 @@ class NodeDefinition(BaseModel): description: str input: Optional[str] = None output: Optional[str] = None + sourceHandles: int = 1 # noqa: N815 + maxInputs: Optional[int] = 1 # noqa: N815 + maxOutputs: Optional[int] = None # noqa: N815 configType: str # noqa: N815 -NODES: List[NodeDefinition] = [ - NodeDefinition( - type="DataSelector", - name="Data Selector", - icon="FolderIcon", - requiresConfiguration=True, - source=True, - target=False, - predecessors=[], - successors=["DataExploration", "Train"], - description=( - "Loads a dataset stored in the system to be used in the pipeline." - ), - output="Dataset object", - configType="custom", - ), - NodeDefinition( - type="DataExploration", - name="Data Exploration", - icon="InsertChartIcon", - requiresConfiguration=True, - source=False, - target=True, - predecessors=["DataSelector"], - successors=[], - description=( - "Visualize and analyze your dataset to gain insights " - "into its structure and content." - ), - input="Dataset object", - configType="custom", - ), - NodeDefinition( - type="Train", - name="Train Model", - icon="SettingsIcon", - requiresConfiguration=True, - source=True, - target=True, - predecessors=["DataSelector"], - successors=["Prediction"], - description=( - "Trains a model using a selected dataset, with configurable " - "columns, splits, task, model, and metrics." - ), - input="Dataset object", - output="Trained model object and evaluation results", - configType="custom", - ), - NodeDefinition( - type="Prediction", - name="Prediction", - icon="EmojiObjectsIcon", - requiresConfiguration=False, - source=False, - target=True, - predecessors=["Train", "RetrieveModel"], - successors=[], - description="Generates predictions using a trained model.", - input="Trained model and dataset object", - configType="custom", - ), - NodeDefinition( - type="RetrieveModel", - name="Retrieve Model", - icon="ManageHistoryIcon", - requiresConfiguration=True, - source=True, - target=True, - predecessors=["DataSelector"], - successors=["Prediction"], - description="Loads a trained model from saved pipelines.", - input="Dataset object", - output="Trained model object", - configType="custom", - ), -] +NODES: List[NodeDefinition] = contracts_to_node_definitions() NODE_TYPES = [node.model_dump() for node in NODES] diff --git a/DashAI/back/pipeline/validator/pipeline_validator.py b/DashAI/back/pipeline/validator/pipeline_validator.py index c0d54f1db..78a62b597 100644 --- a/DashAI/back/pipeline/validator/pipeline_validator.py +++ b/DashAI/back/pipeline/validator/pipeline_validator.py @@ -9,19 +9,19 @@ def __init__(self, nodes: List[Dict], edges: List[Dict]): self.edges = edges self.errors: Dict[str, List[str]] = {} self.node_map = {n["id"]: n for n in nodes} - self.duplicated_ids = set() self.type_to_rule = { nt["type"]: { "predecessors": set(nt.get("predecessors", [])), "successors": set(nt.get("successors", [])), + "max_inputs": nt.get("maxInputs"), + "max_outputs": nt.get("maxOutputs"), } for nt in NODE_TYPES } self.type_to_name = {nt["type"]: nt["name"] for nt in NODE_TYPES} def validate(self) -> Dict[str, List[str]]: - self._validate_duplicates() self._validate_structure() return self.errors @@ -32,31 +32,9 @@ def _get_node_display_name(self, node: Dict) -> str: def _get_type_display_name(self, node_type: str) -> str: return self.type_to_name.get(node_type, node_type) - def _validate_duplicates(self): - def extract_number(id_str): - import re - - match = re.search(r"-(\d+)$", id_str) - return int(match.group(1)) if match else float("inf") - - type_to_ids = {} - for node in self.nodes: - node_type = node["type"] - node_name = self._get_node_display_name(node) - type_to_ids.setdefault(node_type, []).append((node["id"], node_name)) - - for _, ids in type_to_ids.items(): - sorted_ids = sorted(ids, key=lambda x: extract_number(x[0])) - for id, name in sorted_ids[1:]: - self.duplicated_ids.add(id) - self.errors.setdefault(id, []).append(f"{name} already exists.") - def _validate_structure(self): for node in self.nodes: node_id = node["id"] - if node_id in self.duplicated_ids: - continue - node_type = node["type"] node_name = self._get_node_display_name(node) rule = self.type_to_rule.get(node_type) @@ -66,10 +44,13 @@ def _validate_structure(self): expected_predecessors = rule["predecessors"] predecessors = [e for e in self.edges if e["target"] == node_id] - if len(predecessors) > 1: - self.errors.setdefault(node_id, []).append( - f"{node_name} cannot have more than one input." - ) + max_inputs = rule.get("max_inputs", 1) + if max_inputs is not None and len(predecessors) > max_inputs: + if max_inputs == 1: + message = f"{node_name} cannot have more than one input." + else: + message = f"{node_name} supports up to {max_inputs} inputs." + self.errors.setdefault(node_id, []).append(message) predecessor_types = [ self.node_map[e["source"]]["type"] @@ -99,3 +80,11 @@ def _validate_structure(self): self.errors.setdefault(node_id, []).append( f"{node_name} should not have any outputs." ) + + max_outputs = rule.get("max_outputs") + if max_outputs is not None: + outgoing_edges = [e for e in self.edges if e["source"] == node_id] + if len(outgoing_edges) > max_outputs: + self.errors.setdefault(node_id, []).append( + f"{node_name} supports up to {max_outputs} outputs." + ) diff --git a/DashAI/back/pipeline/validator/validator.py b/DashAI/back/pipeline/validator/validator.py index 27e985a70..d9e07855a 100644 --- a/DashAI/back/pipeline/validator/validator.py +++ b/DashAI/back/pipeline/validator/validator.py @@ -54,31 +54,34 @@ def validate(self) -> Dict[str, str]: return {"status": "ok"} -class TrainValidator(BaseNodeValidator): +class RetrieveModelValidator(BaseNodeValidator): def validate(self) -> Dict[str, str]: - input_cols = self.data.get("input_columns") - output_cols = self.data.get("output_columns") - task = self.data.get("task") - splits = self.data.get("splits", {}) - metrics = self.data.get("metrics") model = self.data.get("model") + model_path = self.data.get("model_path") + input_columns = self.data.get("input_columns") + task = self.data.get("task") - if not input_cols or not output_cols: + if not model or not model_path or not input_columns or not task: return { "status": "error", - "message": "Input and output columns are required", + "message": "Error in trained model selected", } - if task is None: - return {"status": "error", "message": "Task is required"} + return {"status": "ok"} - if model is None: - return {"status": "error", "message": "Model is required"} - if not metrics: +class SplitDataValidator(BaseNodeValidator): + """Validator for the SplitData node configuration.""" + + def validate(self) -> Dict[str, str]: + input_cols = self.data.get("input_columns") + output_cols = self.data.get("output_columns") + splits = self.data.get("splits", {}) + + if not input_cols or not output_cols: return { "status": "error", - "message": "At least one metric must be selected", + "message": "Input and output columns are required", } train, val, test = ( @@ -95,17 +98,32 @@ def validate(self) -> Dict[str, str]: return {"status": "ok"} -class RetrieveModelValidator(BaseNodeValidator): +class TaskAndModelValidator(BaseNodeValidator): + """Validator for the TaskAndModel node configuration.""" + def validate(self) -> Dict[str, str]: - model = self.data.get("model") - model_path = self.data.get("model_path") - input_columns = self.data.get("input_columns") task = self.data.get("task") + model = self.data.get("model") - if not model or not model_path or not input_columns or not task: + if task is None: + return {"status": "error", "message": "Task is required"} + + if model is None: + return {"status": "error", "message": "Model is required"} + + return {"status": "ok"} + + +class MetricsEvalValidator(BaseNodeValidator): + """Validator for the MetricsEval node configuration.""" + + def validate(self) -> Dict[str, str]: + metrics = self.data.get("metrics") + + if not metrics: return { "status": "error", - "message": "Error in trained model selected", + "message": "At least one metric must be selected", } return {"status": "ok"} @@ -114,6 +132,8 @@ def validate(self) -> Dict[str, str]: VALIDATOR_MAP = { "DataSelector": DataSelectorValidator, "DataExploration": DataExplorationValidator, - "Train": TrainValidator, "RetrieveModel": RetrieveModelValidator, + "SplitData": SplitDataValidator, + "TaskAndModel": TaskAndModelValidator, + "MetricsEval": MetricsEvalValidator, } diff --git a/DashAI/front/src/App.jsx b/DashAI/front/src/App.jsx index 6f02656fe..0a8f35b11 100644 --- a/DashAI/front/src/App.jsx +++ b/DashAI/front/src/App.jsx @@ -16,7 +16,6 @@ import ModelsPage from "./pages/models/Models"; import Home from "./pages/home/Home"; import ResponsiveAppBar from "./components/ResponsiveAppBar"; import PluginsPage from "./pages/plugins/Plugins"; -import PipelinesPage from "./pages/pipelines/Pipelines"; import PluginsDetails from "./pages/plugins/components/PluginsDetails"; import Generative from "./pages/generative/Generative"; import { GenerativeProvider } from "./components/generative/GenerativeContext"; @@ -159,7 +158,7 @@ function App() { path="/app/generative/sessions/:id" element={} /> - } /> + } /> } diff --git a/DashAI/front/src/api/pipeline.ts b/DashAI/front/src/api/pipeline.ts index 9ad5e7812..947201e54 100644 --- a/DashAI/front/src/api/pipeline.ts +++ b/DashAI/front/src/api/pipeline.ts @@ -23,6 +23,11 @@ export const getNodes = async () => { return response.data; }; +export const getNodeContracts = async () => { + const response = await api.get(`${pipelineEndpoint}/contracts`); + return response.data; +}; + export const updatePipeline = async ( id: number, formData: object, @@ -61,6 +66,11 @@ export const validatePipeline = async ( return response.data; }; +export const validateEdge = async (payload: object) => { + const response = await api.post(`${pipelineEndpoint}/validate_edge`, payload); + return response.data; +}; + export const getPipelinePredictionSummary = async (predictionId: string) => { const response = await api.get(`${pipelineEndpoint}/predict_summary`, { params: { @@ -77,6 +87,25 @@ export const getExplorationResults = async (id: number) => { return response.data; }; +export const getLatestPipelineRun = async ( + pipelineId: number, +): Promise<{ + id: number | null; + status: string | null; + error_message?: string | null; + node_runs: Array<{ + node_id: string; + node_type: string; + status: string | null; + error_message?: string | null; + }>; +}> => { + const response = await api.get( + `${pipelineEndpoint}/${pipelineId}/runs/latest`, + ); + return response.data; +}; + export const filterModels = async ( dataset_id: string, pipeline_id: number | null = null, diff --git a/DashAI/front/src/components/datasets/DatasetModal.jsx b/DashAI/front/src/components/datasets/DatasetModal.jsx index 627b076d5..922b43b2f 100644 --- a/DashAI/front/src/components/datasets/DatasetModal.jsx +++ b/DashAI/front/src/components/datasets/DatasetModal.jsx @@ -22,7 +22,7 @@ import ConfigureAndUploadDataset from "./ConfigureAndUploadDataset"; import { useSnackbar } from "notistack"; import { enqueueDatasetJob as enqueueDatasetRequest } from "../../api/job"; import DatasetPreviewStep from "./DatasetPreviewStep"; -import { previewWithTypes } from "../../api/datasets"; +import { previewWithTypes, createDataset } from "../../api/datasets"; const SKIP_PREVIEW_DATALOADERS = new Set([]); @@ -64,18 +64,18 @@ function DatasetModal({ open, setOpen, updateDatasets }) { const handleSubmitNewDataset = async () => { try { - const name = - newDataset.params.name === null - ? newDataset.file.name - : newDataset.params.name; + const name = newDataset.params.name || newDataset.file.name; newDataset.params["dataloader"] = newDataset.dataloader; - await enqueueDatasetRequest(newDataset.file, name, newDataset.url, { + + const data = await createDataset(name); + await enqueueDatasetRequest(data.id, newDataset.file, newDataset.url, { ...newDataset.params, + name: name, inferred_types: columnsSpec, }); enqueueSnackbar("Dataset upload job started", { variant: "success" }); - updateDatasets(); + updateDatasets(data.id); handleCloseDialog(); } catch (error) { console.error(error); @@ -89,7 +89,13 @@ function DatasetModal({ open, setOpen, updateDatasets }) { const handlePreviewDataset = async (file, url) => { const formData = new FormData(); formData.append("file", newDataset.file); - formData.append("params", JSON.stringify(newDataset.params)); + formData.append( + "params", + JSON.stringify({ + ...newDataset.params, + dataloader_name: newDataset.dataloader, + }), + ); try { const preview = await previewWithTypes(formData); diff --git a/DashAI/front/src/components/explorations/ExplorationRunner.jsx b/DashAI/front/src/components/explorations/ExplorationRunner.jsx index 9e5675d1e..70d37cc4d 100644 --- a/DashAI/front/src/components/explorations/ExplorationRunner.jsx +++ b/DashAI/front/src/components/explorations/ExplorationRunner.jsx @@ -14,7 +14,6 @@ import { Check as CheckIcon, } from "@mui/icons-material"; -import { LoadingButton } from "@mui/material"; import { useSnackbar } from "notistack"; import { getComponents } from "../../api/component"; diff --git a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx index a7d347e9f..1bf77d7ea 100644 --- a/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx +++ b/DashAI/front/src/components/explorations/Steps/ConfigureExplorersStep.jsx @@ -20,7 +20,7 @@ import { useTranslation } from "react-i18next"; import useSchema from "../../../hooks/useSchema"; import { useExplorationsContext } from "../context"; -import ExplorersTable from "../ExplorationsTable"; +import ExplorersTable from "../explorers/ExplorersTable"; import { getComponents } from "../../../api/component"; import { evaluateColumnEligibility } from "../../../utils/columnEligibility"; @@ -269,21 +269,9 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { - {/* Form to add a single explorer to the exploration */} - - - - setExplorerData({ ...explorerData, name: e.target.value }) - } - fullWidth - /> - - - + + {} }) { /> - + + + setExplorerData({ ...explorerData, name: e.target.value }) + } + fullWidth + /> + + + @@ -318,7 +318,6 @@ function ConfigureExplorersStep({ onValidation = () => {} }) { - {/* Explorers table */} {loading && ( diff --git a/DashAI/front/src/components/pipelines/CustomNode.jsx b/DashAI/front/src/components/pipelines/CustomNode.jsx index 09c6e7482..4530e0d64 100644 --- a/DashAI/front/src/components/pipelines/CustomNode.jsx +++ b/DashAI/front/src/components/pipelines/CustomNode.jsx @@ -5,8 +5,11 @@ import InsertChartIcon from "@mui/icons-material/InsertChart"; import SettingsIcon from "@mui/icons-material/Settings"; import EmojiObjectsIcon from "@mui/icons-material/EmojiObjects"; import ManageHistoryIcon from "@mui/icons-material/ManageHistory"; +import CallSplitIcon from "@mui/icons-material/CallSplit"; +import ModelTrainingIcon from "@mui/icons-material/ModelTraining"; +import AssessmentIcon from "@mui/icons-material/Assessment"; import CloseIcon from "@mui/icons-material/Close"; -import { useTheme } from "@mui/material/styles"; +import { useTheme, alpha } from "@mui/material/styles"; import { useState } from "react"; const iconMap = { @@ -15,25 +18,104 @@ const iconMap = { SettingsIcon: SettingsIcon, EmojiObjectsIcon: EmojiObjectsIcon, ManageHistoryIcon: ManageHistoryIcon, + CallSplitIcon: CallSplitIcon, + ModelTrainingIcon: ModelTrainingIcon, + AssessmentIcon: AssessmentIcon, }; const CustomNode = ({ data, isConnectable }) => { const theme = useTheme(); const [hovered, setHovered] = useState(false); + const isLightCanvas = data.canvasMode === "light"; + const IconComponent = iconMap[data.icon] || SettingsIcon; const isDisabled = data.errors?.some((err) => err.includes("already exists")) ?? false; - const borderColor = - data.notConfigured && !isDisabled - ? `2px solid ${theme.palette.warning.main}` - : `1px solid ${theme.palette.ui.borderLight}`; - const iconColor = isDisabled - ? theme.palette.text.secondary - : theme.palette.text.primary; - const bgColor = isDisabled - ? theme.palette.ui.panelMedium - : theme.palette.background.paper; + + const runStatusColors = { + DELIVERED: theme.palette.status.started, + STARTED: theme.palette.status.started, + FINISHED: theme.palette.status.finished, + ERROR: theme.palette.status.error, + }; + const runColor = runStatusColors[data.status]; + const isRunning = data.status === "STARTED" || data.status === "DELIVERED"; + const statusBg = + data.status === "FINISHED" + ? alpha(theme.palette.status.finished, isLightCanvas ? 0.14 : 0.2) + : data.status === "ERROR" + ? alpha(theme.palette.status.error, isLightCanvas ? 0.12 : 0.2) + : null; + + const borderColor = runColor + ? `2px solid ${runColor}` + : isLightCanvas + ? data.notConfigured && !isDisabled + ? "2px solid #f9a825" + : "1px solid #f9a825" + : data.notConfigured && !isDisabled + ? `2px solid ${theme.palette.warning.main}` + : `1px solid ${theme.palette.ui.borderLight}`; + + const iconColor = isLightCanvas + ? isDisabled + ? "#999999" + : "#333333" + : isDisabled + ? theme.palette.text.secondary + : theme.palette.text.primary; + + const bgColor = isLightCanvas + ? isDisabled + ? "#f5f5f5" + : "#ffffff" + : isDisabled + ? theme.palette.ui.panelMedium + : theme.palette.background.paper; + + const textColor = isLightCanvas ? "#1a1a1a" : theme.palette.text.primary; + const secondaryTextColor = isLightCanvas + ? "#666666" + : theme.palette.text.secondary; + const handleColor = isLightCanvas ? "#333333" : theme.palette.text.primary; + const handleDisabledColor = isLightCanvas + ? "#cccccc" + : theme.palette.ui.border; + const sourceHandles = Math.max(1, Number(data.sourceHandles) || 1); + const targetHandles = Math.max(1, Number(data.targetHandles) || 1); + + const getSourceHandleStyle = (index) => { + if (sourceHandles === 1) { + return { + top: "50%", + transform: "translateY(-50%)", + }; + } + + const spacing = sourceHandles + 1; + const top = ((index + 1) / spacing) * 100; + return { + top: `${top}%`, + transform: "translateY(-50%)", + }; + }; + + const getTargetHandleStyle = (index) => { + if (targetHandles === 1) { + return { + top: "50%", + transform: "translateY(-50%)", + }; + } + + const spacing = targetHandles + 1; + const top = ((index + 1) / spacing) * 100; + return { + top: `${top}%`, + transform: "translateY(-50%)", + }; + }; const nodeContent = ( { justifyContent: "center", display: "flex", borderRadius: 2, - backgroundColor: bgColor, + backgroundColor: statusBg || bgColor, border: borderColor, textAlign: "center", position: "relative", + transition: "background-color 0.3s, border-color 0.3s", + "@keyframes dashaiNodePulse": { + "0%": { + boxShadow: `0 0 0 0 ${alpha(runColor || "#000000", 0.45)}`, + }, + "70%": { + boxShadow: `0 0 0 7px ${alpha(runColor || "#000000", 0)}`, + }, + "100%": { + boxShadow: `0 0 0 0 ${alpha(runColor || "#000000", 0)}`, + }, + }, + animation: isRunning + ? "dashaiNodePulse 1.4s ease-out infinite" + : undefined, }} > {data.onDelete && hovered && ( @@ -70,47 +167,51 @@ const CustomNode = ({ data, isConnectable }) => { }, }} > - + )} - {data.target && ( - - )} + {data.target && + Array.from({ length: targetHandles }).map((_, index) => ( + + ))} - {data.source && ( - - )} + {data.source && + Array.from({ length: sourceHandles }).map((_, index) => ( + + ))} ); @@ -122,10 +223,7 @@ const CustomNode = ({ data, isConnectable }) => { alignItems: "center", }} > - + {data.name || data.label} diff --git a/DashAI/front/src/components/pipelines/NodeSidebar.jsx b/DashAI/front/src/components/pipelines/NodeSidebar.jsx index b0fa2d8ca..ded7b830d 100644 --- a/DashAI/front/src/components/pipelines/NodeSidebar.jsx +++ b/DashAI/front/src/components/pipelines/NodeSidebar.jsx @@ -1,11 +1,40 @@ import React from "react"; import { Box, Typography } from "@mui/material"; -import { useTheme } from "@mui/material/styles"; +import { useTheme, alpha } from "@mui/material/styles"; import HelpOutlineIcon from "@mui/icons-material/HelpOutline"; +import PlayArrowIcon from "@mui/icons-material/PlayArrow"; +import ArrowForwardIcon from "@mui/icons-material/ArrowForward"; import { getNodeHelp } from "./nodeHelp"; -function NodeSidebar({ availableNodes, onDragStart, nodeHelp }) { +/** + * NodeSidebar – draggable node palette + contextual help. + * + * Highlighting props make pipeline construction more intuitive without ever + * restricting what can be dragged: + * - highlightMode "start": the canvas is empty; root nodes (the ones that can + * open a flow) are emphasized as the suggested starting point. + * - highlightMode "successors": a node is focused on the canvas; the nodes + * that can legally come after it are emphasized and the rest are dimmed. + * The set of emphasized node types is passed in `highlightedTypes`. Non-matching + * nodes stay fully draggable — they are only visually de-emphasized. + */ +function NodeSidebar({ + availableNodes, + onDragStart, + nodeHelp, + highlightMode = null, + highlightedTypes = [], + highlightLabel = "", +}) { const theme = useTheme(); + + const highlightActive = Boolean(highlightMode) && highlightedTypes.length > 0; + const highlightSet = new Set(highlightedTypes); + const accentColor = + highlightMode === "start" + ? theme.palette.success.main + : theme.palette.primary.main; + return ( Nodes - {availableNodes.map((node) => ( + + {highlightActive && highlightLabel && ( onDragStart(e, node.type)} - draggable sx={{ + display: "flex", + alignItems: "center", + gap: 1, mb: 2, - p: 2, - backgroundColor: theme.palette.ui.border, - color: theme.palette.text.primary, + px: 1.5, + py: 1, borderRadius: 1, - textAlign: "center", - cursor: "grab", + backgroundColor: alpha(accentColor, 0.12), + border: `1px solid ${alpha(accentColor, 0.5)}`, }} > - {node.name || node.type} + {highlightMode === "start" ? ( + + ) : ( + + )} + + {highlightLabel} + - ))} + )} + + {availableNodes.map((node) => { + const isHighlighted = highlightActive && highlightSet.has(node.type); + const isDimmed = highlightActive && !isHighlighted; + + return ( + onDragStart(e, node.type)} + draggable + sx={{ + mb: 2, + p: 2, + backgroundColor: isHighlighted + ? alpha(accentColor, 0.16) + : theme.palette.ui.border, + color: theme.palette.text.primary, + borderRadius: 1, + textAlign: "center", + cursor: "grab", + border: isHighlighted + ? `2px solid ${accentColor}` + : "2px solid transparent", + boxShadow: isHighlighted + ? `0 0 8px ${alpha(accentColor, 0.5)}` + : "none", + opacity: isDimmed ? 0.35 : 1, + transition: + "opacity 0.15s ease, box-shadow 0.15s ease, border-color 0.15s ease", + }} + > + {node.name || node.type} + + ); + })} + + + + Pipeline Chatbot + + {onToggle && ( + + + + )} + + + + + + + + Ideas de prompts: + + + - "Revisa este pipeline y sugiere mejoras" + + + - "Explica por que falla la validacion" + + + - "Sugiere el siguiente nodo" + + + + + + + + setMessage(event.target.value)} + /> + + + + ); +} + +export default PipelineChatSidebar; diff --git a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx index 44c5c69c7..d5dfa0f20 100644 --- a/DashAI/front/src/components/pipelines/PipelineDesigner.jsx +++ b/DashAI/front/src/components/pipelines/PipelineDesigner.jsx @@ -1,6 +1,6 @@ -import React, { useCallback } from "react"; +import React, { useCallback, useEffect } from "react"; import { Box, Tooltip } from "@mui/material"; -import { useTheme } from "@mui/material/styles"; +import { useTheme, alpha } from "@mui/material/styles"; import ReactFlow, { addEdge, Background, @@ -16,6 +16,8 @@ function PipelineDesigner({ setEdges, onNodesChange, onEdgesChange, + onConnect, + onEdgeDoubleClick, nodeTypes, onNodeClick, onNodeHelp, @@ -31,23 +33,51 @@ function PipelineDesigner({ nodeIdCounter, setNodeIdCounter, availableNodes, + canvasMode, }) { const theme = useTheme(); const { screenToFlowPosition } = useReactFlow(); - const onConnect = (params) => { - setEdges((eds) => - addEdge( - { - ...params, - markerEnd: { - type: "arrowclosed", - }, - }, - eds, - ), + const isLightCanvas = canvasMode === "light"; + const canvasBg = isLightCanvas ? "#ffffff" : theme.palette.background.box; + const canvasBorder = isLightCanvas + ? "1px solid #e0e0e0" + : `1px solid ${theme.palette.ui.borderLight}`; + const gridColor = isLightCanvas + ? alpha("#9e9e9e", 0.7) + : alpha(theme.palette.text.secondary, 0.6); + + // Sync canvasMode into existing nodes so CustomNode can read it + useEffect(() => { + setNodes((nds) => + nds.map((n) => ({ + ...n, + data: { ...n.data, canvasMode }, + })), ); - }; + }, [canvasMode, setNodes]); + + const handleConnect = useCallback( + (params) => { + if (onConnect) { + onConnect(params); + return; + } + + setEdges((eds) => + addEdge( + { + ...params, + markerEnd: { + type: "arrowclosed", + }, + }, + eds, + ), + ); + }, + [onConnect, setEdges], + ); const onDragOver = useCallback((event) => { event.preventDefault(); @@ -83,6 +113,9 @@ function PipelineDesigner({ errors: nodeErrors, source: nodeInfo?.source || false, target: nodeInfo?.target || false, + sourceHandles: nodeInfo?.sourceHandles || 1, + targetHandles: nodeInfo?.targetHandles || 1, + canvasMode, onDelete: () => { setNodes((nds) => nds.filter((n) => n.id !== nodeId)); setNodeData((prev) => { @@ -103,6 +136,7 @@ function PipelineDesigner({ setNodeIdCounter((prev) => prev + 1); }, [ + canvasMode, dragging, setNodes, setNodeData, @@ -122,7 +156,8 @@ function PipelineDesigner({ edges={edges} onNodesChange={onNodesChange} onEdgesChange={onEdgesChange} - onConnect={onConnect} + onConnect={handleConnect} + onEdgeDoubleClick={onEdgeDoubleClick} onDragOver={onDragOver} onDrop={onDrop} onNodeClick={onNodeHelp} @@ -136,8 +171,8 @@ function PipelineDesigner({ width: "100%", height: "100%", borderRadius: 12, - background: theme.palette.background.default, - border: `1px solid ${theme.palette.ui.border}`, + background: canvasBg, + border: canvasBorder, boxShadow: "0 2px 6px rgba(0,0,0,0.1)", }} > @@ -166,7 +201,7 @@ function PipelineDesigner({ )} - + ); diff --git a/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx new file mode 100644 index 000000000..9b25eca6d --- /dev/null +++ b/DashAI/front/src/components/pipelines/PipelineHistorySidebar.jsx @@ -0,0 +1,217 @@ +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { + Box, + Button, + Divider, + IconButton, + LinearProgress, + Typography, +} from "@mui/material"; +import { + AddCircleOutline as AddIcon, + MoreVert as MoreVertIcon, + Update as UpdateIcon, + Visibility as ViewIcon, +} from "@mui/icons-material"; +import HistoryIcon from "@mui/icons-material/History"; +import { useTheme } from "@mui/material/styles"; +import { DataGrid, GridActionsCellItem, GridToolbar } from "@mui/x-data-grid"; +import { Menu, MenuItem } from "@mui/material"; +import { useNavigate } from "react-router-dom"; +import { useSnackbar } from "notistack"; +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; + +import { deletePipeline, getPipelines } from "../../api/pipeline"; +import { formatDate } from "../../utils"; +import DeleteItemModal from "../custom/DeleteItemModal"; + +function PipelineHistorySidebar({ currentPipelineId }) { + const theme = useTheme(); + const navigate = useNavigate(); + const { enqueueSnackbar } = useSnackbar(); + const [loading, setLoading] = useState(true); + const [pipelines, setPipelines] = useState([]); + const [rowMenuAnchor, setRowMenuAnchor] = useState(null); + const [rowMenuPipeline, setRowMenuPipeline] = useState(null); + + const fetchPipelines = useCallback(async () => { + setLoading(true); + try { + const data = await getPipelines(); + setPipelines(data || []); + } catch (error) { + enqueueSnackbar("Error while fetching pipelines", { variant: "error" }); + console.error(error); + } finally { + setLoading(false); + } + }, [enqueueSnackbar]); + + useEffect(() => { + fetchPipelines(); + }, [fetchPipelines]); + + const handleDelete = async (id) => { + try { + await deletePipeline(id); + enqueueSnackbar("Pipeline deleted successfully", { variant: "success" }); + if (String(currentPipelineId) === String(id)) { + navigate("/app/pipelines"); + } + fetchPipelines(); + } catch (error) { + enqueueSnackbar("Failed to delete pipeline", { variant: "error" }); + console.error(error); + } + }; + + const selectedPipeline = useMemo(() => { + const parsedId = Number(currentPipelineId); + return { + type: "include", + ids: new Set(Number.isFinite(parsedId) ? [parsedId] : []), + }; + }, [currentPipelineId]); + + const handleOpenRowMenu = (event, pipeline) => { + event.stopPropagation(); + setRowMenuAnchor(event.currentTarget); + setRowMenuPipeline(pipeline); + }; + + const handleCloseRowMenu = () => { + setRowMenuAnchor(null); + setRowMenuPipeline(null); + }; + + const columns = useMemo( + () => [ + { + field: "name", + headerName: "Name", + minWidth: 150, + flex: 1, + }, + { + field: "actions", + type: "actions", + minWidth: 120, + getActions: (params) => [ + } + label="View" + onClick={() => navigate(`/app/pipelines/${params.id}`)} + />, + } + label="Details" + onClick={(event) => handleOpenRowMenu(event, params.row)} + />, + handleDelete(params.id)} + />, + ], + }, + ], + [navigate, handleDelete], + ); + + return ( + + + + + Pipelines History + + + + + + + + + + + + + + + + + Created:{" "} + {rowMenuPipeline ? formatDate(rowMenuPipeline.created) : "-"} + + + Edited:{" "} + {rowMenuPipeline ? formatDate(rowMenuPipeline.last_modified) : "-"} + + + + + ); +} + +export default PipelineHistorySidebar; diff --git a/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx b/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx new file mode 100644 index 000000000..214117e5d --- /dev/null +++ b/DashAI/front/src/components/pipelines/PipelineNodeConfigSidebar.jsx @@ -0,0 +1,74 @@ +import React from "react"; +import { Box, Divider, IconButton, Typography } from "@mui/material"; +import SettingsIcon from "@mui/icons-material/Settings"; +import CloseIcon from "@mui/icons-material/Close"; +import { useTheme } from "@mui/material/styles"; +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; + +function PipelineNodeConfigSidebar({ selectedNode, onClose, children }) { + const theme = useTheme(); + const nodeTitle = + selectedNode?.data?.name || selectedNode?.data?.label || selectedNode?.type; + + return ( + + + + + + {nodeTitle ? "Node Configuration" : "Pipeline Node Configuration"} + + + + {selectedNode && ( + + + + )} + + + + + + + {selectedNode ? ( + children || ( + + This node does not require configuration. + + ) + ) : ( + + Select a node in the canvas to configure it. + + )} + + + ); +} + +export default PipelineNodeConfigSidebar; diff --git a/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx b/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx new file mode 100644 index 000000000..7322fe055 --- /dev/null +++ b/DashAI/front/src/components/pipelines/PipelineTemplatesSidebar.jsx @@ -0,0 +1,98 @@ +import React from "react"; +import { + Box, + Button, + Card, + CardActions, + CardContent, + Chip, + IconButton, + Typography, +} from "@mui/material"; +import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; +import { ChevronRight } from "@mui/icons-material"; +import { useTheme } from "@mui/material/styles"; +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; + +function PipelineTemplatesSidebar({ + onToggle, + templates = [], + onSelectTemplate, +}) { + const theme = useTheme(); + + return ( + + + + + Prefabricated Pipelines + + {onToggle && ( + + + + )} + + + + {templates.map((template) => ( + onSelectTemplate?.(template.id)} + > + + + {template.name} + + + {(template.labels || template.steps).join(" + ")} + + + {(template.labels || template.steps).map((step) => ( + + ))} + + + + + + + ))} + + + ); +} + +export default PipelineTemplatesSidebar; diff --git a/DashAI/front/src/components/pipelines/PipelineToolbar.jsx b/DashAI/front/src/components/pipelines/PipelineToolbar.jsx index e89d70850..00fdb8638 100644 --- a/DashAI/front/src/components/pipelines/PipelineToolbar.jsx +++ b/DashAI/front/src/components/pipelines/PipelineToolbar.jsx @@ -1,5 +1,15 @@ import React from "react"; -import { Box, TextField, Button } from "@mui/material"; +import { + Box, + TextField, + Button, + IconButton, + Tooltip, + CircularProgress, +} from "@mui/material"; +import { useTheme } from "@mui/material/styles"; +import LightModeIcon from "@mui/icons-material/LightMode"; +import DarkModeIcon from "@mui/icons-material/DarkMode"; function PipelineToolbar({ pipelineName, @@ -8,14 +18,27 @@ function PipelineToolbar({ nameError, nameErrorMessage, handlePipelineNameChange, + canvasMode, + setCanvasMode, + isRunning = false, }) { + const theme = useTheme(); + + const toggleCanvasMode = () => { + setCanvasMode((prev) => (prev === "dark" ? "light" : "dark")); + }; + return ( - + + + + {canvasMode === "dark" ? ( + + ) : ( + + )} + + + + ); } diff --git a/DashAI/front/src/components/pipelines/index.jsx b/DashAI/front/src/components/pipelines/index.jsx index 460c2490f..e4b54e45f 100644 --- a/DashAI/front/src/components/pipelines/index.jsx +++ b/DashAI/front/src/components/pipelines/index.jsx @@ -3,6 +3,10 @@ export { default as PipelineHeader } from "./PipelineHeader"; export { default as PipelineToolbar } from "./PipelineToolbar"; export { default as PipelineDesigner } from "./PipelineDesigner"; export { default as NodeSidebar } from "./NodeSidebar"; +export { default as PipelineHistorySidebar } from "./PipelineHistorySidebar"; +export { default as PipelineTemplatesSidebar } from "./PipelineTemplatesSidebar"; +export { default as PipelineChatSidebar } from "./PipelineChatSidebar"; +export { default as PipelineNodeConfigSidebar } from "./PipelineNodeConfigSidebar"; export { default as PipelinesTable } from "./PipelinesTable"; export { default as CustomNode } from "./CustomNode"; export { default as Run } from "./Run"; @@ -10,8 +14,10 @@ export { default as Run } from "./Run"; // Node components export { default as DataSelectorNode } from "./nodes/DataSelectorNode"; export { default as DataExplorationNode } from "./nodes/DataExplorationNode"; -export { default as TrainNode } from "./nodes/TrainNode"; export { default as RetrieveModelNode } from "./nodes/RetrieveModelNode"; +export { default as SplitDataNode } from "./nodes/SplitDataNode"; +export { default as TaskAndModelNode } from "./nodes/TaskAndModelNode"; +export { default as MetricsEvalNode } from "./nodes/MetricsEvalNode"; export { default as ParamsSettings } from "./ParamsSettings"; export { default as ExplorationModal } from "./nodes/ExplorationModal"; diff --git a/DashAI/front/src/components/pipelines/nodeHelp.jsx b/DashAI/front/src/components/pipelines/nodeHelp.jsx index c0ac8677a..7821631b0 100644 --- a/DashAI/front/src/components/pipelines/nodeHelp.jsx +++ b/DashAI/front/src/components/pipelines/nodeHelp.jsx @@ -2,7 +2,8 @@ let nodeHelp = { Pipeline: { description: `Design a machine learning workflow by dragging and connecting nodes. Single-click a node to see more information. - Double-click a node to configure parameters.`, + Double-click a node to configure parameters. + Double-click an edge to remove the connection.`, }, }; diff --git a/DashAI/front/src/components/pipelines/nodeRegistry.jsx b/DashAI/front/src/components/pipelines/nodeRegistry.jsx index b88982351..d488b5a90 100644 --- a/DashAI/front/src/components/pipelines/nodeRegistry.jsx +++ b/DashAI/front/src/components/pipelines/nodeRegistry.jsx @@ -1,15 +1,19 @@ import DataSelectorNode from "./nodes/DataSelectorNode"; import DataExplorationNode from "./nodes/DataExplorationNode"; -import TrainNode from "./nodes/TrainNode"; import RetrieveModelNode from "./nodes/RetrieveModelNode"; import ConfigurableNode from "./nodes/ConfigurableNode"; +import SplitDataNode from "./nodes/SplitDataNode"; +import TaskAndModelNode from "./nodes/TaskAndModelNode"; +import MetricsEvalNode from "./nodes/MetricsEvalNode"; const nodeRegistry = { DataSelector: DataSelectorNode, DataExploration: DataExplorationNode, - Train: TrainNode, RetrieveModel: RetrieveModelNode, Configurable: ConfigurableNode, + SplitData: SplitDataNode, + TaskAndModel: TaskAndModelNode, + MetricsEval: MetricsEvalNode, }; export default nodeRegistry; diff --git a/DashAI/front/src/components/pipelines/nodeTypes.jsx b/DashAI/front/src/components/pipelines/nodeTypes.jsx index e53e53486..48510906c 100644 --- a/DashAI/front/src/components/pipelines/nodeTypes.jsx +++ b/DashAI/front/src/components/pipelines/nodeTypes.jsx @@ -1,9 +1,9 @@ -import { getNodes } from "../../api/pipeline"; +import { getNodeContracts } from "../../api/pipeline"; import CustomNode from "./CustomNode"; export async function getNodeTypes() { try { - const response = await getNodes(); + const response = await getNodeContracts(); if (Array.isArray(response)) { return response; } else { diff --git a/DashAI/front/src/components/pipelines/nodes/ConfigurableNode.jsx b/DashAI/front/src/components/pipelines/nodes/ConfigurableNode.jsx index 6ae74d871..c09f18ec4 100644 --- a/DashAI/front/src/components/pipelines/nodes/ConfigurableNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/ConfigurableNode.jsx @@ -16,6 +16,7 @@ function ConfigurableNode({ onSave, savedConfig = {}, configSchema, + displayMode = "dialog", }) { const [formValues, setFormValues] = useState({}); @@ -41,38 +42,54 @@ function ConfigurableNode({ onClose(); }; + const fields = (configSchema?.fields || []).map((field) => ( + + {field.type === "select" ? ( + + {field.options.map((opt) => ( + + {opt} + + ))} + + ) : ( + + )} + + )); + + if (displayMode === "panel") { + if (!open) return null; + + return ( + + {fields} + + + + + + ); + } + return ( Configure Node - - {(configSchema?.fields || []).map((field) => ( - - {field.type === "select" ? ( - - {field.options.map((opt) => ( - - {opt} - - ))} - - ) : ( - - )} - - ))} - + {fields} diff --git a/DashAI/front/src/components/pipelines/nodes/ExplorationModal.jsx b/DashAI/front/src/components/pipelines/nodes/ExplorationModal.jsx index 31865db70..d99b29372 100644 --- a/DashAI/front/src/components/pipelines/nodes/ExplorationModal.jsx +++ b/DashAI/front/src/components/pipelines/nodes/ExplorationModal.jsx @@ -10,6 +10,7 @@ import { useMediaQuery, useTheme, Box, + Typography, } from "@mui/material"; import CloseIcon from "@mui/icons-material/Close"; import ConfigureExplorersStep from "../../../components/explorations/Steps/ConfigureExplorersStep"; @@ -17,7 +18,13 @@ import { useExplorationsContext } from "../../../components/explorations/context import { validateNode } from "../../../api/pipeline"; import { useSnackbar } from "notistack"; -function ConfigureExplorersModal({ open, onClose, onSave, savedConfig }) { +function ConfigureExplorersModal({ + open, + onClose, + onSave, + savedConfig, + displayMode = "dialog", +}) { const theme = useTheme(); const fullScreen = useMediaQuery(theme.breakpoints.down("sm")); const [valid, setValid] = useState(false); @@ -83,10 +90,36 @@ function ConfigureExplorersModal({ open, onClose, onSave, savedConfig }) { } }; - if (loading && open) { + if (!open) { return null; } + if (loading) { + return null; + } + + if (displayMode === "panel") { + return ( + + Configure Explorers + + + + + + + + + ); + } + return ( { + const [metrics, setMetrics] = useState(savedConfig?.metrics || []); + const [availableMetrics, setAvailableMetrics] = useState([]); + const { enqueueSnackbar } = useSnackbar(); + + const taskNode = prevNodes?.find((node) => node?.task); + const task = taskNode?.task || ""; + + useEffect(() => { + setMetrics(savedConfig?.metrics || []); + }, [savedConfig]); + + useEffect(() => { + const fetchMetrics = async () => { + if (!task) { + setAvailableMetrics([]); + return; + } + + try { + const metricsList = await getComponentsRequest({ + selectTypes: ["Metric"], + relatedComponent: task, + }); + setAvailableMetrics(metricsList); + } catch (error) { + console.error("Error fetching metrics:", error); + } + }; + + fetchMetrics(); + }, [task]); + + const handleSave = async () => { + const payload = { metrics }; + + try { + const response = await validateNode("MetricsEval", payload); + if (response.status === "ok") { + onSave(payload); + onClose(); + } else { + enqueueSnackbar(response.message || "Validation failed", { + variant: "error", + }); + } + } catch (e) { + enqueueSnackbar("Error validating node", { variant: "error" }); + console.error(e); + } + }; + + return ( + + + + + Metric Evaluation + + + + + setMetrics(e.target.value)} + margin="normal" + disabled={!task} + slotProps={{ + select: { + multiple: true, + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, + }, + }, + }, + renderValue: (selected) => ( + + {selected.map((metricName) => ( + + ))} + + ), + }, + }} + sx={{ + "& .MuiInputBase-root": { + minHeight: 58, + }, + "& .MuiSelect-select": { + minHeight: "56px", + display: "flex", + alignItems: "center", + }, + }} + > + {availableMetrics.map((metric) => ( + + {metric.name} + + ))} + + + + {!task && ( + + ⚠️ Connect a Task & Model node first so compatible metrics can + be listed. + + )} + + + + + + + + ); +}; + +MetricsEvalNode.propTypes = { + open: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + onSave: PropTypes.func.isRequired, + savedConfig: PropTypes.object, + prevNodes: PropTypes.arrayOf(PropTypes.object), +}; + +MetricsEvalNode.defaultProps = { + savedConfig: null, + prevNodes: [], +}; + +export default MetricsEvalNode; diff --git a/DashAI/front/src/components/pipelines/nodes/RetrieveModelNode.jsx b/DashAI/front/src/components/pipelines/nodes/RetrieveModelNode.jsx index 6092e68cb..794d0f081 100644 --- a/DashAI/front/src/components/pipelines/nodes/RetrieveModelNode.jsx +++ b/DashAI/front/src/components/pipelines/nodes/RetrieveModelNode.jsx @@ -11,6 +11,24 @@ import { filterModels } from "../../../api/pipeline"; import { useSnackbar } from "notistack"; import { validateNode } from "../../../api/pipeline"; import { useParams } from "react-router-dom"; + +const getModelEntry = (pipeline) => { + const pick = (obj) => { + if (!obj || typeof obj !== "object") return null; + if (obj.model_path) return obj; // flat (legacy) structure + const values = Object.values(obj); // aggregated: keyed by node id + return values.find((v) => v && v.model_path) || null; + }; + const entry = pick(pipeline?.train) || pick(pipeline?.task_and_model); + if (!entry) return null; + return { + model: entry.info || entry.model || "", + model_path: entry.model_path || "", + input_columns: entry.input_columns || [], + task: entry.task || "", + }; +}; + function RetrieveModelNode({ onClose, onSave, @@ -31,7 +49,7 @@ function RetrieveModelNode({ useEffect(() => { if (savedConfig?.model_path && pipelines.length > 0) { const matching = pipelines.find( - (p) => p.train?.model_path === savedConfig.model_path, + (p) => getModelEntry(p)?.model_path === savedConfig.model_path, ); if (matching) { setSelectedModelId(matching.id); @@ -67,12 +85,18 @@ function RetrieveModelNode({ const selected = pipelines.find((m) => m.id === selectedModelId); if (!selected) return; - const train = selected.train || {}; + const entry = getModelEntry(selected); + if (!entry) { + enqueueSnackbar("Selected pipeline has no trained model", { + variant: "warning", + }); + return; + } const nodeData = { - model: train.info || "", - model_path: train.model_path || "", - input_columns: train.input_columns || [], - task: train.task || "", + model: entry.model, + model_path: entry.model_path, + input_columns: entry.input_columns, + task: entry.task, }; try { @@ -97,7 +121,7 @@ function RetrieveModelNode({ id: "model", header: "Model", grow: true, - accessorFn: (row) => row.train?.info || "Unknown", + accessorFn: (row) => getModelEntry(row)?.model || "Unknown", }, ], [], diff --git a/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx b/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx new file mode 100644 index 000000000..8125ec24c --- /dev/null +++ b/DashAI/front/src/components/pipelines/nodes/SplitDataNode.jsx @@ -0,0 +1,354 @@ +import React, { useState, useEffect, useRef } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Button, + DialogContent, + Typography, + TextField, + Grid, + FormControlLabel, + Checkbox, +} from "@mui/material"; +import { getDatasetInfo as getDatasetInfoRequest } from "../../../api/datasets"; +import { validateNode } from "../../../api/pipeline"; +import { parseRangeToIndex } from "../../../utils/parseRange"; +import { useSnackbar } from "notistack"; + +/** + * SplitDataNode – configures input/output columns and train/val/test splits. + * + * Expects `prevNodes` to contain the DataSelector config (with id & file_path) + * so it can fetch column metadata from the dataset. + */ +const SplitDataNode = ({ open, onClose, onSave, savedConfig, prevNodes }) => { + // Column ranges + const [inputColumns, setInputColumns] = useState( + savedConfig?.input_columns || "", + ); + const [outputColumns, setOutputColumns] = useState( + savedConfig?.output_columns || "", + ); + + // Splits + const [splits, setSplits] = useState( + savedConfig?.splits || { + train: 0.6, + validation: 0.2, + test: 0.2, + shuffle: true, + stratify: false, + splitType: "random", + }, + ); + + // Dataset info + const datasetNode = prevNodes?.find((node) => node?.file_path && node?.id); + const datasetId = datasetNode?.id ?? null; + const [datasetInfo, setDatasetInfo] = useState({}); + const [infoLoading, setInfoLoading] = useState(true); + + // Validation + const [inputError, setInputError] = useState(false); + const [outputError, setOutputError] = useState(false); + const [inputErrorMessage, setInputErrorMessage] = useState(""); + const [outputErrorMessage, setOutputErrorMessage] = useState(""); + const [validationStatus, setValidationStatus] = useState(""); + + const { enqueueSnackbar } = useSnackbar(); + const hasWarnedRef = useRef(false); + + const getDatasetInfo = async () => { + setInfoLoading(true); + try { + const info = await getDatasetInfoRequest(datasetId); + setDatasetInfo(info); + } catch (error) { + enqueueSnackbar("Error while trying to obtain the dataset info.", { + variant: "error", + }); + console.error(error); + } finally { + setInfoLoading(false); + } + }; + + useEffect(() => { + if (!datasetId) { + setInfoLoading(false); + if (!hasWarnedRef.current) { + enqueueSnackbar( + "Missing dataset – connect a DataSelector node first.", + { + variant: "warning", + }, + ); + hasWarnedRef.current = true; + } + return; + } + getDatasetInfo(); + }, [datasetId]); + + useEffect(() => { + const total = datasetInfo?.total_columns; + setInputColumns( + savedConfig?.input_columns?.join(",") || (total ? `1-${total - 1}` : ""), + ); + setOutputColumns( + savedConfig?.output_columns?.join(",") || (total ? `${total}` : ""), + ); + setSplits( + savedConfig?.splits || { + train: 0.6, + validation: 0.2, + test: 0.2, + shuffle: true, + stratify: false, + splitType: "random", + }, + ); + }, [savedConfig, datasetInfo]); + + useEffect(() => { + const maxValue = datasetInfo?.total_columns; + if (!maxValue) return; + + try { + if (inputColumns) parseRangeToIndex(inputColumns, maxValue); + setInputError(false); + setInputErrorMessage(""); + } catch (err) { + setInputError(true); + setInputErrorMessage(err.message); + } + + try { + if (outputColumns) parseRangeToIndex(outputColumns, maxValue); + setOutputError(false); + setOutputErrorMessage(""); + } catch (err) { + setOutputError(true); + setOutputErrorMessage(err.message); + } + }, [inputColumns, outputColumns, datasetInfo]); + + const handleSave = async () => { + const maxValue = datasetInfo?.total_columns; + const columnNames = datasetInfo?.column_names || []; + + const parsedInput = inputColumns + ? parseRangeToIndex(inputColumns, maxValue) + : []; + const parsedOutput = outputColumns + ? parseRangeToIndex(outputColumns, maxValue) + : []; + + const inputColumnNames = parsedInput.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + const outputColumnNames = parsedOutput.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + + const payload = { + input_columns: parsedInput, + output_columns: parsedOutput, + splits: { + train: splits.train, + validation: splits.validation, + test: splits.test, + shuffle: splits.shuffle, + stratify: splits.stratify, + splitType: splits.splitType, + }, + }; + + try { + const response = await validateNode("SplitData", payload); + if (response.status === "ok") { + setValidationStatus("ok"); + onSave(payload); + onClose(); + } else { + setValidationStatus("error"); + enqueueSnackbar(response.message || "Validation failed", { + variant: "error", + }); + } + } catch (e) { + setValidationStatus("error"); + enqueueSnackbar("Error validating node", { variant: "error" }); + console.error(e); + } + }; + + const splitsSum = + Number(splits.train) + Number(splits.validation) + Number(splits.test); + const splitsValid = Math.abs(splitsSum - 1) < 0.001; + + return ( + + + + + Column Selection + + + + + + setInputColumns(e.target.value)} + margin="normal" + error={inputError} + helperText={inputError ? inputErrorMessage : "e.g. 1-4 or 1,3,5"} + /> + setOutputColumns(e.target.value)} + margin="normal" + error={outputError} + helperText={outputError ? outputErrorMessage : "e.g. 5 or 5-6"} + /> + + + + + + Data Splits + + + + + + + + setSplits({ ...splits, test: parseFloat(e.target.value) }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + + + + + setSplits({ ...splits, train: parseFloat(e.target.value) }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + + + + + setSplits({ + ...splits, + validation: parseFloat(e.target.value), + }) + } + margin="normal" + inputProps={{ step: 0.05, min: 0, max: 1 }} + /> + + + + + {!splitsValid && ( + + + Splits must sum to 1.0 (current: {splitsSum.toFixed(2)}) + + + )} + + + + + + setSplits({ ...splits, shuffle: e.target.checked }) + } + /> + } + label="Shuffle" + sx={{ m: 0 }} + /> + + + + setSplits({ ...splits, stratify: e.target.checked }) + } + /> + } + label="Stratify" + sx={{ m: 0 }} + /> + + + + + + + + + + ); +}; + +SplitDataNode.propTypes = { + open: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + onSave: PropTypes.func.isRequired, + savedConfig: PropTypes.object, + prevNodes: PropTypes.arrayOf(PropTypes.object), +}; + +SplitDataNode.defaultProps = { + savedConfig: null, + prevNodes: [], +}; + +export default SplitDataNode; diff --git a/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx new file mode 100644 index 000000000..c565e45b7 --- /dev/null +++ b/DashAI/front/src/components/pipelines/nodes/TaskAndModelNode.jsx @@ -0,0 +1,456 @@ +import React, { useState, useEffect, useRef } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Button, + DialogContent, + Typography, + TextField, + MenuItem, + Grid, + IconButton, +} from "@mui/material"; +import SettingsIcon from "@mui/icons-material/Settings"; +import useSchema from "../../../hooks/useSchema"; +import ParamsSettings from "../ParamsSettings"; +import { getComponents as getComponentsRequest } from "../../../api/component"; +import { validateColumns as validateColumnsRequest } from "../../../api/modelSession"; +import { getDatasetInfo as getDatasetInfoRequest } from "../../../api/datasets"; +import { validateNode } from "../../../api/pipeline"; +import { parseRangeToIndex } from "../../../utils/parseRange"; +import { useSnackbar } from "notistack"; + +/** + * TaskAndModelNode – selects ML task & model, configures hyper-parameters. + * + * Receives from SplitData (via prevNodes): + * input_columns, output_columns, splits, id (dataset id), file_path + */ +const TaskAndModelNode = ({ + open, + onClose, + onSave, + savedConfig, + prevNodes, +}) => { + const [task, setTask] = useState(savedConfig?.task || ""); + const [model, setModel] = useState(savedConfig?.model || ""); + const [openSettings, setOpenSettings] = useState(false); + const [modelParams, setModelParams] = useState(savedConfig?.parameters || {}); + const [initialized, setInitialized] = useState(false); + + const [availableTasks, setAvailableTasks] = useState([]); + const [availableModels, setAvailableModels] = useState([]); + const [validTasks, setValidTasks] = useState([]); + const [validationErrors, setValidationErrors] = useState({}); + + const [datasetInfo, setDatasetInfo] = useState({}); + const [infoLoading, setInfoLoading] = useState(true); + + const { enqueueSnackbar } = useSnackbar(); + const hasWarnedRef = useRef(false); + + const splitDataNode = prevNodes?.find( + (node) => node?.input_columns || node?.splits, + ); + const datasetNode = prevNodes?.find((node) => node?.file_path && node?.id); + const datasetId = datasetNode?.id ?? null; + const inputColumnsRaw = splitDataNode?.input_columns; + const outputColumnsRaw = splitDataNode?.output_columns; + + useEffect(() => { + if (!datasetId) { + setInfoLoading(false); + if (!hasWarnedRef.current) { + enqueueSnackbar("Missing dataset – connect a SplitData node first.", { + variant: "warning", + }); + hasWarnedRef.current = true; + } + return; + } + const fetchInfo = async () => { + setInfoLoading(true); + try { + const info = await getDatasetInfoRequest(datasetId); + setDatasetInfo(info); + } catch (error) { + enqueueSnackbar("Error fetching dataset info.", { variant: "error" }); + console.error(error); + } finally { + setInfoLoading(false); + } + }; + fetchInfo(); + }, [datasetId]); + + useEffect(() => { + const fetchComponents = async () => { + try { + const tasks = await getComponentsRequest({ selectTypes: ["Task"] }); + const models = await getComponentsRequest({ + selectTypes: ["Model"], + relatedComponent: task, + }); + setAvailableTasks(tasks); + setAvailableModels(models); + } catch (error) { + console.error("Error fetching components:", error); + } + }; + fetchComponents(); + }, [task]); + + const { defaultValues, modelSchema, yupSchema, loading } = useSchema({ + modelName: model, + }); + + useEffect(() => { + if (savedConfig) { + setTask(savedConfig.task || ""); + setModel(savedConfig.model || ""); + setModelParams(savedConfig.parameters || {}); + } + }, [savedConfig]); + + useEffect(() => { + setInitialized(false); + }, [model]); + + useEffect(() => { + if (loading || initialized) return; + if ( + savedConfig?.model === model && + savedConfig?.parameters && + Object.keys(savedConfig.parameters).length > 0 + ) { + setModelParams(savedConfig.parameters); + } else if (defaultValues && Object.keys(defaultValues).length > 0) { + setModelParams(defaultValues); + } else { + setModelParams({}); + } + setInitialized(true); + }, [loading, model, defaultValues, savedConfig, initialized]); + + const handleChange = (newValues) => setModelParams(newValues); + + useEffect(() => { + const validateAvailableTasks = async () => { + if (!datasetId || !inputColumnsRaw || !outputColumnsRaw) { + setValidTasks([]); + return; + } + + const maxValue = datasetInfo?.total_columns; + const columnNames = datasetInfo?.column_names || []; + + // inputColumnsRaw may be an array of indices already (from SplitData) + let inputColumnNames; + let outputColumnNames; + if (Array.isArray(inputColumnsRaw)) { + inputColumnNames = inputColumnsRaw.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + outputColumnNames = (outputColumnsRaw || []).map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + } else { + try { + const parsedInput = parseRangeToIndex( + String(inputColumnsRaw), + maxValue, + ); + const parsedOutput = parseRangeToIndex( + String(outputColumnsRaw), + maxValue, + ); + inputColumnNames = parsedInput.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + outputColumnNames = parsedOutput.map( + (idx) => columnNames[idx - 1] || `Column_${idx}`, + ); + } catch { + setValidTasks([]); + return; + } + } + + const results = await Promise.all( + availableTasks.map(async (taskObj) => { + try { + const validation = await validateColumnsRequest( + taskObj.name, + datasetId, + inputColumnNames, + outputColumnNames, + ); + if (validation.dataset_status === "valid") { + return { name: taskObj.name, valid: true }; + } + return { + name: taskObj.name, + valid: false, + error: validation.error, + }; + } catch (error) { + return { + name: taskObj.name, + valid: false, + error: "Request failed", + }; + } + }), + ); + + const validList = results.filter((r) => r.valid).map((r) => r.name); + const errorsMap = {}; + results.forEach((r) => { + if (!r.valid) errorsMap[r.name] = r.error; + }); + setValidationErrors(errorsMap); + setValidTasks(validList); + }; + + validateAvailableTasks(); + }, [ + availableTasks, + inputColumnsRaw, + outputColumnsRaw, + datasetId, + datasetInfo, + ]); + + const handleSave = async () => { + const payload = { + task, + model, + parameters: modelParams, + }; + + try { + const response = await validateNode("TaskAndModel", payload); + if (response.status === "ok") { + onSave(payload); + onClose(); + } else { + enqueueSnackbar(response.message || "Validation failed", { + variant: "error", + }); + } + } catch (e) { + enqueueSnackbar("Error validating node", { variant: "error" }); + console.error(e); + } + }; + + return ( + <> + + + + + + Task and Model + + + + + + {validTasks.length === 0 && + Object.keys(validationErrors).length > 0 && ( + + + ⚠️ No compatible tasks available. Validation errors: + + {Object.entries(validationErrors).map(([taskName, error]) => ( + + {taskName}: {error} + + ))} + + )} + + + + { + setTask(e.target.value); + setModel(""); + setModelParams({}); + }} + margin="normal" + disabled={validTasks.length === 0} + slotProps={{ + select: { + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, + }, + }, + }, + }, + }} + sx={{ + width: "100%", + minWidth: 0, + display: "block", + "& .MuiInputBase-root": { + width: "100%", + minHeight: 58, + }, + "& .MuiOutlinedInput-root": { + width: "100%", + }, + "& .MuiInputBase-input": { + width: "100%", + }, + "& .MuiSelect-select": { + whiteSpace: "normal", + minHeight: "1.4375em !important", + display: "flex", + alignItems: "center", + }, + }} + > + {availableTasks.map((taskObj) => ( + + {taskObj.name} + + ))} + + + + + + + setModel(e.target.value)} + margin="normal" + disabled={!task} + slotProps={{ + select: { + MenuProps: { + PaperProps: { + sx: { + maxHeight: 420, + }, + }, + }, + }, + }} + sx={{ + width: "100%", + minWidth: 0, + display: "block", + "& .MuiInputBase-root": { + width: "100%", + minHeight: 58, + }, + "& .MuiOutlinedInput-root": { + width: "100%", + }, + "& .MuiInputBase-input": { + width: "100%", + }, + "& .MuiSelect-select": { + whiteSpace: "normal", + minHeight: "1.4375em !important", + display: "flex", + alignItems: "center", + }, + }} + > + {availableModels.map((m) => ( + + {m.name} + + ))} + + + + + + + + + + + setOpenSettings(false)} + /> + + ); +}; + +TaskAndModelNode.propTypes = { + open: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + onSave: PropTypes.func.isRequired, + savedConfig: PropTypes.object, + prevNodes: PropTypes.arrayOf(PropTypes.object), +}; + +TaskAndModelNode.defaultProps = { + savedConfig: null, + prevNodes: [], +}; + +export default TaskAndModelNode; diff --git a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx b/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx deleted file mode 100644 index 8a4d43d8d..000000000 --- a/DashAI/front/src/components/pipelines/nodes/TrainNode.jsx +++ /dev/null @@ -1,506 +0,0 @@ -import React, { useState, useEffect, useRef } from "react"; -import PropTypes from "prop-types"; -import { - Box, - Button, - DialogContent, - Typography, - TextField, - MenuItem, - Grid, - IconButton, - FormControlLabel, - Checkbox, -} from "@mui/material"; -import SettingsIcon from "@mui/icons-material/Settings"; -import useSchema from "../../../hooks/useSchema"; -import ParamsSettings from "../ParamsSettings"; -import { getComponents as getComponentsRequest } from "../../../api/component"; -import { validateColumns as validateColumnsRequest } from "../../../api/modelSession"; -import { useSnackbar } from "notistack"; -import { getDatasetInfo as getDatasetInfoRequest } from "../../../api/datasets"; -import { parseRangeToIndex } from "../../../utils/parseRange"; -import { validateNode } from "../../../api/pipeline"; - -const Train = ({ - open, - onClose, - onSave, - savedConfig = null, - prevNodes = [], -}) => { - const [inputColumns, setInputColumns] = useState( - savedConfig?.input_columns || "", - ); - const [outputColumns, setOutputColumns] = useState( - savedConfig?.output_columns || "", - ); - const [splits, setSplits] = useState( - savedConfig?.splits || { - train: 60, - validation: 20, - test: 20, - shuffle: true, - stratify: false, - splitType: "random", - }, - ); - const [task, setTask] = useState(savedConfig?.task || ""); - const [model, setModel] = useState(savedConfig?.model || ""); - const [metrics, setMetrics] = useState(savedConfig?.metrics || []); - const [openSettings, setOpenSettings] = useState(false); - const [modelParams, setModelParams] = useState(savedConfig?.parameters || {}); - const [initialized, setInitialized] = useState(false); - const [availableTasks, setAvailableTasks] = useState([]); - const [availableModels, setAvailableModels] = useState([]); - const [availableMetrics, setAvailableMetrics] = useState([]); - const { enqueueSnackbar } = useSnackbar(); - const datasetNode = prevNodes?.find((node) => node?.file_path && node?.id); - const datasetId = datasetNode?.id ?? null; - const [validTasks, setValidTasks] = useState([]); - const [datasetInfo, setDatasetInfo] = useState({}); - const [infoLoading, setInfoLoading] = useState(true); - const [inputError, setInputError] = useState(false); - const [outputError, setOutputError] = useState(false); - const [inputErrorMessage, setInputErrorMessage] = useState(""); - const [outputErrorMessage, setOutputErrorMessage] = useState(""); - const [validationStatus, setValidationStatus] = useState(""); - - useEffect(() => { - const fetchComponents = async () => { - try { - const tasks = await getComponentsRequest({ selectTypes: ["Task"] }); - const models = await getComponentsRequest({ - selectTypes: ["Model"], - relatedComponent: task, - }); - const metrics = await getComponentsRequest({ - selectTypes: ["Metric"], - relatedComponent: task, - }); - - setAvailableTasks(tasks); - setAvailableModels(models); - setAvailableMetrics(metrics); - } catch (error) { - console.error("Error fetching components:", error); - } - }; - - fetchComponents(); - }, [task]); - - const { defaultValues, modelSchema, yupSchema, loading } = useSchema({ - modelName: model, - }); - - useEffect(() => { - const total = datasetInfo?.total_columns; - setInputColumns( - savedConfig?.input_columns?.join(",") || (total ? `1-${total - 1}` : ""), - ); - setOutputColumns( - savedConfig?.output_columns?.join(",") || (total ? `${total}` : ""), - ); - setSplits( - savedConfig?.splits || { - train: 0.6, - validation: 0.2, - test: 0.2, - shuffle: true, - stratify: false, - splitType: "random", - }, - ); - setTask(savedConfig?.task || ""); - setModel(savedConfig?.model || ""); - setMetrics(savedConfig?.metrics || []); - setModelParams(savedConfig?.parameters || {}); - }, [savedConfig, datasetInfo]); - - useEffect(() => { - setInitialized(false); - }, [model]); - - useEffect(() => { - if (loading || initialized) return; - - if ( - savedConfig?.model === model && - savedConfig?.parameters && - Object.keys(savedConfig.parameters).length > 0 - ) { - setModelParams(savedConfig.parameters); - } else if (defaultValues && Object.keys(defaultValues).length > 0) { - setModelParams(defaultValues); - } else { - setModelParams({}); - } - - setInitialized(true); - }, [loading, model, defaultValues, savedConfig, initialized]); - - const handleChange = (newValues) => { - setModelParams(newValues); - }; - - const handleSave = async () => { - const maxValue = datasetInfo?.total_columns; - const parsedInput = inputColumns - ? parseRangeToIndex(inputColumns, maxValue) - : []; - const parsedOutput = outputColumns - ? parseRangeToIndex(outputColumns, maxValue) - : []; - const payload = { - input_columns: parsedInput, - output_columns: parsedOutput, - splits: { - train: splits.train, - validation: splits.validation, - test: splits.test, - shuffle: splits.shuffle, - stratify: splits.stratify, - splitType: splits.splitType, - }, - task: task, - model: model, - metrics: metrics, - parameters: modelParams, - }; - - try { - const response = await validateNode("Train", payload); - if (response.status === "ok") { - setValidationStatus("ok"); - onSave(payload); - onClose(); - } else { - setValidationStatus("error"); - enqueueSnackbar("Validation failed", { variant: "error" }); - } - } catch (e) { - setValidationStatus("error"); - enqueueSnackbar("Error validating node", { variant: "error" }); - console.error(e); - } - }; - - const getDatasetInfo = async () => { - setInfoLoading(true); - try { - const datasetInfo = await getDatasetInfoRequest(datasetId); - setDatasetInfo(datasetInfo); - } catch (error) { - enqueueSnackbar("Error while trying to obtain the dataset info."); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - } finally { - setInfoLoading(false); - } - }; - - useEffect(() => { - if (!datasetId) { - setInfoLoading(false); - return; - } - getDatasetInfo(); - }, [datasetId]); - - const hasWarnedRef = useRef(false); - - useEffect(() => { - const validateAvailableTasks = async () => { - if (!datasetId) { - if (!hasWarnedRef.current) { - enqueueSnackbar("Missing dataset", { variant: "warning" }); - hasWarnedRef.current = true; - } - setValidTasks([]); - return; - } - - const maxValue = datasetInfo?.total_columns; - let input = []; - let output = []; - let parseError = false; - - try { - input = inputColumns ? parseRangeToIndex(inputColumns, maxValue) : []; - setInputError(false); - setInputErrorMessage(""); - } catch (err) { - setInputError(true); - setInputErrorMessage(err.message); - parseError = true; - } - - try { - output = outputColumns - ? parseRangeToIndex(outputColumns, maxValue) - : []; - setOutputError(false); - setOutputErrorMessage(""); - } catch (err) { - setOutputError(true); - setOutputErrorMessage(err.message); - parseError = true; - } - - if (parseError) { - setValidTasks([]); - return; - } - - const results = await Promise.all( - availableTasks.map(async (taskObj) => { - try { - const validation = await validateColumnsRequest( - taskObj.name, - datasetId, - input, - output, - ); - return validation.dataset_status === "valid" ? taskObj.name : null; - } catch (error) { - console.warn(`Error validating task ${taskObj.name}`, error); - return null; - } - }), - ); - - const valid = results.filter(Boolean); - setValidTasks(valid); - }; - - validateAvailableTasks(); - }, [availableTasks, inputColumns, outputColumns, datasetId, datasetInfo]); - - return ( - <> - - - - Split Data: - - - - setInputColumns(e.target.value)} - margin="normal" - error={inputError} - helperText={inputError ? inputErrorMessage : ""} - /> - setOutputColumns(e.target.value)} - margin="normal" - error={outputError} - helperText={outputError ? outputErrorMessage : ""} - /> - - - - setSplits({ ...splits, shuffle: e.target.checked }) - } - /> - } - label="Shuffle" - sx={{ ml: 2 }} - /> - - setSplits({ ...splits, stratify: e.target.checked }) - } - /> - } - label="Stratify" - /> - - - - - - setSplits({ ...splits, train: parseFloat(e.target.value, 10) }) - } - margin="normal" - /> - - setSplits({ - ...splits, - validation: parseFloat(e.target.value, 10), - }) - } - margin="normal" - /> - - setSplits({ ...splits, test: parseFloat(e.target.value, 10) }) - } - margin="normal" - /> - - - - Task and Model: - - - - { - setTask(e.target.value); - setModel(""); - setMetrics([]); - setModelParams({}); - }} - margin="normal" - > - {availableTasks.map((taskObj) => ( - - {taskObj.name} - - ))} - - - - - - - { - setModel(e.target.value); - }} - margin="normal" - disabled={!task} - > - {availableModels.map((model) => ( - - {model.name} - - ))} - - - - setOpenSettings(true)} - disabled={!model} - aria-label="model settings" - sx={{ mt: "8px" }} - > - - - - - - - - Metrics: - - - - setMetrics(e.target.value)} - margin="normal" - disabled={!task} - slotProps={{ - select: { multiple: true }, - }} - > - {availableMetrics.map((metric) => ( - - {metric.name} - - ))} - - - - - - - - - - setOpenSettings(false)} - /> - - ); -}; - -Train.propTypes = { - open: PropTypes.bool.isRequired, - onClose: PropTypes.func.isRequired, - onSave: PropTypes.func.isRequired, - savedConfig: PropTypes.object, - prevNodes: PropTypes.arrayOf(PropTypes.object), -}; - -export default Train; diff --git a/DashAI/front/src/components/pipelines/results/Results.jsx b/DashAI/front/src/components/pipelines/results/Results.jsx index 4a14450a3..7753edbd0 100644 --- a/DashAI/front/src/components/pipelines/results/Results.jsx +++ b/DashAI/front/src/components/pipelines/results/Results.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from "react"; +import React, { useEffect, useState, useRef, useCallback } from "react"; import { Box, Typography, @@ -9,6 +9,7 @@ import { Tab, Paper, Button, + CircularProgress, } from "@mui/material"; import { ExpandMore, ArrowBackIosNew } from "@mui/icons-material"; import { getPipelineById } from "../../../api/pipeline"; @@ -17,16 +18,147 @@ import PipelineResultsGraphs from "./ResultsGraphs"; import PipelineResultsPrediction from "./ResultsPrediction"; import ResultsTabParameters from "../../../pages/results/components/ResultsTabParameters"; import ResultsExploration from "./ResultsExploration"; +import DatasetSummaryTable from "../../datasets/DatasetSummaryTable"; + +const POLL_INTERVAL_MS = 3000; + +function normalizeNodeResults(raw, valueKey = null) { + if (!raw) { + return []; + } + + if (typeof raw === "string") { + return valueKey + ? [{ node_id: "legacy", [valueKey]: raw }] + : [{ node_id: "legacy", value: raw }]; + } + + if (typeof raw !== "object") { + return []; + } + + const entries = Object.entries(raw); + if (entries.length === 0) { + return []; + } + + const looksGrouped = entries.every( + ([, value]) => value && typeof value === "object" && !Array.isArray(value), + ); + + if (looksGrouped) { + return entries.map(([nodeId, value]) => ({ + node_id: value.node_id || nodeId, + ...value, + })); + } + + return [{ node_id: "legacy", ...raw }]; +} + +function buildBranchResults({ + trainEntries, + taskEntries, + metricsEntries, + predictionEntries, +}) { + const branches = new Map(); + + const ensureBranch = (branchKey) => { + if (!branches.has(branchKey)) { + branches.set(branchKey, { + branchKey, + modelName: "-", + taskName: "-", + datasetName: "Unknown Dataset", + parameters: null, + metrics: null, + prediction: null, + }); + } + return branches.get(branchKey); + }; + + [...trainEntries, ...taskEntries].forEach((entry, idx) => { + const branchKey = entry.model_node_id || entry.node_id || `train-${idx}`; + const branch = ensureBranch(branchKey); + + branch.modelName = entry.model || entry.info || branch.modelName; + branch.taskName = entry.task || branch.task_name || branch.taskName; + branch.datasetName = entry.dataset_name || branch.datasetName; + branch.parameters = entry.parameters ?? branch.parameters; + }); + + metricsEntries.forEach((entry, idx) => { + const branchKey = entry.model_node_id || entry.node_id || `metrics-${idx}`; + const branch = ensureBranch(branchKey); + + branch.datasetName = entry.dataset_name || branch.datasetName; + branch.modelName = entry.model_name || branch.modelName; + branch.taskName = entry.task_name || branch.taskName; + branch.metrics = entry.results || branch.metrics; + }); + + predictionEntries.forEach((entry, idx) => { + const branchKey = + entry.model_node_id || entry.node_id || `prediction-${idx}`; + const branch = ensureBranch(branchKey); + + branch.datasetName = entry.dataset_name || branch.datasetName; + branch.modelName = entry.model_name || branch.modelName; + branch.taskName = entry.task_name || branch.taskName; + branch.prediction = entry.prediction || branch.prediction; + }); + + return Array.from(branches.values()); +} function PipelineResults({ pipelineId, onClose }) { const [results, setResults] = useState(null); const [trainTab, setTrainTab] = useState(0); + const [metricsTab, setMetricsTab] = useState(0); + const [polling, setPolling] = useState(false); + const pollingRef = useRef(null); + + const hasAnyResult = useCallback((data) => { + if (!data) return false; + const hasExpl = + data.exploration && data.exploration !== "No exploration data"; + const hasTr = data.train && Object.keys(data.train).length > 0; + const hasAtomized = + (data.task_and_model && Object.keys(data.task_and_model).length > 0) || + (data.metrics_result && Object.keys(data.metrics_result).length > 0); + const hasPred = data.prediction && data.prediction !== "No prediction data"; + return hasExpl || hasTr || hasAtomized || hasPred; + }, []); useEffect(() => { + let cancelled = false; + const fetchResults = async () => { try { const response = await getPipelineById(pipelineId); + if (cancelled) return; setResults(response); + + if (!hasAnyResult(response)) { + setPolling(true); + if (pollingRef.current) clearInterval(pollingRef.current); + pollingRef.current = setInterval(async () => { + try { + const updated = await getPipelineById(pipelineId); + if (cancelled) return; + setResults(updated); + if (hasAnyResult(updated)) { + clearInterval(pollingRef.current); + pollingRef.current = null; + setPolling(false); + } + } catch (err) { + console.error("Error polling pipeline results:", err); + } + }, POLL_INTERVAL_MS); + } } catch (error) { console.error("Error fetching results:", error); } @@ -35,30 +167,77 @@ function PipelineResults({ pipelineId, onClose }) { if (pipelineId) { fetchResults(); } - }, [pipelineId]); + + return () => { + cancelled = true; + if (pollingRef.current) { + clearInterval(pollingRef.current); + pollingRef.current = null; + } + }; + }, [pipelineId, hasAnyResult]); const handleTabChange = (event, newValue) => { setTrainTab(newValue); }; + const handleMetricsTabChange = (event, newValue) => { + setMetricsTab(newValue); + }; + if (!results) { return Loading results...; } + const taskEntries = normalizeNodeResults(results.task_and_model); + const trainEntries = normalizeNodeResults(results.train); + const metricsEntries = normalizeNodeResults(results.metrics_result); + const predictionEntries = normalizeNodeResults( + results.prediction, + "prediction", + ); + + const branchResults = buildBranchResults({ + trainEntries, + taskEntries, + metricsEntries, + predictionEntries, + }); + const hasExploration = results.exploration && results.exploration !== "No exploration data"; - const hasTrain = results.train && Object.keys(results.train).length > 0; + const hasTrain = + branchResults.filter((branch) => branch.parameters).length > 0; + const hasMetrics = + branchResults.filter((branch) => !!branch.metrics).length > 0; const hasPrediction = - results.prediction && results.prediction !== "No prediction data"; + branchResults.filter((branch) => !!branch.prediction).length > 0; - const paramData = { - parameters: - results.train && results.train.parameters - ? results.train.parameters - : null, - }; + const steps = Array.isArray(results.steps) ? results.steps : []; + const dataSelectorSteps = steps.filter((s) => s.type === "DataSelector"); + const splitDataSteps = steps.filter((s) => s.type === "SplitData"); + const retrieveModelSteps = steps.filter((s) => s.type === "RetrieveModel"); + const totalRows = dataSelectorSteps[0]?.config?.total_rows ?? null; + + const hasDataSelector = dataSelectorSteps.length > 0; + const hasSplitData = splitDataSteps.length > 0; + const hasRetrieveModel = retrieveModelSteps.length > 0; + + const splitSize = (ratio) => + totalRows != null && ratio != null ? Math.round(ratio * totalRows) : null; + + const formatColumns = (cols) => + Array.isArray(cols) && cols.length > 0 ? cols.join(", ") : "-"; - if (!hasExploration && !hasTrain && !hasPrediction) { + if ( + !hasExploration && + !hasTrain && + !hasMetrics && + !hasPrediction && + !hasDataSelector && + !hasSplitData && + !hasRetrieveModel + ) { return ( {onClose && ( @@ -66,13 +245,144 @@ function PipelineResults({ pipelineId, onClose }) { Volver )} - No results available + {polling ? ( + + + + Pipeline is running. Results will appear automatically… + + + ) : ( + No results available + )} ); } return ( + {polling && ( + + + + Pipeline is running. Remaining results will appear automatically… + + + )} + + {hasDataSelector && ( + + }> + Data Selector + + + + {dataSelectorSteps.map((step) => { + const cfg = step.config || {}; + return ( + + + Dataset: {cfg.name || "-"} + + + {cfg.total_rows ?? "?"} rows × {cfg.total_columns ?? "?"}{" "} + columns + {cfg.task ? ` | ${cfg.task}` : ""} + + {cfg.id != null && ( + + )} + + ); + })} + + + + )} + + {hasSplitData && ( + + }> + Split Data + + + + {splitDataSteps.map((step) => { + const cfg = step.config || {}; + const splits = cfg.splits || {}; + return ( + + + Input columns:{" "} + {formatColumns(cfg.input_columns)} + + + Output columns:{" "} + {formatColumns(cfg.output_columns)} + + {["train", "validation", "test"].map((key) => { + const ratio = splits[key]; + const size = splitSize(ratio); + return ( + + {key.charAt(0).toUpperCase() + key.slice(1)}:{" "} + {ratio != null ? `${Math.round(ratio * 100)}%` : "-"} + {size != null ? ` (~${size} rows)` : ""} + + ); + })} + + ); + })} + + + + )} + + {hasRetrieveModel && ( + + }> + Retrieve Model + + + + {retrieveModelSteps.map((step) => { + const cfg = step.config || {}; + return ( + + + {cfg.model || "-"} + + + {cfg.task ? `Task: ${cfg.task}` : ""} + + + Input columns:{" "} + {formatColumns(cfg.input_columns)} + + {cfg.model_path && ( + + {cfg.model_path} + + )} + + ); + })} + + + + )} + {hasExploration && ( }> @@ -98,46 +408,102 @@ function PipelineResults({ pipelineId, onClose }) { Train - - - - - - - - - - {trainTab === 0 && ( - - Model Name - - {results.train.info ?? "-"} + + {branchResults + .filter((branch) => branch.parameters) + .map((branch, branchIndex) => { + const paramData = { parameters: branch.parameters }; + + return ( + + + + Path {branchIndex + 1} | Dataset: {branch.datasetName} + + + {branch.taskName} | {branch.modelName} + + + + + + + + + + {trainTab === 0 && ( + + + Model Name + + + {branch.modelName} + + + )} + {trainTab === 1 && ( + + + + )} + + + ); + })} + + + + )} + + {hasMetrics && ( + + }> + Metrics + + + + {branchResults + .filter((branch) => !!branch.metrics) + .map((branch, branchIndex) => ( + + + + Path {branchIndex + 1} | Dataset: {branch.datasetName} + + + {branch.taskName} | {branch.modelName} - )} - {trainTab === 1 && ( - - - - )} - {trainTab === 2 && ( - - - - )} - {trainTab === 3 && ( - - + + + + + + + + {metricsTab === 0 && ( + + + + )} + {metricsTab === 1 && ( + + + + )} - )} - - + + ))} @@ -149,8 +515,20 @@ function PipelineResults({ pipelineId, onClose }) { Prediction - - + + {branchResults + .filter((branch) => !!branch.prediction) + .map((branch, branchIndex) => ( + + + Path {branchIndex + 1} | Dataset: {branch.datasetName} + + + {branch.taskName} | {branch.modelName} + + + + ))} diff --git a/DashAI/front/src/components/pipelines/results/ResultsExploration.jsx b/DashAI/front/src/components/pipelines/results/ResultsExploration.jsx index 381d10c17..38aaa81a8 100644 --- a/DashAI/front/src/components/pipelines/results/ResultsExploration.jsx +++ b/DashAI/front/src/components/pipelines/results/ResultsExploration.jsx @@ -59,10 +59,36 @@ function Results({ pipelineId }) { return null; }; + const normalizeGroupedResults = (results) => { + if (!results || typeof results !== "object") { + return {}; + } + + const entries = Object.entries(results); + if (entries.length === 0) { + return {}; + } + + const alreadyGrouped = entries.every(([, value]) => Array.isArray(value)); + if (alreadyGrouped) { + return results; + } + + return { + "Unknown Dataset": entries.map(([explorationId, result]) => ({ + ...result, + exploration_id: explorationId, + node_id: "legacy", + })), + }; + }; + if (!explorationResults) { return Loading...; } + const groupedResults = normalizeGroupedResults(explorationResults); + return ( { - setSelectedChart(chartType); - }; - - const handleToggleParameter = (parameter) => { - const updatedParameters = selectedParameters.includes(parameter) - ? selectedParameters.filter((param) => param !== parameter) - : [...selectedParameters, parameter]; - setSelectedParameters(updatedParameters); - }; - - const handleToggleMetrics = () => { - setShowCustomMetrics(!showCustomMetrics); - setSelectedParameters([]); - }; - - useEffect(() => { - const processData = () => { - try { - const phases = Object.keys(metrics); - const metricNames = Object.keys(metrics[phases[0]]); - const tabularMetrics = []; - - phases.forEach((phase) => { - metricNames.forEach((metric) => { - tabularMetrics.push(`${phase} ${metric}`); - }); - }); - - const concatenatedMetrics = Array.from( - new Set(phases.concat(metricNames)), - ); - setConcatenatedMetrics(concatenatedMetrics); - setTabularMetrics(tabularMetrics); - - let parameterIndex = []; - let generalParameters = []; - - if (showCustomMetrics) { - parameterIndex = selectedParameters.map((param) => - tabularMetrics.indexOf(param), - ); - } else { - const criteria = {}; - concatenatedMetrics.forEach((item) => { - criteria[item] = item; - }); - - tabularMetrics.forEach((metric, index) => { - Object.entries(criteria).forEach(([metricName, substring]) => { - if ( - selectedGeneralMetric === metricName && - metric.includes(substring) - ) { - parameterIndex.push(index); - generalParameters.push(metric); - } - }); - }); - } - - const syntheticRun = { - status: 3, - train_metrics: metrics.train, - validation_metrics: metrics.validation, - test_metrics: metrics.test, - }; - - const newFilteredData = [syntheticRun]; - setFilteredDataProcess(newFilteredData); - - const graphsToView = {}; - let pieCounter = 0; - - const numericValues = []; - ["train", "validation", "test"].forEach((split) => { - metricNames.forEach((metric) => { - numericValues.push(metrics[split][metric]); - }); - }); - - const relevantNumericValues = parameterIndex.map( - (index) => numericValues[index], - ); - - graphsMaking( - graphsToView, - syntheticRun, - relevantNumericValues, - showCustomMetrics, - selectedParameters, - generalParameters, - pieCounter, - ); - - const { generalLayout, pieLayout } = layoutMaking( - selectedChart, - graphsToView, - ); - - const graphsToViewKeys = Object.keys(graphsToView); - const radarValues = graphsToView[graphsToViewKeys[0]]; - const barValues = graphsToView[graphsToViewKeys[1]]; - const pieValues = graphsToView[graphsToViewKeys[2]]; +function normalizeSplitMetrics(splitMetrics) { + return splitMetrics && typeof splitMetrics === "object" ? splitMetrics : {}; +} - setChartData({ - generalLayout, - pieLayout, - radarValues, - barValues, - pieValues, - }); - } catch (err) { - enqueueSnackbar("Error processing metrics", { variant: "error" }); - console.error(err); - } - }; +function PipelineResultsGraphs({ metrics }) { + const trainMetrics = normalizeSplitMetrics(metrics?.train); + const validationMetrics = normalizeSplitMetrics(metrics?.validation); + const testMetrics = normalizeSplitMetrics(metrics?.test); + + const hasAnyMetrics = + Object.keys(trainMetrics).length > 0 || + Object.keys(validationMetrics).length > 0 || + Object.keys(testMetrics).length > 0; + + const syntheticRuns = useMemo( + () => [ + { + status: 3, + train_metrics: trainMetrics, + validation_metrics: validationMetrics, + test_metrics: testMetrics, + }, + ], + [trainMetrics, validationMetrics, testMetrics], + ); - processData(); - }, [metrics, selectedParameters, selectedChart]); + if (!hasAnyMetrics) { + return ( + + No metric data available + Please provide valid metrics to render the graphs. + + ); + } - return ( - <> - {filteredDataProcess.length === 0 ? ( - - No metric data available - Please provide valid metrics to render the graphs. - - ) : ( - - )} - - ); + return ; } +PipelineResultsGraphs.propTypes = { + metrics: PropTypes.shape({ + train: PropTypes.object, + validation: PropTypes.object, + test: PropTypes.object, + }), +}; + +PipelineResultsGraphs.defaultProps = { + metrics: {}, +}; + export default PipelineResultsGraphs; diff --git a/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx b/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx index 15e9da9fe..5bac4fd4a 100644 --- a/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx +++ b/DashAI/front/src/components/pipelines/results/ResultsMetrics.jsx @@ -7,6 +7,10 @@ import ResultsTabMetricsToggle from "../../../pages/results/components/ResultsTa function PipelineResultsMetrics({ metricsData = {} }) { const [displaySet, setDisplaySet] = useState("test_metrics"); + // The default parameter only covers `undefined`; guard against an explicit + // `null` so the component never crashes if it is ever rendered without data. + const safeMetricsData = metricsData || {}; + const displaySetMap = { train_metrics: "train", test_metrics: "test", @@ -14,11 +18,11 @@ function PipelineResultsMetrics({ metricsData = {} }) { }; const currentKey = displaySetMap[displaySet]; - const currentMetrics = metricsData[currentKey] || {}; + const currentMetrics = safeMetricsData[currentKey] || {}; - const hasTrainData = !!metricsData.train; - const hasTestData = !!metricsData.test; - const hasValidationData = !!metricsData.validation; + const hasTrainData = !!safeMetricsData.train; + const hasTestData = !!safeMetricsData.test; + const hasValidationData = !!safeMetricsData.validation; return ( @@ -37,12 +41,14 @@ function PipelineResultsMetrics({ metricsData = {} }) { ) : ( Object.entries(currentMetrics).map(([metricName, value]) => ( - - - {metricName} + + + + {metricName} + - - + + {typeof value === "number" ? value.toFixed(4) : value} diff --git a/DashAI/front/src/components/pipelines/results/ResultsPredictionModal.jsx b/DashAI/front/src/components/pipelines/results/ResultsPredictionModal.jsx index e7cb8add5..3b55ef1b6 100644 --- a/DashAI/front/src/components/pipelines/results/ResultsPredictionModal.jsx +++ b/DashAI/front/src/components/pipelines/results/ResultsPredictionModal.jsx @@ -6,6 +6,15 @@ import { Tab, Paper, CircularProgress, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + Grid, + Card, + CardContent, } from "@mui/material"; import { useSnackbar } from "notistack"; import { getPipelinePredictionSummary } from "../../../api/pipeline"; @@ -61,10 +70,143 @@ function PredictionSummary({ predictName }) { )} - {tab === 1 && summary.data_type !== "string" && ( - <>Prediction Summary Tab Placeholder + {tab === 1 && summary.data_type === "regression" && ( + + + Regression Statistics + + + + + + + Mean + + + {summary.statistics?.mean?.toFixed(2) ?? "-"} + + + + + + + + + Median + + + {summary.statistics?.median?.toFixed(2) ?? "-"} + + + + + + + + + Std Deviation + + + {summary.statistics?.std?.toFixed(2) ?? "-"} + + + + + + + + + Minimum + + + {summary.statistics?.min?.toFixed(2) ?? "-"} + + + + + + + + + Maximum + + + {summary.statistics?.max?.toFixed(2) ?? "-"} + + + + + + + + + Total Predictions + + + {summary.total_data_points ?? "-"} + + + + + + + )} + {tab === 1 && summary.data_type === "classification" && ( + + + Class Distribution + + + + + + Class + Occurrences + Percentage + + + + {summary.class_distribution?.map((row) => ( + + {row.Class} + {row.Ocurrences} + {row.Percentage}% + + ))} + +
+ + + )} + {tab === 2 && ( + + + Sample Predictions (First 50) + + + + + + Index + Predicted Value + + + + {summary.sample_data?.map((row) => ( + + {row.id} + + {typeof row.value === "number" + ? row.value.toFixed(2) + : row.value} + + + ))} + +
+ + )} - {tab === 2 && <>Prediction Sample Table Placeholder} diff --git a/DashAI/front/src/components/threeSectionLayout/panels/RightPanel.jsx b/DashAI/front/src/components/threeSectionLayout/panels/RightPanel.jsx index 0a2dfc5a7..439ca2aac 100644 --- a/DashAI/front/src/components/threeSectionLayout/panels/RightPanel.jsx +++ b/DashAI/front/src/components/threeSectionLayout/panels/RightPanel.jsx @@ -1,6 +1,7 @@ import { Box, IconButton } from "@mui/material"; import { ChevronLeft, ChevronRight } from "@mui/icons-material"; import { useThreePanelLayoutContext } from "./ThreePanelLayoutContext"; + export default function RightPanel({ toggleButtonTop = "50%", children, @@ -12,7 +13,21 @@ export default function RightPanel({ isTogglingRight, bindRightResize, handleToggleRight, + isRightOverlayActive, } = useThreePanelLayoutContext(); + + const toggleRightOffset = rightBarVisible + ? isRightOverlayActive + ? 8 + : `calc(${rightBarWidth}% - 9px)` + : 8; + + const panelWidth = rightBarVisible + ? isRightOverlayActive + ? "100%" + : `${rightBarWidth}%` + : "0%"; + return ( <> diff --git a/DashAI/front/src/hooks/usePipelineState.js b/DashAI/front/src/hooks/usePipelineState.js index d29fe770b..e1411ef5a 100644 --- a/DashAI/front/src/hooks/usePipelineState.js +++ b/DashAI/front/src/hooks/usePipelineState.js @@ -1,5 +1,5 @@ -import { useState, useEffect, useMemo } from "react"; -import { useEdgesState, useNodesState } from "reactflow"; +import { useState, useEffect, useMemo, useRef, useCallback } from "react"; +import { addEdge, useEdgesState, useNodesState } from "reactflow"; import { useSnackbar } from "notistack"; import { validatePipeline, @@ -8,10 +8,43 @@ import { getNodeTypesMap, buildNodeHelp, } from "../components/pipelines"; -import { getPipelineById, getPipelines } from "../api/pipeline"; +import { + getPipelineById, + getPipelines, + validateEdge, + getLatestPipelineRun, +} from "../api/pipeline"; import { generateSequentialName } from "../utils/nameGenerator"; import RunPipeline from "../components/pipelines/Run"; +const TWO_OUTPUT_NODE_TYPES = new Set([ + "DataSelector", + "SplitData", + "TaskAndModel", +]); + +const TWO_INPUT_NODE_TYPES = new Set(["MetricsEval"]); + +const normalizeNodeHandles = (nodes = []) => + nodes.map((node) => { + const shouldNormalizeSource = TWO_OUTPUT_NODE_TYPES.has(node.type); + const shouldNormalizeTarget = TWO_INPUT_NODE_TYPES.has(node.type); + + if (!shouldNormalizeSource && !shouldNormalizeTarget) { + return node; + } + + return { + ...node, + ...(shouldNormalizeSource + ? { sourceHandles: Math.max(2, Number(node.sourceHandles) || 0) } + : {}), + ...(shouldNormalizeTarget + ? { targetHandles: Math.max(2, Number(node.targetHandles) || 0) } + : {}), + }; + }); + export function usePipelineState(pipelineId, location, navigate) { const [nodes, setNodes, onNodesChange] = useNodesState([]); const [edges, setEdges, onEdgesChange] = useEdgesState([]); @@ -28,16 +61,115 @@ export function usePipelineState(pipelineId, location, navigate) { const [nodeHelp, setNodeHelp] = useState({}); const [availableNodes, setAvailableNodes] = useState([]); const [nodeTypesMap, setNodeTypesMap] = useState([]); - - // Name validation states const [nameError, setNameError] = useState(false); const [nameErrorMessage, setNameErrorMessage] = useState(""); const [userHasModifiedName, setUserHasModifiedName] = useState(false); const [existingPipelines, setExistingPipelines] = useState([]); const [nodeIdCounter, setNodeIdCounter] = useState(0); + const [isRunning, setIsRunning] = useState(false); + const pollRef = useRef(null); + const finishTimerRef = useRef(null); const { enqueueSnackbar } = useSnackbar(); - // Generate default name for new pipelines + const stopPolling = useCallback(() => { + if (pollRef.current) { + clearInterval(pollRef.current); + pollRef.current = null; + } + if (finishTimerRef.current) { + clearTimeout(finishTimerRef.current); + finishTimerRef.current = null; + } + }, []); + + const applyNodeStatuses = useCallback( + (nodeRuns) => { + const statusById = {}; + (nodeRuns || []).forEach((nr) => { + if (nr?.node_id) statusById[nr.node_id] = nr.status; + }); + setNodes((nds) => + nds.map((n) => ({ + ...n, + data: { ...n.data, status: statusById[n.id] ?? "NOT_STARTED" }, + })), + ); + }, + [setNodes], + ); + + const resetNodeStatuses = useCallback(() => { + setNodes((nds) => + nds.map((n) => ({ ...n, data: { ...n.data, status: "NOT_STARTED" } })), + ); + }, [setNodes]); + + const markAllNodesFinished = useCallback(() => { + setNodes((nds) => + nds.map((n) => ({ ...n, data: { ...n.data, status: "FINISHED" } })), + ); + }, [setNodes]); + + const startRunPolling = useCallback( + (pid, sinceRunId) => { + stopPolling(); + setIsRunning(true); + const poll = async () => { + try { + const run = await getLatestPipelineRun(pid); + if (!run || run.id == null) return; + if (sinceRunId != null && run.id <= sinceRunId) return; + applyNodeStatuses(run.node_runs); + if (run.status === "FINISHED") { + stopPolling(); + setIsRunning(false); + markAllNodesFinished(); + enqueueSnackbar("Pipeline execution finished", { + variant: "success", + }); + finishTimerRef.current = setTimeout(() => { + setActiveTab("results"); + }, 1000); + } else if (run.status === "ERROR") { + stopPolling(); + setIsRunning(false); + enqueueSnackbar(run.error_message || "Pipeline execution failed", { + variant: "error", + }); + } + } catch (e) { + // transient error + } + }; + poll(); + pollRef.current = setInterval(poll, 800); + }, + [ + applyNodeStatuses, + markAllNodesFinished, + enqueueSnackbar, + stopPolling, + setActiveTab, + ], + ); + + useEffect(() => stopPolling, [stopPolling]); + + useEffect(() => { + const since = location.state?.runningSinceRunId; + if (since === undefined || since === null) return; + if (!pipelineId || nodes.length === 0) return; + startRunPolling(Number(pipelineId), since); + navigate(location.pathname, { replace: true, state: {} }); + }, [ + pipelineId, + location.state, + location.pathname, + nodes.length, + startRunPolling, + navigate, + ]); + const { defaultName } = useMemo(() => { if (pipelineId) return { defaultName: null }; @@ -49,7 +181,6 @@ export function usePipelineState(pipelineId, location, navigate) { return result; }, [existingPipelines, pipelineId]); - // Load existing pipelines for name generation useEffect(() => { const fetchPipelines = async () => { try { @@ -65,10 +196,9 @@ export function usePipelineState(pipelineId, location, navigate) { } }, [pipelineId]); - // Load node types and available nodes useEffect(() => { const fetchData = async () => { - const nodes = await getNodeTypes(); + const nodes = normalizeNodeHandles(await getNodeTypes()); setAvailableNodes(nodes); buildNodeHelp(nodes); }; @@ -88,38 +218,28 @@ export function usePipelineState(pipelineId, location, navigate) { [nodeTypesMap], ); - // Set initial name validation and default value for new pipelines useEffect(() => { if (defaultName && !userHasModifiedName && !pipelineId) { - if ( - pipelineName === "undefined" || - pipelineName.startsWith("Pipeline_") - ) { - setPipelineName(defaultName); - setNameError(false); - setNameErrorMessage(""); - } + setPipelineName(defaultName); + setNameError(false); + setNameErrorMessage(""); } else if (pipelineName === "" && userHasModifiedName) { setNameError(true); setNameErrorMessage("Name is required"); } }, [defaultName, pipelineName, userHasModifiedName, pipelineId]); - // Handle active tab changes useEffect(() => { if (location.state?.activeTab) { setActiveTab(location.state.activeTab); } }, [location.state?.activeTab]); - // Load existing pipeline useEffect(() => { if (pipelineId) { (async () => { const pipeline = await getPipelineById(pipelineId); - - // Get node types to assign source/target properties - const nodeTypes = await getNodeTypes(); + const nodeTypes = normalizeNodeHandles(await getNodeTypes()); const loadedNodes = pipeline.steps.map((step, idx) => { const nodeInfo = nodeTypes.find((n) => n.type === step.type); @@ -131,6 +251,8 @@ export function usePipelineState(pipelineId, location, navigate) { label: step.label, source: nodeInfo?.source || false, target: nodeInfo?.target || false, + sourceHandles: nodeInfo?.sourceHandles || 1, + targetHandles: nodeInfo?.targetHandles || 1, onDelete: () => { setNodes((nds) => nds.filter((n) => n.id !== step.id)); setNodeData((prev) => { @@ -161,7 +283,6 @@ export function usePipelineState(pipelineId, location, navigate) { setPipelineName(pipeline.name); setUserHasModifiedName(false); - // Update nodeIdCounter const maxCounter = Math.max( ...loadedNodes.map((node) => { const match = node.id.match(/-(\d+)$/); @@ -174,7 +295,22 @@ export function usePipelineState(pipelineId, location, navigate) { } }, [pipelineId]); - // Validate pipeline + useEffect(() => { + if (!pipelineId) { + setNodes([]); + setEdges([]); + setNodeData({}); + setSelectedNode(null); + setResultId(null); + setValidationErrors({}); + setHoveredNode(null); + setNodeHelp({}); + setNodeIdCounter(0); + setUserHasModifiedName(false); + setActiveTab("flow"); + } + }, [pipelineId]); + useEffect(() => { const validate = async () => { const errors = await validatePipeline(nodes, edges); @@ -183,7 +319,6 @@ export function usePipelineState(pipelineId, location, navigate) { validate(); }, [nodes.length, edges]); - // Update node data with validation info useEffect(() => { setNodes((prevNodes) => prevNodes.map((node) => { @@ -203,6 +338,8 @@ export function usePipelineState(pipelineId, location, navigate) { name: nodeInfo?.name || node.type, source: nodeInfo?.source || false, target: nodeInfo?.target || false, + sourceHandles: nodeInfo?.sourceHandles || 1, + targetHandles: nodeInfo?.targetHandles || 1, type: nodeInfo?.type || node.type, configType: nodeInfo?.configType, configSchema: nodeInfo?.configSchema || null, @@ -212,14 +349,12 @@ export function usePipelineState(pipelineId, location, navigate) { ); }, [validationErrors, nodeData, availableNodes]); - // Event handlers const onDragStart = (event, nodeType) => { setDragging(nodeType); event.dataTransfer.setData("text/plain", nodeType); event.dataTransfer.effectAllowed = "move"; }; - // Handle pipeline name input changes with validation const handlePipelineNameChange = (event) => { setPipelineName(event.target.value); setUserHasModifiedName(true); @@ -246,7 +381,6 @@ export function usePipelineState(pipelineId, location, navigate) { }; const handleRun = async () => { - // Check if name is valid if ( !pipelineName || pipelineName.trim() === "" || @@ -265,6 +399,20 @@ export function usePipelineState(pipelineId, location, navigate) { enqueueSnackbar("Error in pipeline", { variant: "error" }); return; } + + stopPolling(); + resetNodeStatuses(); + + let sinceRunId = 0; + if (pipelineId) { + try { + const prevRun = await getLatestPipelineRun(Number(pipelineId)); + sinceRunId = prevRun?.id ?? 0; + } catch (e) { + sinceRunId = 0; + } + } + const newId = await RunPipeline( sortedNodes, nodeData, @@ -275,8 +423,14 @@ export function usePipelineState(pipelineId, location, navigate) { ); if (newId) { setResultId(newId); - setActiveTab("results"); - navigate(`/app/pipelines/${newId}`); + setActiveTab("flow"); + if (String(newId) !== String(pipelineId)) { + navigate(`/app/pipelines/${newId}`, { + state: { runningSinceRunId: sinceRunId }, + }); + } else { + startRunPolling(Number(newId), sinceRunId); + } } }; @@ -300,7 +454,108 @@ export function usePipelineState(pipelineId, location, navigate) { setNodeHelp(null); }; - // Clean up hovered node when nodes change + const handleEdgeRemove = (event, edge) => { + if (event?.stopPropagation) { + event.stopPropagation(); + } + setEdges((eds) => + eds.filter((e) => { + if (edge?.id && e.id === edge.id) { + return false; + } + return !( + e.source === edge.source && + e.target === edge.target && + e.sourceHandle === edge.sourceHandle && + e.targetHandle === edge.targetHandle + ); + }), + ); + }; + + const handleConnect = async (params) => { + const sourceNode = nodes.find((node) => node.id === params.source); + const targetNode = nodes.find((node) => node.id === params.target); + + if (!sourceNode || !targetNode) { + setEdges((eds) => + addEdge( + { + ...params, + markerEnd: { + type: "arrowclosed", + }, + }, + eds, + ), + ); + return; + } + + const currentOutputs = edges.filter( + (edge) => + edge.source === params.source && + (params.sourceHandle + ? edge.sourceHandle === params.sourceHandle + : true), + ).length; + const currentInputs = edges.filter( + (edge) => + edge.target === params.target && + (params.targetHandle + ? edge.targetHandle === params.targetHandle + : true), + ).length; + + let validation = null; + try { + validation = await validateEdge({ + source: { + nodeId: params.source, + type: sourceNode.type, + port: params.sourceHandle || null, + currentConnections: currentOutputs, + }, + target: { + nodeId: params.target, + type: targetNode.type, + port: params.targetHandle || null, + currentConnections: currentInputs, + }, + }); + } catch (error) { + console.error("Edge validation error:", error); + } + + const shouldWarn = validation && validation.ok === false; + const edgeColor = validation?.style?.edgeColor; + const edgeClass = validation?.style?.edgeClass; + + if (shouldWarn && validation?.message) { + enqueueSnackbar(validation.message, { + variant: validation.severity === "warning" ? "warning" : "error", + }); + } + + setEdges((eds) => + addEdge( + { + ...params, + className: shouldWarn ? edgeClass : undefined, + style: + shouldWarn && edgeColor + ? { stroke: edgeColor, strokeWidth: 2 } + : undefined, + markerEnd: { + type: "arrowclosed", + color: shouldWarn && edgeColor ? edgeColor : undefined, + }, + }, + eds, + ), + ); + }; + useEffect(() => { if ( hoveredNode && @@ -329,6 +584,7 @@ export function usePipelineState(pipelineId, location, navigate) { nodeIdCounter, nameError, nameErrorMessage, + isRunning, // Setters setNodes, @@ -356,6 +612,8 @@ export function usePipelineState(pipelineId, location, navigate) { onNodeMouseEnter, onNodeMouseLeave, onPaneClick, + handleConnect, + handleEdgeRemove, handlePipelineNameChange, }; } diff --git a/DashAI/front/src/hooks/useThreePanelsLayout.js b/DashAI/front/src/hooks/useThreePanelsLayout.js index 273aad5d6..7de93be03 100644 --- a/DashAI/front/src/hooks/useThreePanelsLayout.js +++ b/DashAI/front/src/hooks/useThreePanelsLayout.js @@ -1,4 +1,6 @@ import { useState, useRef, useEffect, useCallback } from "react"; +import { useTheme } from "@mui/material/styles"; +import useMediaQuery from "@mui/material/useMediaQuery"; const DEFAULT_LAYOUT = { leftBarVisible: true, @@ -19,7 +21,11 @@ const readPersisted = (storageKey) => { } }; -export function useThreePanelLayout({ storageKey } = {}) { +export function useThreePanelLayout({ + storageKey, + overlayRightOnCompact = false, + compactBreakpoint = "md", +} = {}) { const initial = readPersisted(storageKey); const [leftBarVisible, setLeftBarVisible] = useState(initial.leftBarVisible); const [rightBarVisible, setRightBarVisible] = useState( @@ -108,13 +114,19 @@ export function useThreePanelLayout({ storageKey } = {}) { }, 300); }, []); + const theme = useTheme(); + const isCompact = useMediaQuery(theme.breakpoints.down(compactBreakpoint)); + const isRightOverlayActive = overlayRightOnCompact && isCompact; + const effectiveRightVisible = isRightOverlayActive ? false : rightBarVisible; + const effectiveRightWidth = isRightOverlayActive ? 0 : rightBarWidth; + const centerWidth = - leftBarVisible && rightBarVisible - ? 100 - leftBarWidth - rightBarWidth + leftBarVisible && effectiveRightVisible + ? 100 - leftBarWidth - effectiveRightWidth : leftBarVisible ? 100 - leftBarWidth - : rightBarVisible - ? 100 - rightBarWidth + : effectiveRightVisible + ? 100 - effectiveRightWidth : 100; return { @@ -123,6 +135,8 @@ export function useThreePanelLayout({ storageKey } = {}) { leftBarWidth, rightBarWidth, centerWidth, + isCompact, + isRightOverlayActive, handleToggleLeft, handleToggleRight, diff --git a/DashAI/front/src/index.css b/DashAI/front/src/index.css index f5ea348a6..0239e91fc 100644 --- a/DashAI/front/src/index.css +++ b/DashAI/front/src/index.css @@ -31,3 +31,18 @@ code { [data-theme="dark"] .updatemenu-item-rect { fill: var(--dashai-plot-menu-highlight, #1f1e1d) !important; } + +.react-flow__edge.edge-error .react-flow__edge-path { + stroke: #d32f2f; + stroke-width: 2; +} + +.react-flow__edge.edge-warning .react-flow__edge-path { + stroke: #f57c00; + stroke-width: 2; +} + +.react-flow__edge.edge-ok .react-flow__edge-path { + stroke: #4caf50; + stroke-width: 2; +} diff --git a/DashAI/front/src/pages/home/Home.jsx b/DashAI/front/src/pages/home/Home.jsx index e3e339ea0..0ccc03a26 100644 --- a/DashAI/front/src/pages/home/Home.jsx +++ b/DashAI/front/src/pages/home/Home.jsx @@ -301,11 +301,11 @@ function Home() { p: 4, display: { xs: "flex", lg: "grid" }, flexDirection: { xs: "column", lg: "row" }, - gridTemplateColumns: { lg: "1fr 1fr" }, + gridTemplateColumns: { lg: "1fr 1fr 1fr" }, gridTemplateRows: { lg: "1fr 1fr" }, gap: { xs: 2, lg: 4 }, height: { xs: "auto", lg: "70%" }, - width: { xs: "100%", lg: "80%" }, + width: { xs: "100%", lg: "90%" }, minHeight: 0, }} > diff --git a/DashAI/front/src/pages/pipelines/NewPipeline.jsx b/DashAI/front/src/pages/pipelines/NewPipeline.jsx index cc6a1b5f5..c5e883ac0 100644 --- a/DashAI/front/src/pages/pipelines/NewPipeline.jsx +++ b/DashAI/front/src/pages/pipelines/NewPipeline.jsx @@ -1,33 +1,88 @@ -import React, { useRef } from "react"; +import React, { useRef, useState, useMemo } from "react"; import { Box, + Button, + Card, + CardActions, + CardContent, + Chip, + Grid, Typography, - Dialog, - DialogTitle, - IconButton, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import CloseIcon from "@mui/icons-material/Close"; import { ReactFlowProvider } from "reactflow"; import CustomLayout from "../../components/custom/CustomLayout"; +import ModuleContainer from "../../components/layout/ModuleContainer"; +import LeftPanel from "../../components/threeSectionLayout/panels/LeftPanel"; +import CenterPanel from "../../components/threeSectionLayout/panels/CenterPanel"; +import RightPanel from "../../components/threeSectionLayout/panels/RightPanel"; +import { ThreePanelLayoutContext } from "../../components/threeSectionLayout/panels/ThreePanelLayoutContext"; +import OptionBox from "../../components/threeSectionLayout/OptionBox"; import { Results as PipelineResults } from "../../components/pipelines"; import { useParams, useNavigate, useLocation } from "react-router-dom"; import { usePipelineState } from "../../hooks/usePipelineState"; import { useConnectedNodeData } from "../../hooks/useConnectedNodeData"; +import { useThreePanelLayout } from "../../hooks/useThreePanelsLayout"; import { PipelineHeader, PipelineToolbar, PipelineDesigner, NodeSidebar, + PipelineHistorySidebar, + PipelineNodeConfigSidebar, nodeRegistry, } from "../../components/pipelines"; +const PIPELINE_TEMPLATES = [ + { + id: "data_exploration", + name: "Data Exploration", + steps: ["DataSelector", "DataExploration"], + labels: ["Data Selector", "Data Exploration"], + }, + { + id: "train_model", + name: "Train Model", + steps: [ + "DataSelector", + "SplitData", + "TaskAndModel", + "MetricsEval", + "Prediction", + ], + labels: [ + "Data Selector", + "Split Data", + "Task & Model", + "Metrics", + "Prediction", + ], + }, + { + id: "retrieve_model", + name: "Retrieve Model", + steps: ["DataSelector", "RetrieveModel", "Prediction"], + labels: ["Data Selector", "Retrieve Model", "Prediction"], + }, +]; + function NewPipeline() { const theme = useTheme(); const location = useLocation(); const { pipelineId } = useParams(); const navigate = useNavigate(); + const isNewPipelineRoute = location.pathname.endsWith("/new"); + const isWorkspaceOpen = Boolean(pipelineId) || isNewPipelineRoute; const flowWrapperRef = useRef(null); + const [canvasMode, setCanvasMode] = useState("dark"); + const [centerMenuView, setCenterMenuView] = useState("home"); + const threePanelLayout = useThreePanelLayout({ + initialLeftWidth: 20, + initialRightWidth: 20, + overlayRightOnCompact: true, + compactBreakpoint: "md", + }); const { // State @@ -47,6 +102,7 @@ function NewPipeline() { nodeIdCounter, nameError, nameErrorMessage, + isRunning, // Setters setNodes, @@ -74,12 +130,158 @@ function NewPipeline() { onNodeMouseEnter, onNodeMouseLeave, onPaneClick, + handleConnect, + handleEdgeRemove, handlePipelineNameChange, } = usePipelineState(pipelineId, location, navigate); const { getConnectedNodeData } = useConnectedNodeData(nodes, nodeData, edges); - const renderNodeDialogContent = () => { + // Contextual palette highlighting to guide pipeline construction (never + // restricts what can be dragged): + // - a node is focused → emphasize the node types that can come after it; + // - the canvas is empty → suggest the root node(s) as the starting point. + const nodeHighlight = useMemo(() => { + const focusedType = nodeHelp?.type || nodeHelp?.data?.type; + const focusedContract = focusedType + ? availableNodes.find((n) => n.type === focusedType) + : null; + + if (focusedContract) { + return { + highlightMode: "successors", + highlightedTypes: focusedContract.successors || [], + highlightLabel: `Puede ir después de: ${ + focusedContract.name || focusedType + }`, + }; + } + + if (nodes.length === 0 && availableNodes.length > 0) { + const roots = availableNodes.filter((n) => n.source && !n.target); + return { + highlightMode: "start", + highlightedTypes: roots.map((n) => n.type), + highlightLabel: "Selecciona y arrastra para comenzar tu pipeline", + }; + } + + return { highlightMode: null, highlightedTypes: [], highlightLabel: "" }; + }, [nodeHelp, availableNodes, nodes.length]); + + const handleApplyTemplate = (templateId) => { + const selectedTemplate = PIPELINE_TEMPLATES.find( + (t) => t.id === templateId, + ); + if (!selectedTemplate) return; + + if (!isWorkspaceOpen) { + navigate("/app/pipelines/new", { + state: { initialTemplateId: templateId }, + }); + return; + } + + const startX = 120; + const startY = 170; + const spacingX = 230; + + const createdNodes = selectedTemplate.steps.map((nodeType, index) => { + const nodeInfo = availableNodes.find((node) => node.type === nodeType); + const nodeId = `${nodeType}-${index}`; + + return { + id: nodeId, + type: nodeType, + position: { + x: startX + index * spacingX, + y: startY, + }, + data: { + label: nodeInfo?.name || nodeType, + source: nodeInfo?.source || false, + target: nodeInfo?.target || false, + sourceHandles: nodeInfo?.sourceHandles || 1, + targetHandles: nodeInfo?.targetHandles || 1, + canvasMode, + onDelete: () => { + setNodes((nds) => nds.filter((n) => n.id !== nodeId)); + setNodeData((prev) => { + const newData = { ...prev }; + delete newData[nodeId]; + return newData; + }); + setEdges((eds) => + eds.filter( + (edge) => edge.source !== nodeId && edge.target !== nodeId, + ), + ); + }, + }, + sourcePosition: "right", + targetPosition: "left", + }; + }); + + const createdEdges = createdNodes.slice(0, -1).map((node, index) => ({ + id: `e-${node.id}-${createdNodes[index + 1].id}`, + source: node.id, + target: createdNodes[index + 1].id, + markerEnd: { + type: "arrowclosed", + }, + })); + + setSelectedNode(null); + setDragging(null); + setNodeHelp(null); + setHoveredNode(null); + setValidationErrors({}); + setResultId(null); + setNodeData({}); + setNodes(createdNodes); + setEdges(createdEdges); + setNodeIdCounter(createdNodes.length); + setActiveTab("flow"); + }; + + React.useEffect(() => { + const templateIdFromState = location.state?.initialTemplateId; + + if (templateIdFromState && isWorkspaceOpen && availableNodes.length > 0) { + handleApplyTemplate(templateIdFromState); + navigate(location.pathname, { replace: true, state: {} }); + } + }, [ + availableNodes.length, + isWorkspaceOpen, + location.pathname, + location.state, + navigate, + ]); + + React.useEffect(() => { + if (isWorkspaceOpen) { + setCenterMenuView("home"); + } + }, [isWorkspaceOpen]); + + const handleNodeConfigOpen = (event, node) => { + if (!threePanelLayout.rightBarVisible) { + threePanelLayout.handleToggleRight(); + } + onNodeClick(event, node); + }; + + const renderNodeConfigContent = () => { + if (activeTab !== "flow") { + return ( + + Switch to the Flow tab to configure nodes. + + ); + } + if (!selectedNode) return null; const { type, id, data } = selectedNode; @@ -92,11 +294,18 @@ function NewPipeline() { NodeComponent = nodeRegistry["Configurable"]; } - if (!NodeComponent) return null; + if (!NodeComponent) { + return ( + + This node does not require configuration. + + ); + } return ( handleSaveNodeData(id, data)} savedConfig={nodeData[id]} @@ -107,110 +316,254 @@ function NewPipeline() { }; return ( - + - - - {activeTab === "flow" ? ( - <> - - - - - - - - - - {renderNodeDialogContent() && ( - {}} - disableEscapeKeyDown - maxWidth="md" - fullWidth + + + + + + + + {!isWorkspaceOpen ? ( + + {centerMenuView === "home" ? ( + <> + + Pipelines Module + + + Create and manage your pipelines. + + + + + navigate("/app/pipelines/new")} + /> + + + setCenterMenuView("prefabricated")} + /> + + + + ) : ( + <> + + Choose a Prefabricated Pipeline + + + Select one of the templates to open it in the workspace. + + + + {PIPELINE_TEMPLATES.map((template) => ( + + handleApplyTemplate(template.id)} + > + + + {template.name} + + + {(template.labels || template.steps).join( + " + ", + )} + + + {(template.labels || template.steps).map( + (step) => ( + + ), + )} + + + + + + + + ))} + + + + + + + )} + + ) : ( + - + + + + + {activeTab === "flow" ? ( - - {selectedNode?.type === "Train" - ? "Select Train Parameters" - : selectedNode?.type === "Exploration" - ? "Exploration Configuration" - : `Configure ${selectedNode?.type || "Node"}`} - - + + - - + + + + - - {renderNodeDialogContent()} - + ) : ( + + {resultId ? ( + + ) : ( + No results yet. + )} + + )} + )} - - - ) : ( - - {resultId ? ( - - ) : ( - No results yet. - )} - - )} + + + + + {renderNodeConfigContent()} + + + + ); diff --git a/DashAI/front/src/styles/theme.js b/DashAI/front/src/styles/theme.js index 7cc706fa7..f55b41770 100644 --- a/DashAI/front/src/styles/theme.js +++ b/DashAI/front/src/styles/theme.js @@ -142,6 +142,14 @@ const getTheme = (mode) => ({ mode === "dark" ? "rgba(254,232,255,0.22)" : "rgba(165,77,169,0.22)", coralGlow: mode === "dark" ? "rgba(254,232,255,0.04)" : "rgba(165,77,169,0.04)", + // Pipelines → green + green: mode === "dark" ? "#8FE388" : "#2E7D32", + greenDim: + mode === "dark" ? "rgba(143,227,136,0.12)" : "rgba(46,125,50,0.12)", + greenBorder: + mode === "dark" ? "rgba(143,227,136,0.22)" : "rgba(46,125,50,0.22)", + greenGlow: + mode === "dark" ? "rgba(143,227,136,0.04)" : "rgba(46,125,50,0.04)", }, accentInverse: { diff --git a/DashAI/front/src/types/pipeline.ts b/DashAI/front/src/types/pipeline.ts index c9cb333b1..af746f803 100644 --- a/DashAI/front/src/types/pipeline.ts +++ b/DashAI/front/src/types/pipeline.ts @@ -12,4 +12,32 @@ export interface IPipeline { created: Date; last_modified: Date; steps: IStep[]; + exploration?: Record | string; + train?: + | { + info?: string; + parameters?: Record; + metrics?: Record; + model_path?: string; + input_columns?: string[]; + task?: string; + } + | Record; + task_and_model?: + | { + task?: string; + model?: string; + parameters?: Record; + model_path?: string; + input_columns?: string[]; + } + | Record; + metrics_result?: + | { + metric_names?: string[]; + results?: Record; + } + | Record; + split_data?: Record; + prediction?: string | Record; } diff --git a/pyproject.toml b/pyproject.toml index 6035f7e75..0228f32c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -189,6 +189,11 @@ exclude = [ [tool.ruff.lint.pydocstyle] convention = "numpy" # use numpy-style docstrings. +[tool.ruff.lint.isort] +# `alembic` is the PyPI package; the local `alembic/` migrations directory makes +# import sorting non-deterministic across platforms unless pinned as third-party. +known-third-party = ["alembic"] + [tool.ruff.format] docstring-code-format = true diff --git a/tests/back/migrations/test_add_preprocessing_to_model_session.py b/tests/back/migrations/test_add_preprocessing_to_model_session.py index 6dd1ddf2b..ce2c352c4 100644 --- a/tests/back/migrations/test_add_preprocessing_to_model_session.py +++ b/tests/back/migrations/test_add_preprocessing_to_model_session.py @@ -8,7 +8,12 @@ def test_migration_adds_preprocessing_columns(tmp_path): alembic_cfg = Config("alembic.ini") alembic_cfg.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}") - command.upgrade(alembic_cfg, "head") + # Upgrade to the migration under test rather than to ``head``: once this + # branch is merged into develop the head becomes a merge revision (two + # parents), which makes the relative ``downgrade("-1")`` below ambiguous. + # Targeting the revision keeps the downgrade unambiguous and tests exactly + # this migration's up/down behavior. + command.upgrade(alembic_cfg, "f4a91c62d8e7") engine = sa.create_engine(f"sqlite:///{db_path}") inspector = sa.inspect(engine)