fix: Add pre-validation for Bedrock and MLflow role permissions

sagemaker-core/src/sagemaker/core/helper/iam_policies.py

@@ -812,6 +812,157 @@
             },
         },
     },
+    "model_eval": {
+        "role_name": "SageMaker-AutoRole-Evaluation",
+        "trust_policy": {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Principal": {
+                        "Service": "sagemaker.amazonaws.com"
+                    },
+                    "Action": "sts:AssumeRole",
+                    "Condition": {
+                        "StringEquals": {"aws:SourceAccount": "ACCOUNT_PLACEHOLDER"}
+                    },
+                }
+            ],
+        },
+        "policies": {
+            # --- Training permissions (superset of "training" role type) ---
+            "s3_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "s3:GetObject",
+                            "s3:PutObject",
+                            "s3:ListBucket",
+                            "s3:GetBucketLocation",
+                        ],
+                        "Resource": "S3_PLACEHOLDER",
+                    }
+                ],
+            },
+            "ecr_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": ["ecr:GetAuthorizationToken"],
+                        "Resource": "*",
+                    },
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "ecr:GetDownloadUrlForLayer",
+                            "ecr:BatchGetImage",
+                            "ecr:BatchCheckLayerAvailability",
+                        ],
+                        "Resource": "arn:aws:ecr:*:*:repository/*",
+                    },
+                ],
+            },
+            "cloudwatch_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": ["cloudwatch:PutMetricData"],
+                        "Resource": "*",
+                    },
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "logs:CreateLogGroup",
+                            "logs:CreateLogStream",
+                            "logs:PutLogEvents",
+                            "logs:DescribeLogStreams",
+                        ],
+                        "Resource": "arn:aws:logs:*:*:log-group:/aws/sagemaker/TrainingJobs*",
+                    },
+                ],
+            },
+            "kms_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": ["kms:Encrypt", "kms:Decrypt", "kms:GenerateDataKey"],
+                        "Resource": "KMS_PLACEHOLDER",
+                    }
+                ],
+            },
+            # --- Evaluation-specific permissions ---
+            "bedrock_evaluation_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "bedrock:CreateEvaluationJob",
+                            "bedrock:GetEvaluationJob",
+                        ],
+                        "Resource": "*",
+                    },
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "bedrock:InvokeModel",
+                            "bedrock:InvokeModelWithResponseStream",
+                        ],
+                        "Resource": [
+                            "arn:aws:bedrock:*:*:foundation-model/*",
+                            "arn:aws:bedrock:*::foundation-model/*",
+                        ],
+                    },
+                ],
+            },
+            "mlflow_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "sagemaker-mlflow:GetExperimentByName",
+                            "sagemaker-mlflow:CreateExperiment",
+                            "sagemaker-mlflow:CreateRun",
+                            "sagemaker-mlflow:LogBatch",
+                            "sagemaker-mlflow:LogMetric",
+                            "sagemaker-mlflow:LogParam",
+                            "sagemaker-mlflow:SetTag",
+                            "sagemaker-mlflow:UpdateRun",
+                        ],
+                        "Resource": "arn:aws:sagemaker:*:*:mlflow-app/*",
+                    }
+                ],
+            },
+            "sagemaker_evaluation_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "sagemaker:CreateTrainingJob",
+                            "sagemaker:DescribeTrainingJob",
+                            "sagemaker:StopTrainingJob",
+                            "sagemaker:CreatePipeline",
+                            "sagemaker:DescribePipeline",
+                            "sagemaker:StartPipelineExecution",
+                            "sagemaker:DescribePipelineExecution",
+                            "sagemaker:AddTags",
+                        ],
+                        "Resource": [
+                            "arn:aws:sagemaker:*:*:training-job/*",
+                            "arn:aws:sagemaker:*:*:pipeline/*",
+                        ],
+                    }
+                ],
+            },
+        },
+    },
 }
 
 # Actions the *caller* must have to orchestrate Pipeline-based evaluations

sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py

@@ -25,7 +25,7 @@
 
 logger = logging.getLogger(__name__)
 
-ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod")
+ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod", "model_eval")
 
 # Permissions the HyperPod CLI flow needs on the *caller* identity — the local
 # principal that runs `hyperpod connect-cluster` and `hyperpod start-job`. The CLI

sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py

@@ -738,14 +738,23 @@ def _generate_default_eval_name(cls, v: Optional[str], values: dict) -> str:
             return f"eval-{model_name}-{short_uuid}"
         return v
 
-    def _get_aws_execution_context(self) -> Dict[str, str]:
+    def _get_aws_execution_context(self, role_type: str = "training") -> Dict[str, str]:
         """Get AWS execution context (role ARN, region, account ID).
 
         Validates both the *execution* role (what the pipeline assumes to run
         training jobs) and the *caller* role (what your identity needs to
         create/start the pipeline). The execution role is validated via
-        :func:`resolve_and_validate_role` with ``role_type="training"``. The
-        caller role is validated via :func:`verify_evaluation_caller_permissions`.
+        :func:`resolve_and_validate_role`. The caller role is validated via
+        :func:`verify_evaluation_caller_permissions`.
+
+        Args:
+            role_type (str): Role type to validate the execution role against.
+                Defaults to ``"training"`` — every evaluator runs as a SageMaker
+                Pipeline / Training Job, so the standard training permissions are
+                the right smoke test. Only :class:`LLMAsJudgeEvaluator` passes
+                ``"model_eval"`` to additionally gate the Amazon Bedrock
+                Evaluations permissions it requires; the other evaluators do not
+                use Bedrock Evaluations and must not be gated on it.
 
         Returns:
             dict: Dictionary containing:
@@ -757,13 +766,13 @@ def _get_aws_execution_context(self) -> Dict[str, str]:
         #   1. self.role, if explicitly provided.
         #   2. Otherwise the caller's own identity role.
         # The resolved role is validated (read-only) for the required permissions.
-        # This is the job execution role for the
-        # serverless / SMTJ evaluation backends. The HyperPod backend submits via
-        # the CLI under the caller's own credentials (see _submit_hyperpod_eval_job)
-        # and does not resolve a role here, so "training" is always correct here.
+        # This is the job execution role for the serverless / SMTJ evaluation
+        # backends. The HyperPod backend submits via the CLI under the caller's own
+        # credentials (see _submit_hyperpod_eval_job) and does not resolve a role
+        # here.
         role_arn = resolve_and_validate_role(
             provided_role=self.role,
-            role_type="training",
+            role_type=role_type,
             sagemaker_session=self.sagemaker_session,
         )
 

sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py

@@ -753,7 +753,7 @@ def _get_llmaj_template_additions(self, eval_name: str) -> dict:
             'top_p': str(1.0),
             'evaluate_base_model': self.evaluate_base_model,
         }
-    
+
     @_telemetry_emitter(
         feature=Feature.MODEL_CUSTOMIZATION,
         func_name="LLMAsJudgeEvaluator.evaluate",
@@ -819,8 +819,11 @@ def evaluate(self, dry_run: bool = False):
                 "Please use a Model Package ARN or JumpStart model ID instead."
             )
 
-        # Get AWS execution context (role ARN, region, account ID)
-        aws_context = self._get_aws_execution_context()
+        # Get AWS execution context (role ARN, region, account ID).
+        # LLM-as-Judge is the only evaluator backed by Amazon Bedrock Evaluations,
+        # so it validates the execution role against the "model_eval" role type,
+        # which additionally gates the required Bedrock permissions.
+        aws_context = self._get_aws_execution_context(role_type="model_eval")
         region = aws_context['region']
         role_arn = aws_context['role_arn']
 

sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py

@@ -787,6 +787,35 @@ def test_get_aws_execution_context_without_region(self, mock_resolve, mock_role,
         assert context['region'] == DEFAULT_REGION  # From mock_session
         assert context['account_id'] == '123456789012'
 
+    @patch("sagemaker.train.evaluate.base_evaluator.resolve_and_validate_role")
+    @patch("sagemaker.train.common_utils.model_resolution._resolve_base_model")
+    def test_get_aws_execution_context_role_type_override(self, mock_resolve, mock_role, mock_session, mock_model_info):
+        """An explicit role_type (e.g. LLM-as-Judge's "model_eval") is forwarded.
+
+        Only the Bedrock-backed LLM-as-Judge path passes role_type="model_eval";
+        all other evaluators keep the "training" default so they are not gated on
+        Bedrock permissions they never use.
+        """
+        mock_resolve.return_value = mock_model_info
+        mock_role.return_value = DEFAULT_ROLE_ARN
+
+        evaluator = BaseEvaluator(
+            model=DEFAULT_MODEL,
+            s3_output_path=DEFAULT_S3_OUTPUT,
+            mlflow_resource_arn=DEFAULT_MLFLOW_ARN,
+            model_package_group=DEFAULT_MODEL_PACKAGE_GROUP_ARN,
+            sagemaker_session=mock_session,
+            region=DEFAULT_REGION,
+        )
+
+        evaluator._get_aws_execution_context(role_type="model_eval")
+
+        mock_role.assert_called_once_with(
+            provided_role=None,
+            role_type="model_eval",
+            sagemaker_session=mock_session,
+        )
+
 
 class TestTemplateRendering:
     """Tests for template selection and rendering."""