diff --git a/sagemaker-core/src/sagemaker/core/helper/iam_policies.py b/sagemaker-core/src/sagemaker/core/helper/iam_policies.py index 2613418ea8..4384005144 100644 --- a/sagemaker-core/src/sagemaker/core/helper/iam_policies.py +++ b/sagemaker-core/src/sagemaker/core/helper/iam_policies.py @@ -812,6 +812,157 @@ }, }, }, + "model_eval": { + "role_name": "SageMaker-AutoRole-Evaluation", + "trust_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "sagemaker.amazonaws.com" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": {"aws:SourceAccount": "ACCOUNT_PLACEHOLDER"} + }, + } + ], + }, + "policies": { + # --- Training permissions (superset of "training" role type) --- + "s3_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket", + "s3:GetBucketLocation", + ], + "Resource": "S3_PLACEHOLDER", + } + ], + }, + "ecr_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["ecr:GetAuthorizationToken"], + "Resource": "*", + }, + { + "Effect": "Allow", + "Action": [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + ], + "Resource": "arn:aws:ecr:*:*:repository/*", + }, + ], + }, + "cloudwatch_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["cloudwatch:PutMetricData"], + "Resource": "*", + }, + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams", + ], + "Resource": "arn:aws:logs:*:*:log-group:/aws/sagemaker/TrainingJobs*", + }, + ], + }, + "kms_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:Encrypt", "kms:Decrypt", "kms:GenerateDataKey"], + "Resource": "KMS_PLACEHOLDER", + } + ], + }, + # --- Evaluation-specific permissions --- + "bedrock_evaluation_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:CreateEvaluationJob", + "bedrock:GetEvaluationJob", + ], + "Resource": "*", + }, + { + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + ], + "Resource": [ + "arn:aws:bedrock:*:*:foundation-model/*", + "arn:aws:bedrock:*::foundation-model/*", + ], + }, + ], + }, + "mlflow_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sagemaker-mlflow:GetExperimentByName", + "sagemaker-mlflow:CreateExperiment", + "sagemaker-mlflow:CreateRun", + "sagemaker-mlflow:LogBatch", + "sagemaker-mlflow:LogMetric", + "sagemaker-mlflow:LogParam", + "sagemaker-mlflow:SetTag", + "sagemaker-mlflow:UpdateRun", + ], + "Resource": "arn:aws:sagemaker:*:*:mlflow-app/*", + } + ], + }, + "sagemaker_evaluation_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sagemaker:CreateTrainingJob", + "sagemaker:DescribeTrainingJob", + "sagemaker:StopTrainingJob", + "sagemaker:CreatePipeline", + "sagemaker:DescribePipeline", + "sagemaker:StartPipelineExecution", + "sagemaker:DescribePipelineExecution", + "sagemaker:AddTags", + ], + "Resource": [ + "arn:aws:sagemaker:*:*:training-job/*", + "arn:aws:sagemaker:*:*:pipeline/*", + ], + } + ], + }, + }, + }, } # Actions the *caller* must have to orchestrate Pipeline-based evaluations diff --git a/sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py b/sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py index 3baa1205da..eee9b0eed5 100644 --- a/sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py +++ b/sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod") +ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod", "model_eval") # Permissions the HyperPod CLI flow needs on the *caller* identity — the local # principal that runs `hyperpod connect-cluster` and `hyperpod start-job`. The CLI diff --git a/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py b/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py index 928ba678dc..c445f466c7 100644 --- a/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py +++ b/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py @@ -738,14 +738,23 @@ def _generate_default_eval_name(cls, v: Optional[str], values: dict) -> str: return f"eval-{model_name}-{short_uuid}" return v - def _get_aws_execution_context(self) -> Dict[str, str]: + def _get_aws_execution_context(self, role_type: str = "training") -> Dict[str, str]: """Get AWS execution context (role ARN, region, account ID). Validates both the *execution* role (what the pipeline assumes to run training jobs) and the *caller* role (what your identity needs to create/start the pipeline). The execution role is validated via - :func:`resolve_and_validate_role` with ``role_type="training"``. The - caller role is validated via :func:`verify_evaluation_caller_permissions`. + :func:`resolve_and_validate_role`. The caller role is validated via + :func:`verify_evaluation_caller_permissions`. + + Args: + role_type (str): Role type to validate the execution role against. + Defaults to ``"training"`` — every evaluator runs as a SageMaker + Pipeline / Training Job, so the standard training permissions are + the right smoke test. Only :class:`LLMAsJudgeEvaluator` passes + ``"model_eval"`` to additionally gate the Amazon Bedrock + Evaluations permissions it requires; the other evaluators do not + use Bedrock Evaluations and must not be gated on it. Returns: dict: Dictionary containing: @@ -757,13 +766,13 @@ def _get_aws_execution_context(self) -> Dict[str, str]: # 1. self.role, if explicitly provided. # 2. Otherwise the caller's own identity role. # The resolved role is validated (read-only) for the required permissions. - # This is the job execution role for the - # serverless / SMTJ evaluation backends. The HyperPod backend submits via - # the CLI under the caller's own credentials (see _submit_hyperpod_eval_job) - # and does not resolve a role here, so "training" is always correct here. + # This is the job execution role for the serverless / SMTJ evaluation + # backends. The HyperPod backend submits via the CLI under the caller's own + # credentials (see _submit_hyperpod_eval_job) and does not resolve a role + # here. role_arn = resolve_and_validate_role( provided_role=self.role, - role_type="training", + role_type=role_type, sagemaker_session=self.sagemaker_session, ) diff --git a/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py b/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py index 5eed7a2b05..c83a0934ae 100644 --- a/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py +++ b/sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py @@ -753,7 +753,7 @@ def _get_llmaj_template_additions(self, eval_name: str) -> dict: 'top_p': str(1.0), 'evaluate_base_model': self.evaluate_base_model, } - + @_telemetry_emitter( feature=Feature.MODEL_CUSTOMIZATION, func_name="LLMAsJudgeEvaluator.evaluate", @@ -819,8 +819,11 @@ def evaluate(self, dry_run: bool = False): "Please use a Model Package ARN or JumpStart model ID instead." ) - # Get AWS execution context (role ARN, region, account ID) - aws_context = self._get_aws_execution_context() + # Get AWS execution context (role ARN, region, account ID). + # LLM-as-Judge is the only evaluator backed by Amazon Bedrock Evaluations, + # so it validates the execution role against the "model_eval" role type, + # which additionally gates the required Bedrock permissions. + aws_context = self._get_aws_execution_context(role_type="model_eval") region = aws_context['region'] role_arn = aws_context['role_arn'] diff --git a/sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py b/sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py index 5ec2cdaf61..ec737a711e 100644 --- a/sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py +++ b/sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py @@ -787,6 +787,35 @@ def test_get_aws_execution_context_without_region(self, mock_resolve, mock_role, assert context['region'] == DEFAULT_REGION # From mock_session assert context['account_id'] == '123456789012' + @patch("sagemaker.train.evaluate.base_evaluator.resolve_and_validate_role") + @patch("sagemaker.train.common_utils.model_resolution._resolve_base_model") + def test_get_aws_execution_context_role_type_override(self, mock_resolve, mock_role, mock_session, mock_model_info): + """An explicit role_type (e.g. LLM-as-Judge's "model_eval") is forwarded. + + Only the Bedrock-backed LLM-as-Judge path passes role_type="model_eval"; + all other evaluators keep the "training" default so they are not gated on + Bedrock permissions they never use. + """ + mock_resolve.return_value = mock_model_info + mock_role.return_value = DEFAULT_ROLE_ARN + + evaluator = BaseEvaluator( + model=DEFAULT_MODEL, + s3_output_path=DEFAULT_S3_OUTPUT, + mlflow_resource_arn=DEFAULT_MLFLOW_ARN, + model_package_group=DEFAULT_MODEL_PACKAGE_GROUP_ARN, + sagemaker_session=mock_session, + region=DEFAULT_REGION, + ) + + evaluator._get_aws_execution_context(role_type="model_eval") + + mock_role.assert_called_once_with( + provided_role=None, + role_type="model_eval", + sagemaker_session=mock_session, + ) + class TestTemplateRendering: """Tests for template selection and rendering.""" diff --git a/sagemaker-train/tests/unit/train/evaluate/test_bedrock_role_validation.py b/sagemaker-train/tests/unit/train/evaluate/test_bedrock_role_validation.py new file mode 100644 index 0000000000..ebd64ac36b --- /dev/null +++ b/sagemaker-train/tests/unit/train/evaluate/test_bedrock_role_validation.py @@ -0,0 +1,182 @@ +"""Unit tests for the 'evaluation' role type in iam_role_resolver.""" + +from unittest.mock import MagicMock, patch + +from sagemaker.core.helper.iam_role_resolver import ( + resolve_and_validate_role, + _evaluate_permissions, + _role_trusts_service, + _get_smoke_test_actions, + _expected_trust_services, +) + + +class TestEvaluationRoleType: + """Tests for the 'evaluation' role type configuration.""" + + def test_evaluation_role_type_exists(self): + """The 'evaluation' role type should be recognized.""" + from sagemaker.core.helper.iam_policies import IAM_POLICY_CONFIG + assert "model_eval" in IAM_POLICY_CONFIG + + def test_evaluation_trust_includes_sagemaker(self): + """Evaluation role type should require sagemaker.amazonaws.com trust.""" + expected = _expected_trust_services("model_eval") + assert expected == {"sagemaker.amazonaws.com"} + + def test_evaluation_trust_does_not_require_bedrock(self): + """Evaluation role type should NOT require bedrock.amazonaws.com trust. + + The serverless evaluation backend runs as the SageMaker execution role + and calls Bedrock APIs using the role's own credentials. Bedrock does + not need to assume the role, so trust is not required. + """ + expected = _expected_trust_services("model_eval") + bedrock_service = "bedrock" + ".amazonaws.com" + assert bedrock_service not in expected + + def test_evaluation_smoke_actions_include_bedrock(self): + """Smoke test actions should include Bedrock evaluation actions.""" + actions = _get_smoke_test_actions("model_eval") + assert "bedrock:CreateEvaluationJob" in actions + assert "bedrock:GetEvaluationJob" in actions + + def test_evaluation_smoke_actions_exclude_bedrock_invoke(self): + """InvokeModel is resource-scoped, not gated.""" + actions = _get_smoke_test_actions("model_eval") + assert "bedrock:InvokeModel" not in actions + assert "bedrock:InvokeModelWithResponseStream" not in actions + + def test_resolve_raises_when_bedrock_permissions_denied(self): + """Should raise RoleValidationError when Bedrock permissions are denied.""" + mock_iam = MagicMock() + + # Simulate: bedrock:CreateEvaluationJob denied + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "EvaluationResults": [ + {"EvalActionName": "bedrock:CreateEvaluationJob", "EvalDecision": "implicitDeny"}, + {"EvalActionName": "bedrock:GetEvaluationJob", "EvalDecision": "allowed"}, + ] + } + ] + mock_iam.get_paginator.return_value = paginator + + verdict, denied = _evaluate_permissions(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "model_eval") + assert verdict is False + assert "bedrock:CreateEvaluationJob" in denied + + def test_resolve_passes_when_all_allowed(self): + """Should pass when all evaluation permissions are allowed.""" + mock_iam = MagicMock() + + actions = _get_smoke_test_actions("model_eval") + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "EvaluationResults": [ + {"EvalActionName": a, "EvalDecision": "allowed"} for a in actions + ] + } + ] + mock_iam.get_paginator.return_value = paginator + + verdict, denied = _evaluate_permissions(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "model_eval") + assert verdict is True + assert denied == [] + + def test_trust_check_passes_with_sagemaker(self): + """Should pass when trust policy includes sagemaker.amazonaws.com.""" + mock_iam = MagicMock() + mock_iam.get_role.return_value = { + "Role": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "sagemaker.amazonaws.com"}, + "Action": "sts:AssumeRole", + }], + } + } + } + + result = _role_trusts_service(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "model_eval") + assert result is True + + def test_resolve_and_validate_passes_with_sagemaker_trust(self): + """Full resolve_and_validate_role should pass with sagemaker trust only.""" + with patch("sagemaker.core.helper.iam_role_resolver._get_boto_session") as mock_session: + mock_boto = MagicMock() + mock_session.return_value = mock_boto + + mock_iam = MagicMock() + mock_boto.client.return_value = mock_iam + + # Role exists with sagemaker trust only (no bedrock needed) + mock_iam.get_role.return_value = { + "Role": { + "Arn": "arn:aws:iam::123456789012:role/MyRole", + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "sagemaker.amazonaws.com"}, + "Action": "sts:AssumeRole", + }], + } + } + } + + # All permissions allowed + actions = _get_smoke_test_actions("model_eval") + paginator = MagicMock() + paginator.paginate.return_value = [ + {"EvaluationResults": [{"EvalActionName": a, "EvalDecision": "allowed"} for a in actions]} + ] + mock_iam.get_paginator.return_value = paginator + + result = resolve_and_validate_role( + provided_role="arn:aws:iam::123456789012:role/MyRole", + role_type="model_eval", + ) + assert result == "arn:aws:iam::123456789012:role/MyRole" + + def test_resolve_and_validate_passes_with_correct_role(self): + """Full resolve_and_validate_role should pass with correct permissions and trust.""" + with patch("sagemaker.core.helper.iam_role_resolver._get_boto_session") as mock_session: + mock_boto = MagicMock() + mock_session.return_value = mock_boto + + mock_iam = MagicMock() + mock_boto.client.return_value = mock_iam + + # Role exists with correct trust + mock_iam.get_role.return_value = { + "Role": { + "Arn": "arn:aws:iam::123456789012:role/MyRole", + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": ["sagemaker.amazonaws.com", "bedrock.amazonaws.com"]}, + "Action": "sts:AssumeRole", + }], + } + } + } + + # All permissions allowed + actions = _get_smoke_test_actions("model_eval") + paginator = MagicMock() + paginator.paginate.return_value = [ + {"EvaluationResults": [{"EvalActionName": a, "EvalDecision": "allowed"} for a in actions]} + ] + mock_iam.get_paginator.return_value = paginator + + result = resolve_and_validate_role( + provided_role="arn:aws:iam::123456789012:role/MyRole", + role_type="model_eval", + ) + assert result == "arn:aws:iam::123456789012:role/MyRole"