From cf7c8193d0d9d727e66c44f427ba01d50a05bf1d Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Mon, 3 Aug 2026 15:42:53 -0700 Subject: [PATCH] Fix CEL policy YAML parser to use codepoint positions instead of byte positions. PiperOrigin-RevId: 958616629 --- policy/BUILD | 1 + policy/cel_policy_parse_context.cc | 19 ++++++++++-- policy/cel_policy_parse_context.h | 8 +++++ policy/yaml_policy_parser.cc | 49 +++++++++++++++++++++++++++--- policy/yaml_policy_parser_test.cc | 17 +++++++++++ 5 files changed, 88 insertions(+), 6 deletions(-) diff --git a/policy/BUILD b/policy/BUILD index cace76f5b..6a204d07c 100644 --- a/policy/BUILD +++ b/policy/BUILD @@ -84,6 +84,7 @@ cc_library( ":cel_policy_parser", "//common:source", "//internal:status_macros", + "//internal:utf8", "//policy/internal:yaml_string_element_scanner", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/policy/cel_policy_parse_context.cc b/policy/cel_policy_parse_context.cc index 66861d085..a627bfd50 100644 --- a/policy/cel_policy_parse_context.cc +++ b/policy/cel_policy_parse_context.cc @@ -14,12 +14,13 @@ #include "policy/cel_policy_parse_context.h" +#include #include -#include #include #include #include "absl/log/absl_check.h" +#include "common/source.h" #include "policy/cel_policy.h" #include "policy/cel_policy_parse_result.h" @@ -43,7 +44,21 @@ CelPolicyParseResult CelPolicyParseContext::GetResult() { void CelPolicyParseContext::ReportError(CelPolicyElementId element_id, std::string_view message) { - issues_.push_back(CelPolicyIssue(element_id, std::string(message))); + issues_.push_back(CelPolicyIssue(element_id, message)); +} + +SourcePosition CelPolicyParseContext::GetCodepointPosition( + SourcePosition byte_offset) const { + if (byte_offset < 0) { + return -1; + } + if (byte_to_codepoint_mapping_.empty()) { + return byte_offset; + } + if (static_cast(byte_offset) >= byte_to_codepoint_mapping_.size()) { + return byte_to_codepoint_mapping_.back(); + } + return byte_to_codepoint_mapping_[byte_offset]; } } // namespace cel diff --git a/policy/cel_policy_parse_context.h b/policy/cel_policy_parse_context.h index 6482fa1ae..7e9aa3fe3 100644 --- a/policy/cel_policy_parse_context.h +++ b/policy/cel_policy_parse_context.h @@ -20,6 +20,7 @@ #include #include "absl/strings/string_view.h" +#include "common/source.h" #include "policy/cel_policy.h" #include "policy/cel_policy_parse_result.h" @@ -53,11 +54,18 @@ class CelPolicyParseContext { CelPolicyElementId next_element_id() { return next_element_id_++; } + void set_byte_to_codepoint_mapping(std::vector mapping) { + byte_to_codepoint_mapping_ = std::move(mapping); + } + + SourcePosition GetCodepointPosition(SourcePosition byte_offset) const; + private: std::shared_ptr policy_source_; CelPolicyElementId next_element_id_ = 0; std::vector issues_; std::unique_ptr policy_; + std::vector byte_to_codepoint_mapping_; }; } // namespace cel diff --git a/policy/yaml_policy_parser.cc b/policy/yaml_policy_parser.cc index 4b4a37cb9..37dfa1dad 100644 --- a/policy/yaml_policy_parser.cc +++ b/policy/yaml_policy_parser.cc @@ -14,10 +14,12 @@ #include "policy/yaml_policy_parser.h" +#include #include #include #include #include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -25,24 +27,36 @@ #include "absl/strings/string_view.h" #include "common/source.h" #include "internal/status_macros.h" +#include "internal/utf8.h" #include "policy/cel_policy.h" #include "policy/cel_policy_parse_context.h" #include "policy/cel_policy_parse_result.h" #include "policy/cel_policy_parser.h" #include "policy/internal/yaml_string_element_scanner.h" #include "yaml-cpp/exceptions.h" +#include "yaml-cpp/mark.h" #include "yaml-cpp/node/node.h" #include "yaml-cpp/node/parse.h" #include "yaml-cpp/null.h" #include "yaml-cpp/yaml.h" // IWYU pragma: keep namespace cel { +namespace { + +SourcePosition GetMarkCodepointPosition(const CelPolicyParseContext& ctx, + const YAML::Mark& mark) { + if (mark.is_null() || mark.pos < 0) return -1; + return ctx.GetCodepointPosition(mark.pos); +} + +} // namespace CelPolicyElementId YamlPolicyParser::CollectMetadata( CelPolicyParseContext& ctx, const YAML::Node& node) const { CelPolicyElementId element_id = ctx.next_element_id(); if (!node.Mark().is_null()) { - ctx.policy_source().NoteSourcePosition(element_id, node.Mark().pos); + ctx.policy_source().NoteSourcePosition( + element_id, GetMarkCodepointPosition(ctx, node.Mark())); } return element_id; } @@ -62,9 +76,10 @@ std::optional YamlPolicyParser::GetValueString( } if (!node.Mark().is_null() && ctx.policy_source().content() != nullptr) { + SourcePosition codepoint_pos = GetMarkCodepointPosition(ctx, node.Mark()); policy_internal::YamlStringElement element = policy_internal::ScanYamlStringElement( - ctx.policy_source().content()->content(), node.Mark().pos, + ctx.policy_source().content()->content(), codepoint_pos, node.as()); ctx.policy_source().NoteSourcePosition(id, element.starting_position); @@ -87,14 +102,40 @@ absl::Status YamlPolicyParser::ParsePolicy(CelPolicyParseContext& ctx) const { return absl::OkStatus(); } - ctx.policy().set_description(ValueString(-1, source->description())); + // TODO(b/542282964): Fold this mapping into cel::Source decoding happens + // once. std::string text = source->content().ToString(); + std::vector mapping; + mapping.resize(text.size() + 1, 0); + size_t byte_offset = 0; + SourcePosition codepoint = 0; + absl::string_view view = text; + while (!view.empty()) { + auto [code_point, code_units] = cel::internal::Utf8Decode(view); + if (code_units == 0) break; + for (size_t i = 0; i < code_units; ++i) { + if (byte_offset + i < mapping.size()) { + mapping[byte_offset + i] = codepoint; + } + } + byte_offset += code_units; + view.remove_prefix(code_units); + codepoint++; + } + for (size_t i = byte_offset; i < mapping.size(); ++i) { + mapping[i] = codepoint; + } + + ctx.set_byte_to_codepoint_mapping(std::move(mapping)); + + ctx.policy().set_description(ValueString(-1, source->description())); YAML::Node node; try { node = YAML::Load(text); } catch (YAML::Exception& e) { if (!e.mark.is_null()) { - ctx.policy_source().NoteSourcePosition(0, e.mark.pos); + ctx.policy_source().NoteSourcePosition( + 0, GetMarkCodepointPosition(ctx, e.mark)); } ctx.ReportError(0, "Invalid CEL policy YAML syntax"); return absl::OkStatus(); diff --git a/policy/yaml_policy_parser_test.cc b/policy/yaml_policy_parser_test.cc index 4e7dfc49c..f32b7f53a 100644 --- a/policy/yaml_policy_parser_test.cc +++ b/policy/yaml_policy_parser_test.cc @@ -150,6 +150,12 @@ TEST_P(YamlPolicyParseErrorTest, YamlSyntaxError) { std::vector GetParseTestCases() { return { + ParseTestCase{ + .yaml = "name: \"unclosed", + .expected_error = "1:16: Invalid CEL policy YAML syntax\n" + " | name: \"unclosed\n" + " | ...............^", + }, ParseTestCase{ .yaml = R"yaml( ? [ John, Doe ]: age: 30 )yaml", .expected_error = "1:22: Invalid CEL policy YAML syntax\n" @@ -198,6 +204,17 @@ std::vector GetParseTestCases() { " | - cel.expr.conformance\n" " | ....................^", }, + ParseTestCase{ + .yaml = R"yaml( + # Comment with multi-byte char: € + imports: + - name: + - cel.expr.conformance + )yaml", + .expected_error = "5:21: Import name is not a string\n" + " | - cel.expr.conformance\n" + " | ....................^", + }, ParseTestCase{ .yaml = R"yaml( rule: do something