diff --git a/golang/evaluation/RawStringEndToEnd_test.go b/golang/evaluation/RawStringEndToEnd_test.go new file mode 100644 index 0000000..3ee7c0c --- /dev/null +++ b/golang/evaluation/RawStringEndToEnd_test.go @@ -0,0 +1,415 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package evaluation + +import ( + "testing" + + "github.com/stretchr/testify/suite" + "golang.a2z.com/demanddriventrafficevaluator/interfaces" + mockInterfaces "golang.a2z.com/demanddriventrafficevaluator/mocks/interfaces" + "golang.a2z.com/demanddriventrafficevaluator/modelfeature" +) + +// RawStringEndToEndSuite exercises the full evaluation pipeline starting from a raw JSON +// OpenRTB request string. Unlike the other "integration" suites, which hand-build the +// FeatureFieldValueMap, these tests feed the raw string through RequestEvaluator.Evaluate so +// the JSONPath parser (github.com/ohler55/ojg) is exercised end-to-end: raw string → parse → +// JSONPath feature extraction → transform → key building → cache lookup → aggregation → +// filter decision. +// +// Only the local cache is mocked (to control hit/miss); the RuleBasedModelEvaluator and +// ModelResultHandler are real. +type RawStringEndToEndSuite struct { + suite.Suite + mockLocalCacheFactory *mockInterfaces.LocalCacheFactoryInterface + mockDaoFactory *mockInterfaces.DaoFactoryInterface + mockTimeProvider *mockInterfaces.TimeProvider + mockModelConfigHandler *mockInterfaces.ModelConfigurationHandlerInterface + mockTrafficAllocator *mockInterfaces.TrafficAllocatorInterface + mockTrafficAllocationContext *mockInterfaces.TrafficAllocationContextInterface + requestEvaluator *RequestEvaluator +} + +func TestRawStringEndToEndSuite(t *testing.T) { + suite.Run(t, new(RawStringEndToEndSuite)) +} + +func (suite *RawStringEndToEndSuite) SetupTest() { + suite.mockLocalCacheFactory = mockInterfaces.NewLocalCacheFactoryInterface(suite.T()) + suite.mockDaoFactory = mockInterfaces.NewDaoFactoryInterface(suite.T()) + suite.mockTimeProvider = mockInterfaces.NewTimeProvider(suite.T()) + suite.mockModelConfigHandler = mockInterfaces.NewModelConfigurationHandlerInterface(suite.T()) + suite.mockTrafficAllocator = mockInterfaces.NewTrafficAllocatorInterface(suite.T()) + suite.mockTrafficAllocationContext = mockInterfaces.NewTrafficAllocationContextInterface(suite.T()) + + // Real model result handler (only the local cache is mocked) and real rule-based evaluator. + modelResultHandler := modelfeature.NewModelResultHandler( + "ssp", + "./testdata", + suite.mockDaoFactory, + suite.mockModelConfigHandler, + suite.mockLocalCacheFactory, + suite.mockTimeProvider, + ) + ruleBasedEvaluator := NewRuleBasedModelEvaluator(modelResultHandler) + + suite.requestEvaluator = NewRequestEvaluator( + "ssp", + suite.mockTrafficAllocator, + ruleBasedEvaluator, + suite.mockModelConfigHandler, + NewConfigurableAggregator(), + ) +} + +// expectPipelineWiring sets up the traffic-allocation and model-configuration mocks that the +// RequestEvaluator drives for a single-model, max-aggregation (no AggregationSchema) evaluation. +func (suite *RawStringEndToEndSuite) expectPipelineWiring(modelIdentifier string, uniqueFields []string, modelConfiguration interfaces.ModelConfiguration) { + suite.mockTrafficAllocator.EXPECT(). + GetTrafficAllocationContext(). + Return(suite.mockTrafficAllocationContext). + Once() + suite.mockModelConfigHandler.EXPECT(). + GetAllUniqueFeatureFields(). + Return(uniqueFields, nil). + Once() + suite.mockTrafficAllocationContext.EXPECT(). + GetModelIdentifiers(). + Return([]string{modelIdentifier}). + Once() + suite.mockModelConfigHandler.EXPECT(). + Provide(). + Return(&modelConfiguration, nil). + Once() + // Called twice: once in Evaluate() to determine the aggregation path (nil schema → + // max-aggregation), once inside aggregateModelEvaluationResultsOnMax. + suite.mockTrafficAllocationContext.EXPECT(). + GetExperimentDefinitionByType(modelfeature.ExperimentTypeSoftFilter). + Return(&interfaces.ExperimentDefinition{ + Name: ExperimentName, + Type: modelfeature.ExperimentTypeSoftFilter, + AggregationSchema: nil, + }, nil). + Times(2) + suite.mockTrafficAllocationContext.EXPECT(). + GetModelsByExperiment(). + Return(map[string][]string{ExperimentName: {modelIdentifier}}). + Once() + suite.mockTrafficAllocationContext.EXPECT(). + GetTreatmentCodeInInt(ExperimentName). + Return(TreatmentCodeInIntZero). + Once() + suite.mockTrafficAllocationContext.EXPECT(). + GetTreatmentCode(ExperimentName). + Return(TreatmentT). + Once() +} + +// highValueDealModelConfiguration builds a HighValue deal model with a wildcard-extracted +// dealId feature and a scalar publisherId feature. +func highValueDealModelConfiguration(modelIdentifier string) (interfaces.ModelConfiguration, []string) { + modelDefinition := interfaces.ModelDefinition{ + Identifier: modelIdentifier, + Dsp: "adsp", + Name: "high-value-deals", + Version: "v1", + Type: "HighValue", + FeatureExtractorType: "JsonExtractor", + Features: []interfaces.FeatureConfiguration{ + { + Name: "dealId", + Fields: []string{"$.imp[0].pmp.deals[*].id"}, + Transformations: []interfaces.TransformerName{"IncludeDefaultValue"}, + MappingDefaultValue: "no_deal", + }, + { + Name: "publisherId", + Fields: []string{"$.site.publisher.id"}, + Transformations: []interfaces.TransformerName{"GetFirstNotEmpty"}, + }, + }, + } + config := interfaces.ModelConfiguration{ + ModelDefinitionByIdentifier: map[string]interfaces.ModelDefinition{ + modelIdentifier: modelDefinition, + }, + } + uniqueFields := []string{"$.imp[0].pmp.deals[*].id", "$.site.publisher.id"} + return config, uniqueFields +} + +// TestEvaluate_RawJsonString_HighValueDealWildcard_CacheHit_Filters verifies that a raw JSON +// request whose deal IDs are extracted via a [*] wildcard path produces a filter decision of +// 1.0 when one of the permutation keys is a cache hit (HighValue hit value 1.0). +func (suite *RawStringEndToEndSuite) TestEvaluate_RawJsonString_HighValueDealWildcard_CacheHit_Filters() { + const modelIdentifier = "adsp_high-value-deals_v1" + config, uniqueFields := highValueDealModelConfiguration(modelIdentifier) + suite.expectPipelineWiring(modelIdentifier, uniqueFields, config) + + // Raw OpenRTB string with three deals under a wildcard path and a scalar publisher id. + openRtbRequest := `{ + "id": "req-hv-1", + "site": {"publisher": {"id": "pub123"}}, + "imp": [{ + "pmp": { + "deals": [ + {"id": "deal-AAA"}, + {"id": "deal-BBB"}, + {"id": "deal-CCC"} + ] + } + }] + }` + + // dealId after IncludeDefaultValue: [deal-AAA, deal-BBB, deal-CCC, no_deal] + // publisherId after GetFirstNotEmpty: [pub123] + // BuildKeys (Cartesian product): 4 keys. deal-BBB|pub123 is a cache hit at 1.0. + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "deal-AAA|pub123"). + Return(nil, false).Once() + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "deal-BBB|pub123"). + Return(float32(1.0), true).Once() + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "deal-CCC|pub123"). + Return(nil, false).Once() + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "no_deal|pub123"). + Return(nil, false).Once() + + output := suite.requestEvaluator.Evaluate(&BidRequestEvaluatorInput{OpenRtbRequest: openRtbRequest}) + + suite.NotNil(output) + suite.Equal(1, len(output.Response.Slots), "Slots size should be 1") + suite.Equal(float32(1.0), output.Response.Slots[0].FilterDecision, "First cache hit (1.0) should drive the filter decision") + suite.Equal(`{"amazontest":{"decision":1}}`, output.Response.Slots[0].Ext) + suite.Equal(`{"amazontest":{"learning":0}}`, output.Response.Ext) +} + +// TestEvaluate_RawJsonString_HighValueDealWildcard_AllMiss_ReturnsDefault verifies that when +// every permutation key misses the cache, the HighValue default (0.0) becomes the decision. +func (suite *RawStringEndToEndSuite) TestEvaluate_RawJsonString_HighValueDealWildcard_AllMiss_ReturnsDefault() { + const modelIdentifier = "adsp_high-value-deals_v1" + config, uniqueFields := highValueDealModelConfiguration(modelIdentifier) + suite.expectPipelineWiring(modelIdentifier, uniqueFields, config) + + openRtbRequest := `{ + "id": "req-hv-2", + "site": {"publisher": {"id": "pub456"}}, + "imp": [{ + "pmp": { + "deals": [ + {"id": "deal-X"}, + {"id": "deal-Y"} + ] + } + }] + }` + + // BuildKeys: deal-X|pub456, deal-Y|pub456, no_deal|pub456 — all cache misses. + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "deal-X|pub456"). + Return(nil, false).Once() + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "deal-Y|pub456"). + Return(nil, false).Once() + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "no_deal|pub456"). + Return(nil, false).Once() + + output := suite.requestEvaluator.Evaluate(&BidRequestEvaluatorInput{OpenRtbRequest: openRtbRequest}) + + suite.NotNil(output) + suite.Equal(1, len(output.Response.Slots), "Slots size should be 1") + suite.Equal(float32(0.0), output.Response.Slots[0].FilterDecision, "All-miss should yield the HighValue default (0.0)") + suite.Equal(`{"amazontest":{"decision":0}}`, output.Response.Slots[0].Ext) + suite.Equal(`{"amazontest":{"learning":0}}`, output.Response.Ext) +} + +// lowValueScalarModelConfiguration builds a LowValue model using only scalar JSONPath fields. +func lowValueScalarModelConfiguration(modelIdentifier string) (interfaces.ModelConfiguration, []string) { + modelDefinition := interfaces.ModelDefinition{ + Identifier: modelIdentifier, + Dsp: "adsp", + Name: "low-value", + Version: "v2", + Type: "LowValue", + FeatureExtractorType: "JsonExtractor", + Features: []interfaces.FeatureConfiguration{ + { + Name: "publisherId", + Fields: []string{"$.site.publisher.id", "$.app.publisher.id"}, + Transformations: []interfaces.TransformerName{"GetFirstNotEmpty"}, + }, + { + Name: "country", + Fields: []string{"$.device.geo.country"}, + Transformations: []interfaces.TransformerName{}, + }, + { + Name: "deviceType", + Fields: []string{"$.device.devicetype"}, + Transformations: []interfaces.TransformerName{"GetFirstNotEmpty"}, + }, + }, + } + config := interfaces.ModelConfiguration{ + ModelDefinitionByIdentifier: map[string]interfaces.ModelDefinition{ + modelIdentifier: modelDefinition, + }, + } + uniqueFields := []string{ + "$.site.publisher.id", + "$.app.publisher.id", + "$.device.geo.country", + "$.device.devicetype", + } + return config, uniqueFields +} + +// TestEvaluate_RawJsonString_LowValueScalar_CacheHit_Filters verifies a raw JSON request with +// scalar JSONPath fields (including a numeric devicetype) produces a filter decision of 0.0 on +// a cache hit (LowValue hit value 0.0). +func (suite *RawStringEndToEndSuite) TestEvaluate_RawJsonString_LowValueScalar_CacheHit_Filters() { + const modelIdentifier = "adsp_low-value_v2" + config, uniqueFields := lowValueScalarModelConfiguration(modelIdentifier) + suite.expectPipelineWiring(modelIdentifier, uniqueFields, config) + + // devicetype is a JSON number (2); UseNumber preserves it as the string "2" in the key. + openRtbRequest := `{ + "id": "req-lv-1", + "site": {"publisher": {"id": "539014228"}}, + "device": {"geo": {"country": "USA"}, "devicetype": 2} + }` + + // GetFirstNotEmpty(publisherId) → 539014228, country → USA, GetFirstNotEmpty(deviceType) → 2 + // Single permutation key: "539014228|USA|2" — cache hit at 0.0 (LowValue filter). + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "539014228|USA|2"). + Return(float32(0.0), true).Once() + + output := suite.requestEvaluator.Evaluate(&BidRequestEvaluatorInput{OpenRtbRequest: openRtbRequest}) + + suite.NotNil(output) + suite.Equal(1, len(output.Response.Slots), "Slots size should be 1") + suite.Equal(float32(0.0), output.Response.Slots[0].FilterDecision, "LowValue cache hit should filter (0.0)") + suite.Equal(`{"amazontest":{"decision":0}}`, output.Response.Slots[0].Ext) + suite.Equal(`{"amazontest":{"learning":0}}`, output.Response.Ext) +} + +// TestEvaluate_RawJsonString_LowValueScalar_CacheMiss_Forwards verifies a raw JSON request with +// scalar fields produces a forward decision of 1.0 when the key misses the cache (LowValue +// default value 1.0). +func (suite *RawStringEndToEndSuite) TestEvaluate_RawJsonString_LowValueScalar_CacheMiss_Forwards() { + const modelIdentifier = "adsp_low-value_v2" + config, uniqueFields := lowValueScalarModelConfiguration(modelIdentifier) + suite.expectPipelineWiring(modelIdentifier, uniqueFields, config) + + openRtbRequest := `{ + "id": "req-lv-2", + "site": {"publisher": {"id": "pub999"}}, + "device": {"geo": {"country": "CAN"}, "devicetype": 4} + }` + + // Single permutation key "pub999|CAN|4" — cache miss → LowValue default (forward) 1.0. + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "pub999|CAN|4"). + Return(nil, false).Once() + + output := suite.requestEvaluator.Evaluate(&BidRequestEvaluatorInput{OpenRtbRequest: openRtbRequest}) + + suite.NotNil(output) + suite.Equal(1, len(output.Response.Slots), "Slots size should be 1") + suite.Equal(float32(1.0), output.Response.Slots[0].FilterDecision, "LowValue cache miss should forward (1.0)") + suite.Equal(`{"amazontest":{"decision":1}}`, output.Response.Slots[0].Ext) + suite.Equal(`{"amazontest":{"learning":0}}`, output.Response.Ext) +} + +// isMobileModelConfiguration builds a LowValue model with a single "isMobile" feature that +// derives site/app from the presence of $.app via the Exists → ApplyMappings chain. +func isMobileModelConfiguration(modelIdentifier string) (interfaces.ModelConfiguration, []string) { + modelDefinition := interfaces.ModelDefinition{ + Identifier: modelIdentifier, + Dsp: "adsp", + Name: "is-mobile", + Version: "v1", + Type: "LowValue", + FeatureExtractorType: "JsonExtractor", + Features: []interfaces.FeatureConfiguration{ + { + Name: "isMobile", + Fields: []string{"$.app"}, + Transformations: []interfaces.TransformerName{"Exists", "ApplyMappings"}, + Mapping: map[string]string{"0": "site", "1": "app"}, + }, + }, + } + config := interfaces.ModelConfiguration{ + ModelDefinitionByIdentifier: map[string]interfaces.ModelDefinition{ + modelIdentifier: modelDefinition, + }, + } + uniqueFields := []string{"$.app"} + return config, uniqueFields +} + +// TestEvaluate_RawJsonString_IsMobile_AppAbsent_MapsToSite verifies the end-to-end fix: a raw +// request WITHOUT $.app extracts [""] for the definite path, Exists turns it into "0", and +// ApplyMappings resolves it to the "site" lookup key — matching the Java implementation. This +// previously collapsed to no keys (default score) because the missing field yielded []. +func (suite *RawStringEndToEndSuite) TestEvaluate_RawJsonString_IsMobile_AppAbsent_MapsToSite() { + const modelIdentifier = "adsp_is-mobile_v1" + config, uniqueFields := isMobileModelConfiguration(modelIdentifier) + suite.expectPipelineWiring(modelIdentifier, uniqueFields, config) + + // No $.app in the request → definite path $.app resolves to [""] → Exists "0" → "site". + openRtbRequest := `{ + "id": "req-mobile-1", + "site": {"publisher": {"id": "pub123"}} + }` + + // The single lookup key is "site". Cache hit at 0.0 (LowValue filter) proves the key was + // actually built (not collapsed to the empty-key default path). + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "site"). + Return(float32(0.0), true).Once() + + output := suite.requestEvaluator.Evaluate(&BidRequestEvaluatorInput{OpenRtbRequest: openRtbRequest}) + + suite.NotNil(output) + suite.Equal(1, len(output.Response.Slots), "Slots size should be 1") + suite.Equal(float32(0.0), output.Response.Slots[0].FilterDecision, "app-absent should build the 'site' key and filter on hit") + suite.Equal(`{"amazontest":{"decision":0}}`, output.Response.Slots[0].Ext) + suite.Equal(`{"amazontest":{"learning":0}}`, output.Response.Ext) +} + +// TestEvaluate_RawJsonString_IsMobile_AppPresent_MapsToApp verifies that a raw request WITH an +// $.app object extracts a non-empty value, Exists turns it into "1", and ApplyMappings resolves +// it to the "app" lookup key. +func (suite *RawStringEndToEndSuite) TestEvaluate_RawJsonString_IsMobile_AppPresent_MapsToApp() { + const modelIdentifier = "adsp_is-mobile_v1" + config, uniqueFields := isMobileModelConfiguration(modelIdentifier) + suite.expectPipelineWiring(modelIdentifier, uniqueFields, config) + + // $.app present as an object → non-empty extracted value → Exists "1" → "app". + openRtbRequest := `{ + "id": "req-mobile-2", + "app": {"bundle": "com.example.app"}, + "site": {"publisher": {"id": "pub123"}} + }` + + suite.mockLocalCacheFactory.EXPECT(). + GetFromLocalCache(modelIdentifier, "app"). + Return(nil, false).Once() + + output := suite.requestEvaluator.Evaluate(&BidRequestEvaluatorInput{OpenRtbRequest: openRtbRequest}) + + suite.NotNil(output) + suite.Equal(1, len(output.Response.Slots), "Slots size should be 1") + // LowValue cache miss on the "app" key → forward (1.0). + suite.Equal(float32(1.0), output.Response.Slots[0].FilterDecision, "app-present should build the 'app' key and forward on miss") + suite.Equal(`{"amazontest":{"decision":1}}`, output.Response.Slots[0].Ext) + suite.Equal(`{"amazontest":{"learning":0}}`, output.Response.Ext) +} diff --git a/golang/evaluation/RequestEvaluator.go b/golang/evaluation/RequestEvaluator.go index cdec2a1..6f4dace 100644 --- a/golang/evaluation/RequestEvaluator.go +++ b/golang/evaluation/RequestEvaluator.go @@ -11,8 +11,8 @@ import ( "slices" "strings" - "github.com/buger/jsonparser" "github.com/google/uuid" + "github.com/ohler55/ojg/jp" "github.com/rs/zerolog" "golang.a2z.com/demanddriventrafficevaluator/interfaces" "golang.a2z.com/demanddriventrafficevaluator/modelfeature" @@ -252,14 +252,44 @@ func (b *RequestEvaluator) addMissingEntriesToMap(openRtbRequestMap map[string][ if exists && value != nil { fieldValueMap[field] = value } else { - fieldValueMap[field] = []string{} + // A field the SSP did not provide is treated the same as a field that the + // string parser could not resolve: a definite (normal) path yields a single + // empty string [""], an indefinite path yields an empty slice []. This keeps + // the map-input and string-input paths consistent for downstream transformers + // such as Exists. + fieldValueMap[field] = emptyValueForField(field) Logger.Debug().Msgf("field [%v] is not found", field) } } return fieldValueMap, nil } +// emptyValueForField returns the placeholder value used when a field cannot be resolved. +// It mirrors extractField's empty-match behavior so the string-input and map-input code +// paths agree: a definite (normal) JSONPath yields [""] (matching the Java findPath +// contract), while an indefinite path (wildcard, filter, union, slice, descent) yields []. +// A field that is not a valid JSONPath expression is treated as indefinite ([]). +func emptyValueForField(field string) []string { + expr, err := jp.ParseString(field) + if err != nil { + return []string{} + } + if expr.Normal() { + return []string{""} + } + return []string{} +} + // Extract values of all unique fields of all model features. +// +// Values are extracted using JSONPath expressions (via github.com/ohler55/ojg/jp), +// mirroring the Java implementation which relies on Jayway JsonPath configured with +// ALWAYS_RETURN_LIST. Each configured field is a JSONPath expression evaluated against the +// parsed OpenRTB request. A field maps to: +// - a single-element slice for a scalar match (e.g. "$.site.publisher.id"), +// - a multi-element slice for a wildcard/filter match (e.g. "$.imp[0].pmp.deals[*].id"), +// - a single-element slice containing "null" when the field exists but is JSON null, +// - an empty slice when the field does not exist or the path fails to resolve. func (b *RequestEvaluator) parse(openRtbRequest string, externalFields []string) (map[string][]string, error) { uniqueFeatureFields, err := b.modelConfigurationHandler.GetAllUniqueFeatureFields() if err != nil { @@ -268,149 +298,82 @@ func (b *RequestEvaluator) parse(openRtbRequest string, externalFields []string) uniqueFeatureFields = append(uniqueFeatureFields, externalFields...) Logger.Debug().Msgf("uniqueFeatureFields: %v", uniqueFeatureFields) - // Separate wildcard fields (containing [*]) from scalar fields - var scalarFields []string - var wildcardFields []string - for _, field := range uniqueFeatureFields { - if strings.Contains(field, "[*]") { - wildcardFields = append(wildcardFields, field) - } else { - scalarFields = append(scalarFields, field) - } - } - - var fieldValueMap = make(map[string][]string) - - // Process scalar fields using existing EachKey() approach - if len(scalarFields) > 0 { - paths := convertFieldsToPaths(scalarFields) - Logger.Debug().Msgf("paths: %v", paths) - jsonparser.EachKey([]byte(openRtbRequest), func(idx int, value []byte, vt jsonparser.ValueType, err error) { - var str string - switch vt { - case jsonparser.String: - str = string(value) - default: - str = string(value) - } - fieldValueMap[convertPathsToField(paths[idx])] = []string{str} - }, paths...) + // Parse the request once into a generic document. UseNumber preserves the exact + // textual representation of numeric values (e.g. "970", "6.33") instead of coercing + // them through float64 formatting. + document, err := parseJSONDocument(openRtbRequest) + if err != nil { + return nil, fmt.Errorf("fail to parse openRtbRequest as JSON due to %v", err) } - // Process wildcard fields using extractWildcardField - jsonData := []byte(openRtbRequest) - for _, field := range wildcardFields { - fieldValueMap[field] = b.extractWildcardField(jsonData, field) + fieldValueMap := make(map[string][]string, len(uniqueFeatureFields)) + for _, field := range uniqueFeatureFields { + fieldValueMap[field] = b.extractField(document, field) } Logger.Debug().Msgf("fieldValueMap: %v", fieldValueMap) - // Required as JSON Parser forEach doesn't call the iterator function for non-existing keys in JSON - for _, field := range uniqueFeatureFields { - _, exists := fieldValueMap[field] - if !exists { - fieldValueMap[field] = []string{} - Logger.Debug().Msgf("field [%v] is not found", field) - } - } return fieldValueMap, nil } -// extractWildcardField extracts multiple values from a JSON array path containing [*]. -// It splits the path at [*], navigates to the parent array, iterates each element, -// and extracts the suffix field from each element. -func (b *RequestEvaluator) extractWildcardField(jsonData []byte, fieldPath string) []string { - // Split the field path at [*] - parts := strings.SplitN(fieldPath, "[*]", 2) - if len(parts) != 2 { - return []string{} +// parseJSONDocument decodes a raw JSON string into a generic document suitable for +// JSONPath evaluation, preserving numbers as json.Number to retain their exact form. +func parseJSONDocument(rawJSON string) (interface{}, error) { + decoder := json.NewDecoder(strings.NewReader(rawJSON)) + decoder.UseNumber() + var document interface{} + if err := decoder.Decode(&document); err != nil { + return nil, err } + return document, nil +} - prefix := parts[0] // e.g., "$.imp[0].pmp.deals" - suffix := parts[1] // e.g., ".id" - - // Remove leading "." from suffix if present - suffix = strings.TrimPrefix(suffix, ".") - - // Convert prefix to jsonparser path segments - // Remove "$." prefix - prefix = strings.TrimPrefix(prefix, "$.") - - // Parse the prefix path into segments compatible with jsonparser.Get() - prefixSegments := parsePathSegments(prefix) - - // Navigate to the parent array using jsonparser.Get() - arrayData, dataType, _, err := jsonparser.Get(jsonData, prefixSegments...) - if err != nil || dataType != jsonparser.Array { - Logger.Debug().Msgf("wildcard field [%v] prefix path does not resolve to an array: %v", fieldPath, err) +// extractField compiles a single JSONPath expression and evaluates it against the parsed +// document, flattening the matches into a slice of strings. +// +// The empty-match behavior mirrors the Java implementation (Jayway JsonPath with +// ALWAYS_RETURN_LIST as consumed by OpenRtbRequestContextJsonDocument.findPath): +// - A definite (normal) path — only object keys and array indices, e.g. "$.app" or +// "$.site.publisher.id" — that matches nothing is treated like Jayway's +// PathNotFoundException and yields a single empty string [""]. This keeps downstream +// transformers such as Exists deterministic: a missing field becomes "0" rather than +// dropping out of the feature entirely. +// - An indefinite path — containing a wildcard, filter, union, slice, or descent, e.g. +// "$.imp[0].pmp.deals[*].id" — that matches nothing yields an empty slice [], since a +// wildcard over zero elements legitimately produces no values. +// +// An invalid expression yields an empty slice. +func (b *RequestEvaluator) extractField(document interface{}, field string) []string { + expr, err := jp.ParseString(field) + if err != nil { + Logger.Debug().Msgf("field [%v] is not a valid JSONPath expression: %v", field, err) return []string{} } - - // Iterate the array and extract suffix field from each element - var values []string - suffixSegments := parsePathSegments(suffix) - - jsonparser.ArrayEach(arrayData, func(elementValue []byte, dataType jsonparser.ValueType, offset int, err error) { - if err != nil { - return + // jp.Get always returns a slice of matches (equivalent to Jayway ALWAYS_RETURN_LIST). + matches := expr.Get(document) + if len(matches) == 0 { + // No match: a definite path yields [""] (Java findPath contract), an indefinite + // path yields []. This is the same rule as emptyValueForField, applied here with + // the already-parsed expr to avoid re-parsing. + if expr.Normal() { + return []string{""} } - if len(suffixSegments) == 0 { - // No suffix - use the element value directly - values = append(values, string(elementValue)) - return - } - // Extract the suffix field from the array element - val, valType, _, getErr := jsonparser.Get(elementValue, suffixSegments...) - if getErr != nil { - return - } - switch valType { - case jsonparser.String: - values = append(values, string(val)) - case jsonparser.NotExist: - // Skip non-existing fields - default: - values = append(values, string(val)) - } - }) - + return []string{} + } + values := make([]string, 0, len(matches)) + for _, item := range matches { + values = append(values, jsonValueToString(item)) + } return values } -// parsePathSegments converts a dot-notation path string into jsonparser-compatible path segments. -// Handles bracket notation: "imp[0].pmp.deals" → ["imp", "[0]", "pmp", "deals"] -func parsePathSegments(path string) []string { - if path == "" { - return []string{} +// jsonValueToString renders a scalar JSON value as a string. A nil (JSON null) becomes +// "null"; all other values use their natural string representation (json.Number preserves +// the original numeric text). +func jsonValueToString(value interface{}) string { + if value == nil { + return "null" } - // Split on "." but handle bracket notation - rawParts := strings.Split(path, ".") - var segments []string - for _, part := range rawParts { - if part == "" { - continue - } - // Check if part contains bracket notation like "imp[0]" - if bracketIdx := strings.Index(part, "["); bracketIdx >= 0 { - // Split into name and bracket parts - name := part[:bracketIdx] - rest := part[bracketIdx:] - if name != "" { - segments = append(segments, name) - } - // Parse bracket indices - e.g., "[0]" or "[0][1]" - for len(rest) > 0 { - closeIdx := strings.Index(rest, "]") - if closeIdx < 0 { - break - } - segments = append(segments, rest[:closeIdx+1]) - rest = rest[closeIdx+1:] - } - } else { - segments = append(segments, part) - } - } - return segments + return fmt.Sprintf("%v", value) } func (b *RequestEvaluator) getModelDefinitions(context *interfaces.Context) ([]interfaces.ModelDefinition, error) { @@ -485,20 +448,6 @@ func (b *RequestEvaluator) buildResponse(context *interfaces.Context) Response { } } -func convertFieldsToPaths(fields []string) [][]string { - // Remove "$." prefix if present and add delimiter "." around "[]" - var paths [][]string - for _, field := range fields { - field = strings.TrimPrefix(field, "$.") - paths = append(paths, strings.Split(strings.ReplaceAll(field, "[", ".["), ".")) - } - return paths -} - -func convertPathsToField(paths []string) string { - return "$." + strings.ReplaceAll(strings.Join(paths, "."), ".[", "[") -} - func buildSlots(context *interfaces.Context) []Slot { aggregatedModelEvaluationResult := context.AggregatedModelEvaluationResult return []Slot{ diff --git a/golang/evaluation/RequestEvaluator_test.go b/golang/evaluation/RequestEvaluator_test.go index 3d60831..4feb7a3 100644 --- a/golang/evaluation/RequestEvaluator_test.go +++ b/golang/evaluation/RequestEvaluator_test.go @@ -49,20 +49,22 @@ var ( "$.imp[0].banner.pos", "$.device.devicetype", } + // Missing definite (normal) JSONPath fields resolve to a single empty string [""], + // matching the Java implementation (Jayway PathNotFoundException → singletonList("")). CompleteFieldValueMap = map[string][]string{ "$.site.publisher.id": {"539014228"}, "$.imp[0].banner.w": {"970"}, "$.device.geo.country": {"USA"}, - "$.app": {}, - "$.imp[0].video.h": {}, - "$.imp[0].video.pos": {}, + "$.app": {""}, + "$.imp[0].video.h": {""}, + "$.imp[0].video.pos": {""}, "$.id": {"e0371864-238f-41b1-a544-59b4b6a602ec"}, "$.imp[0].banner.h": {"250"}, "$.imp[0].banner.pos": {"1"}, "$.device.devicetype": {"2"}, - "$.imp[0].video": {}, - "$.app.publisher.id": {}, - "$.imp[0].video.w": {}, + "$.imp[0].video": {""}, + "$.app.publisher.id": {""}, + "$.imp[0].video.w": {""}, } IncompleteFieldValueMap = map[string][]string{ "$.site.publisher.id": {"539014228"}, @@ -693,18 +695,20 @@ func (suite *RequestEvaluatorTestSuite) TestAddMissingEntriesToMap() { }, }, { - name: "missing key gets empty slice", + // Missing definite (normal) path resolves to [""], matching the string + // parser's findPath contract so both input paths agree. + name: "missing definite key gets single empty string", inputMap: map[string][]string{ "$.site.publisher.id": {"539014228"}, }, uniqueFields: []string{"$.site.publisher.id", "$.device.geo.country"}, expectedResult: map[string][]string{ "$.site.publisher.id": {"539014228"}, - "$.device.geo.country": {}, + "$.device.geo.country": {""}, }, }, { - name: "nil slice treated as missing", + name: "nil slice treated as missing definite key", inputMap: map[string][]string{ "$.site.publisher.id": {"539014228"}, "$.device.geo.country": nil, @@ -712,7 +716,19 @@ func (suite *RequestEvaluatorTestSuite) TestAddMissingEntriesToMap() { uniqueFields: []string{"$.site.publisher.id", "$.device.geo.country"}, expectedResult: map[string][]string{ "$.site.publisher.id": {"539014228"}, - "$.device.geo.country": {}, + "$.device.geo.country": {""}, + }, + }, + { + // Missing indefinite (wildcard) path resolves to an empty slice, not [""]. + name: "missing wildcard key gets empty slice", + inputMap: map[string][]string{ + "$.site.publisher.id": {"539014228"}, + }, + uniqueFields: []string{"$.site.publisher.id", "$.imp[0].pmp.deals[*].id"}, + expectedResult: map[string][]string{ + "$.site.publisher.id": {"539014228"}, + "$.imp[0].pmp.deals[*].id": {}, }, }, { @@ -730,13 +746,14 @@ func (suite *RequestEvaluatorTestSuite) TestAddMissingEntriesToMap() { }, }, { - name: "all fields missing get empty slices", + // All definite paths missing → each resolves to [""]. + name: "all definite fields missing get single empty strings", inputMap: map[string][]string{}, uniqueFields: []string{"$.site.publisher.id", "$.device.geo.country", "$.imp[0].banner.w"}, expectedResult: map[string][]string{ - "$.site.publisher.id": {}, - "$.device.geo.country": {}, - "$.imp[0].banner.w": {}, + "$.site.publisher.id": {""}, + "$.device.geo.country": {""}, + "$.imp[0].banner.w": {""}, }, }, } @@ -756,8 +773,8 @@ func (suite *RequestEvaluatorTestSuite) TestAddMissingEntriesToMap() { } } -func (suite *RequestEvaluatorTestSuite) TestExtractWildcardField() { - jsonWithDeals := `{"imp":[{"pmp":{"deals":[{"id":"deal-1","bidfloor":1.5},{"id":"deal-2","bidfloor":2.0},{"id":"deal-3","bidfloor":3.0}]}}]}` +func (suite *RequestEvaluatorTestSuite) TestExtractField() { + jsonWithDeals := `{"imp":[{"pmp":{"deals":[{"id":"deal-1","bidfloor":1.5},{"id":"deal-2","bidfloor":2.0},{"id":"deal-3","bidfloor":3.0}]}}],"site":{"publisher":{"id":"pub-1"}},"nullKey":null}` tests := []struct { name string @@ -771,6 +788,24 @@ func (suite *RequestEvaluatorTestSuite) TestExtractWildcardField() { fieldPath: "$.imp[0].pmp.deals[*].id", expected: []string{"deal-1", "deal-2", "deal-3"}, }, + { + name: "scalar path wraps value in single-element slice", + jsonData: jsonWithDeals, + fieldPath: "$.site.publisher.id", + expected: []string{"pub-1"}, + }, + { + name: "numeric scalar preserves original representation", + jsonData: jsonWithDeals, + fieldPath: "$.imp[0].pmp.deals[0].bidfloor", + expected: []string{"1.5"}, + }, + { + name: "present-but-null field returns literal null", + jsonData: jsonWithDeals, + fieldPath: "$.nullKey", + expected: []string{"null"}, + }, { name: "wildcard path on empty array returns empty slice", jsonData: `{"imp":[{"pmp":{"deals":[]}}]}`, @@ -784,25 +819,46 @@ func (suite *RequestEvaluatorTestSuite) TestExtractWildcardField() { expected: []string{}, }, { - name: "malformed JSON returns empty slice", - jsonData: `{not valid json`, - fieldPath: "$.imp[0].pmp.deals[*].id", - expected: []string{}, + // Definite (normal) path that matches nothing → [""], matching Java's + // findPath returning singletonList("") on PathNotFoundException. + name: "non-existent scalar path returns single empty string", + jsonData: jsonWithDeals, + fieldPath: "$.nonexistent.field", + expected: []string{""}, + }, + { + // A definite path whose leaf key is absent under an existing parent also + // resolves to [""]. + name: "missing leaf under existing parent returns single empty string", + jsonData: jsonWithDeals, + fieldPath: "$.site.publisher.name", + expected: []string{""}, }, } for _, tt := range tests { suite.Run(tt.name, func() { - result := suite.evaluator.extractWildcardField([]byte(tt.jsonData), tt.fieldPath) - if len(tt.expected) == 0 { - suite.Empty(result) - } else { - suite.Equal(tt.expected, result) - } + document, err := parseJSONDocument(tt.jsonData) + suite.NoError(err) + result := suite.evaluator.extractField(document, tt.fieldPath) + // Distinguish [] from [""]: both are len-mismatch-sensitive, so compare directly. + suite.Equal(tt.expected, result) }) } } +func (suite *RequestEvaluatorTestSuite) TestParse_ReturnErr_MalformedJSON() { + suite.mockModelConfigHandler.EXPECT(). + GetAllUniqueFeatureFields(). + Return(AllUniqueFeatureFields, nil). + Once() + + fieldValueMap, err := suite.evaluator.parse(`{not valid json`, []string{"$.id"}) + + suite.Nil(fieldValueMap, "Field value map should be nil on malformed JSON") + suite.ErrorContains(err, "fail to parse openRtbRequest as JSON") +} + func (suite *RequestEvaluatorTestSuite) TestEvaluate_UsesConfigurableAggregator_WhenAggregationSchemaIsNonNil() { // When AggregationSchema is configured, ConfigurableAggregator should be used aggregationSchema := &interfaces.AggregationNode{ @@ -1007,9 +1063,18 @@ func (suite *RequestEvaluatorTestSuite) TestParse_MultiValueExtraction() { expectedValue: []string{"539014228"}, }, { - name: "missing field returns empty slice", + // Missing definite (normal) path resolves to a single empty string, + // matching the Java findPath contract. + name: "missing definite field returns single empty string", uniqueFields: []string{"$.nonexistent.field"}, expectedField: "$.nonexistent.field", + expectedValue: []string{""}, + }, + { + // Missing wildcard (indefinite) path resolves to an empty slice. + name: "missing wildcard field returns empty slice", + uniqueFields: []string{"$.imp[0].pmp.deals[*].nonexistent"}, + expectedField: "$.imp[0].pmp.deals[*].nonexistent", expectedValue: []string{}, }, } diff --git a/golang/go.mod b/golang/go.mod index b6799ce..1017df8 100644 --- a/golang/go.mod +++ b/golang/go.mod @@ -8,11 +8,13 @@ require ( github.com/OldPanda/bloomfilter v1.0.0 github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.27.43 + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 github.com/aws/smithy-go v1.22.2 - github.com/buger/jsonparser v1.1.1 github.com/dgraph-io/ristretto/v2 v2.2.0 github.com/google/uuid v1.6.0 + github.com/ohler55/ojg v1.28.5 github.com/rs/zerolog v1.33.0 github.com/stretchr/testify v1.10.0 ) @@ -20,7 +22,6 @@ require ( require ( github.com/Workiva/go-datastructures v1.1.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect @@ -32,7 +33,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/golang/go.sum b/golang/go.sum index ba1e324..fa44231 100644 --- a/golang/go.sum +++ b/golang/go.sum @@ -38,8 +38,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -60,6 +58,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ohler55/ojg v1.28.5 h1:KlNeyCDlwt6CDlv7VP6f9sAe9w4t5trxJCo64vO0/kc= +github.com/ohler55/ojg v1.28.5/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/golang/modelfeature/ModelConfiguration.go b/golang/modelfeature/ModelConfiguration.go index 573d4a4..ed2e086 100644 --- a/golang/modelfeature/ModelConfiguration.go +++ b/golang/modelfeature/ModelConfiguration.go @@ -84,7 +84,7 @@ func ExistsTransformer(modelFeature *interfaces.ModelFeature) (*interfaces.Model // A transformer that retrieves the first non-empty value from a ModelFeature. // // This function transforms a ModelFeature by selecting the first non-null and non-empty -// value from its list of values. If no such value is found, it returns an empty string. +// value from its list of values. If no such value is found, it returns an empty string. func GetFirstNotEmptyTransformer(modelFeature *interfaces.ModelFeature) (*interfaces.ModelFeature, error) { var firstNotEmpty string for _, value := range modelFeature.Values { diff --git a/golang/modelfeature/ModelConfiguration_test.go b/golang/modelfeature/ModelConfiguration_test.go index 60d640b..7d7ca43 100644 --- a/golang/modelfeature/ModelConfiguration_test.go +++ b/golang/modelfeature/ModelConfiguration_test.go @@ -93,3 +93,94 @@ func (suite *ModelConfigurationTestSuite) TestIncludeDefaultValueTransformer() { }) } } + +// TestExistsTransformer verifies the Exists transformer maps empty values to "0" and +// non-empty values to "1", matching the Java implementation. +func (suite *ModelConfigurationTestSuite) TestExistsTransformer() { + tests := []struct { + name string + inputValues []string + expectedValues []string + }{ + { + name: "non-empty values become 1", + inputValues: []string{"a", "b", "c"}, + expectedValues: []string{"1", "1", "1"}, + }, + { + name: "empty values become 0", + inputValues: []string{"", "", ""}, + expectedValues: []string{"0", "0", "0"}, + }, + { + name: "mixed values map positionally", + inputValues: []string{"a", "", "c", ""}, + expectedValues: []string{"1", "0", "1", "0"}, + }, + { + // A missing definite JSONPath field arrives as [""] (see extractField), so + // Exists produces ["0"] rather than dropping the value entirely. + name: "single empty string (missing field) becomes 0", + inputValues: []string{""}, + expectedValues: []string{"0"}, + }, + } + + for _, tt := range tests { + suite.Run(tt.name, func() { + input := &interfaces.ModelFeature{ + Configuration: &interfaces.FeatureConfiguration{}, + Values: tt.inputValues, + } + result, err := ExistsTransformer(input) + suite.Nil(err) + suite.Equal(tt.expectedValues, result.Values) + }) + } +} + +// TestExistsThenApplyMappings_IsMobilePattern verifies the canonical "isMobile" feature +// chain: Exists followed by ApplyMappings with {"0":"site","1":"app"}. Crucially, an +// absent $.app field (extracted as [""]) must resolve to "site", and a present $.app +// (extracted as a non-empty value) must resolve to "app". +func (suite *ModelConfigurationTestSuite) TestExistsThenApplyMappings_IsMobilePattern() { + tests := []struct { + name string + inputValues []string + expected []string + }{ + { + // $.app present (object rendered to a non-empty string) → "1" → "app". + name: "app present maps to app", + inputValues: []string{"map[bundle:com.example]"}, + expected: []string{"app"}, + }, + { + // $.app absent → extractField returns [""] → Exists "0" → ApplyMappings "site". + name: "app absent maps to site", + inputValues: []string{""}, + expected: []string{"site"}, + }, + } + + for _, tt := range tests { + suite.Run(tt.name, func() { + configuration := &interfaces.FeatureConfiguration{ + Name: "isMobile", + Fields: []string{"$.app"}, + Transformations: []interfaces.TransformerName{Exists, ApplyMappings}, + Mapping: map[string]string{"0": "site", "1": "app"}, + } + feature := &interfaces.ModelFeature{ + Configuration: configuration, + Values: tt.inputValues, + } + + afterExists, err := ExistsTransformer(feature) + suite.Nil(err) + result, err := ApplyMappingsTransformer(afterExists) + suite.Nil(err) + suite.Equal(tt.expected, result.Values) + }) + } +}