From 610cd89731ec35b9f4fb05c9c09c77ae2dad1950 Mon Sep 17 00:00:00 2001 From: Shvejan Mutheboyina Date: Mon, 17 Aug 2026 10:04:15 +0000 Subject: [PATCH 1/2] bugfix per tenant cache ttl fallback Signed-off-by: Shvejan Mutheboyina --- docs/configuration/config-file-reference.md | 5 +- pkg/chunk/cache/mock.go | 8 ++ .../tripperware/queryrange/results_cache.go | 20 ++- .../queryrange/results_cache_test.go | 131 +++++++++++++----- pkg/util/validation/limits.go | 5 +- schemas/cortex-config-schema.json | 2 +- 6 files changed, 128 insertions(+), 43 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index b7922a94376..97d1703fa5d 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4737,8 +4737,9 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # Per-tenant TTL for cached query results that overlap with the out-of-order # time window. These results may still receive out-of-order samples, so they -# typically use a shorter TTL. 0 (default) means use the global cache backend -# TTL configuration. +# typically use a shorter TTL. 0 (default) means fall back to +# frontend.results-cache-ttl, and if that is also 0, use the global cache +# backend TTL configuration. # CLI flag: -frontend.out-of-order-results-cache-ttl [out_of_order_results_cache_ttl: | default = 0s] diff --git a/pkg/chunk/cache/mock.go b/pkg/chunk/cache/mock.go index 68b97550226..53405deae79 100644 --- a/pkg/chunk/cache/mock.go +++ b/pkg/chunk/cache/mock.go @@ -11,11 +11,19 @@ type MockCache struct { sync.Mutex cache map[string][]byte lastTTL time.Duration + // DefaultTTL, when set, is applied when a Store call passes a TTL of 0, mirroring + // the global default validity of real cache backends (Memcached/Redis/FIFO). + DefaultTTL time.Duration } +// Store records the resolved TTL. Mirroring real cache backends (Memcached/Redis/FIFO), +// a TTL of 0 falls back to the configured default validity. func (m *MockCache) Store(_ context.Context, keys []string, bufs [][]byte, ttl time.Duration) { m.Lock() defer m.Unlock() + if ttl == 0 { + ttl = m.DefaultTTL + } m.lastTTL = ttl for i := range keys { m.cache[keys[i]] = bufs[i] diff --git a/pkg/querier/tripperware/queryrange/results_cache.go b/pkg/querier/tripperware/queryrange/results_cache.go index f7cbbcee67c..dbb786989e6 100644 --- a/pkg/querier/tripperware/queryrange/results_cache.go +++ b/pkg/querier/tripperware/queryrange/results_cache.go @@ -782,17 +782,23 @@ func (s resultsCache) get(ctx context.Context, key string, ttl time.Duration) ([ // getTTLForExtents calculates the appropriate TTL for given extents based on whether // they overlap with the out-of-order time window. func (s resultsCache) getTTLForExtents(tenantIDs []string, extents []tripperware.Extent) time.Duration { - var resultsCacheTTL, outOfOrderCacheTTL time.Duration - if len(tenantIDs) > 0 { - // Use smallest non-zero TTL to respect the most restrictive tenant's cache policy - resultsCacheTTL = validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, s.limits.ResultsCacheTTL) - outOfOrderCacheTTL = validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, s.limits.OutOfOrderResultsCacheTTL) + if len(tenantIDs) == 0 { + return 0 } if s.extentsOverlapOutOfOrderWindow(extents, tenantIDs) { - return outOfOrderCacheTTL + // Use smallest non-zero TTL to respect the most restrictive tenant's cache policy. + // The out-of-order TTL is resolved per-tenant before aggregating: if a tenant does + // not explicitly set out_of_order_results_cache_ttl (0), it falls back to that + // tenant's results_cache_ttl, and only then to the global cache backend TTL (0). + return validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, func(userID string) time.Duration { + if ttl := s.limits.OutOfOrderResultsCacheTTL(userID); ttl > 0 { + return ttl + } + return s.limits.ResultsCacheTTL(userID) + }) } - return resultsCacheTTL + return validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, s.limits.ResultsCacheTTL) } // extentsOverlapOutOfOrderWindow checks if any extent overlaps with the out-of-order time window. diff --git a/pkg/querier/tripperware/queryrange/results_cache_test.go b/pkg/querier/tripperware/queryrange/results_cache_test.go index 360926b7ee7..daebe1eddc2 100644 --- a/pkg/querier/tripperware/queryrange/results_cache_test.go +++ b/pkg/querier/tripperware/queryrange/results_cache_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strconv" + "strings" "testing" "time" @@ -1877,62 +1878,110 @@ func TestResultsCachePutTTLSelection(t *testing.T) { oneHourAgo := now.Add(-1 * time.Hour).UnixMilli() twoHoursAgo := now.Add(-2 * time.Hour).UnixMilli() + // globalDefaultTTL is the cache backend's configured default validity, applied + // when the results cache passes a TTL of 0. A distinct, non-zero value lets us + // assert that the "fall back to global default" cases really land on it. + const globalDefaultTTL = 90 * time.Minute + tests := []struct { - name string - extents []tripperware.Extent - resultsCacheTTL time.Duration - outOfOrderCacheTTL time.Duration - outOfOrderWindow time.Duration - expectedTTL time.Duration + name string + tenantIDs []string + extents []tripperware.Extent + resultsCacheTTL map[string]time.Duration + outOfOrderResultsCacheTTL map[string]time.Duration + outOfOrderWindow map[string]time.Duration + expectedTTL time.Duration }{ { - name: "old data uses results_cache_ttl", + name: "old data uses results_cache_ttl", + tenantIDs: []string{"tenant-a"}, extents: []tripperware.Extent{ {Start: twoHoursAgo, End: twoHoursAgo + 1000}, // 2 hours ago, no overlap }, - resultsCacheTTL: 24 * time.Hour, - outOfOrderCacheTTL: 5 * time.Minute, - outOfOrderWindow: 1 * time.Hour, - expectedTTL: 24 * time.Hour, + resultsCacheTTL: map[string]time.Duration{"tenant-a": 24 * time.Hour}, + outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 5 * time.Minute}, + outOfOrderWindow: map[string]time.Duration{"tenant-a": 1 * time.Hour}, + expectedTTL: 24 * time.Hour, }, { - name: "recent data uses out_of_order_results_cache_ttl", + name: "recent data uses out_of_order_results_cache_ttl", + tenantIDs: []string{"tenant-a"}, extents: []tripperware.Extent{ {Start: twoHoursAgo, End: oneHourAgo}, // overlaps with 1h window }, - resultsCacheTTL: 24 * time.Hour, - outOfOrderCacheTTL: 5 * time.Minute, - outOfOrderWindow: 1 * time.Hour, - expectedTTL: 5 * time.Minute, + resultsCacheTTL: map[string]time.Duration{"tenant-a": 24 * time.Hour}, + outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 5 * time.Minute}, + outOfOrderWindow: map[string]time.Duration{"tenant-a": 1 * time.Hour}, + expectedTTL: 5 * time.Minute, }, { - name: "zero out-of-order window uses results_cache_ttl", + name: "zero out-of-order window uses results_cache_ttl", + tenantIDs: []string{"tenant-a"}, extents: []tripperware.Extent{ {Start: twoHoursAgo, End: oneHourAgo}, }, - resultsCacheTTL: 12 * time.Hour, - outOfOrderCacheTTL: 5 * time.Minute, - outOfOrderWindow: 0, // no out-of-order support - expectedTTL: 12 * time.Hour, + resultsCacheTTL: map[string]time.Duration{"tenant-a": 12 * time.Hour}, + outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 5 * time.Minute}, + outOfOrderWindow: map[string]time.Duration{"tenant-a": 0}, // no out-of-order support + expectedTTL: 12 * time.Hour, }, { - name: "zero TTLs use backend defaults", + name: "recent data falls back to results_cache_ttl when out_of_order_results_cache_ttl is unset", + tenantIDs: []string{"tenant-a"}, + extents: []tripperware.Extent{ + {Start: twoHoursAgo, End: oneHourAgo}, // overlaps with 1h window + }, + resultsCacheTTL: map[string]time.Duration{"tenant-a": 24 * time.Hour}, + outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 0}, // not set, should fall back to results_cache_ttl + outOfOrderWindow: map[string]time.Duration{"tenant-a": 1 * time.Hour}, + expectedTTL: 24 * time.Hour, + }, + { + name: "recent data uses backend default when both TTLs are unset", + tenantIDs: []string{"tenant-a"}, + extents: []tripperware.Extent{ + {Start: twoHoursAgo, End: oneHourAgo}, // overlaps with 1h window + }, + resultsCacheTTL: map[string]time.Duration{"tenant-a": 0}, + outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 0}, + outOfOrderWindow: map[string]time.Duration{"tenant-a": 1 * time.Hour}, + expectedTTL: globalDefaultTTL, // TTL of 0 falls back to the backend default + }, + { + name: "zero TTLs use backend defaults", + tenantIDs: []string{"tenant-a"}, extents: []tripperware.Extent{ {Start: twoHoursAgo, End: twoHoursAgo + 1000}, }, - resultsCacheTTL: 0, - outOfOrderCacheTTL: 0, - outOfOrderWindow: 0, - expectedTTL: 0, // backend default + resultsCacheTTL: map[string]time.Duration{"tenant-a": 0}, + outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 0}, + outOfOrderWindow: map[string]time.Duration{"tenant-a": 0}, + expectedTTL: globalDefaultTTL, // TTL of 0 falls back to the backend default + }, + { + // Federated query: tenant "a" leaves its OOO TTL unset, so it falls back to + // its own results_cache_ttl (30m); tenant "b" has 1h. The most restrictive is + // 30m. The fallback must be resolved per-tenant *before* aggregating across + // tenants — aggregating the raw configs first would instead yield 1h. + name: "recent data resolves fallback per-tenant before aggregating", + tenantIDs: []string{"a", "b"}, + extents: []tripperware.Extent{ + {Start: twoHoursAgo, End: oneHourAgo}, // overlaps with 1h window + }, + resultsCacheTTL: map[string]time.Duration{"a": 30 * time.Minute, "b": 24 * time.Hour}, + outOfOrderResultsCacheTTL: map[string]time.Duration{"a": 0, "b": 1 * time.Hour}, + outOfOrderWindow: map[string]time.Duration{"a": 1 * time.Hour, "b": 1 * time.Hour}, + expectedTTL: 30 * time.Minute, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { mockCache := cache.NewMockCache() - limits := mockLimits{ + mockCache.(*cache.MockCache).DefaultTTL = globalDefaultTTL + limits := perTenantLimits{ resultsCacheTTL: tc.resultsCacheTTL, - outOfOrderResultsCacheTTL: tc.outOfOrderCacheTTL, + outOfOrderResultsCacheTTL: tc.outOfOrderResultsCacheTTL, outOfOrderWindow: tc.outOfOrderWindow, } @@ -1956,15 +2005,35 @@ func TestResultsCachePutTTLSelection(t *testing.T) { rc := rm.Wrap(nil).(*resultsCache) rc.now = func() time.Time { return now } - ctx := user.InjectOrgID(context.Background(), "tenant-a") - tenantIDs, _ := users.TenantIDs(ctx) - rc.put(ctx, "test-key", tc.extents, tenantIDs) + ctx := user.InjectOrgID(context.Background(), strings.Join(tc.tenantIDs, "|")) + rc.put(ctx, "test-key", tc.extents, tc.tenantIDs) assert.Equal(t, tc.expectedTTL, mockCache.(*cache.MockCache).GetLastTTL()) }) } } +// perTenantLimits returns different TTL/window values per tenant so we can verify +// that the out-of-order TTL fallback is resolved per-tenant before aggregating. +type perTenantLimits struct { + mockLimits + resultsCacheTTL map[string]time.Duration + outOfOrderResultsCacheTTL map[string]time.Duration + outOfOrderWindow map[string]time.Duration +} + +func (m perTenantLimits) ResultsCacheTTL(userID string) time.Duration { + return m.resultsCacheTTL[userID] +} + +func (m perTenantLimits) OutOfOrderResultsCacheTTL(userID string) time.Duration { + return m.outOfOrderResultsCacheTTL[userID] +} + +func (m perTenantLimits) OutOfOrderTimeWindow(userID string) model.Duration { + return model.Duration(m.outOfOrderWindow[userID]) +} + type mockResolver struct { tenantIDs []string err error diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 019a5adc3ed..64cadc82379 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -343,9 +343,10 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { _ = l.MaxCacheFreshness.Set("1m") f.Int64Var(&l.MaxQueryResponseSize, "frontend.max-query-response-size", 0, "The maximum total uncompressed query response size. If the query was sharded the limit is applied to the total response size of all shards. This limit is enforced in query-frontend for `query` and `query_range` APIs. 0 to disable.") f.Var(&l.MaxCacheFreshness, "frontend.max-cache-freshness", "Most recent allowed cacheable result per-tenant, to prevent caching very recent results that might still be in flux.") - // ResultsCacheTTL and OutOfOrderResultsCacheTTL default to 0 (use global cache config expiration) + // ResultsCacheTTL and OutOfOrderResultsCacheTTL default to 0. ResultsCacheTTL falls back to the + // global cache config expiration; OutOfOrderResultsCacheTTL falls back to ResultsCacheTTL first. f.Var(&l.ResultsCacheTTL, "frontend.results-cache-ttl", "Per-tenant TTL for cached query results in the cache backend (Memcached/Redis/FIFO). This is the standard TTL for results that do not overlap with the out-of-order time window. 0 (default) means use the global cache backend TTL configuration.") - f.Var(&l.OutOfOrderResultsCacheTTL, "frontend.out-of-order-results-cache-ttl", "Per-tenant TTL for cached query results that overlap with the out-of-order time window. These results may still receive out-of-order samples, so they typically use a shorter TTL. 0 (default) means use the global cache backend TTL configuration.") + f.Var(&l.OutOfOrderResultsCacheTTL, "frontend.out-of-order-results-cache-ttl", "Per-tenant TTL for cached query results that overlap with the out-of-order time window. These results may still receive out-of-order samples, so they typically use a shorter TTL. 0 (default) means fall back to frontend.results-cache-ttl, and if that is also 0, use the global cache backend TTL configuration.") f.Float64Var(&l.MaxQueriersPerTenant, "frontend.max-queriers-per-tenant", 0, "Maximum number of queriers that can handle requests for a single tenant. If set to 0 or value higher than number of available queriers, *all* queriers will handle requests for the tenant. If the value is < 1, it will be treated as a percentage and the gets a percentage of the total queriers. Each frontend (or query-scheduler, if used) will select the same set of queriers for the same tenant (given that all queriers are connected to all frontends / query-schedulers). This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL.") f.IntVar(&l.QueryVerticalShardSize, "frontend.query-vertical-shard-size", 0, "[Experimental] Number of shards to use when distributing shardable PromQL queries.") f.BoolVar(&l.QueryPriority.Enabled, "frontend.query-priority.enabled", false, "Whether queries are assigned with priorities.") diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index aed2998e063..49ea9edd5e6 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -5975,7 +5975,7 @@ }, "out_of_order_results_cache_ttl": { "default": "0s", - "description": "Per-tenant TTL for cached query results that overlap with the out-of-order time window. These results may still receive out-of-order samples, so they typically use a shorter TTL. 0 (default) means use the global cache backend TTL configuration.", + "description": "Per-tenant TTL for cached query results that overlap with the out-of-order time window. These results may still receive out-of-order samples, so they typically use a shorter TTL. 0 (default) means fall back to frontend.results-cache-ttl, and if that is also 0, use the global cache backend TTL configuration.", "type": "string", "x-cli-flag": "frontend.out-of-order-results-cache-ttl", "x-format": "duration" From 1882f967a6c3826bad01c7ae6ab9afb2fbc4473e Mon Sep 17 00:00:00 2001 From: Shvejan Mutheboyina Date: Mon, 17 Aug 2026 10:06:33 +0000 Subject: [PATCH 2/2] updating changelog Signed-off-by: Shvejan Mutheboyina --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fe16f3a8bd..6960f9758e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,7 @@ * [BUGFIX] Parquet Converter: Fix `auto_forget_delay` having no effect. The ring lifecycler was created without the auto-forget delegate, so unhealthy instances were never automatically removed from the ring. #7752 * [BUGFIX] Alertmanager: Reject the global `mattermost_webhook_url_file` setting in per-tenant configs, consistent with every other global `*_file` setting. #7768 * [BUGFIX] Alertmanager: Tighten per-tenant config validation to reject additional file-based settings. #7767 +* [BUGFIX] Query Frontend: Fix per-tenant results cache TTL for out-of-order results fallback order. It now falls back to `results_cache_ttl` before falling back to the global cache backend TTL. ## 1.21.1 2026-06-04