bugfix per tenant cache ttl fallback

CHANGELOG.md

@@ -91,6 +91,7 @@
 * [BUGFIX] Parquet Converter: Fix `auto_forget_delay` having no effect. The ring lifecycler was created without the auto-forget delegate, so unhealthy instances were never automatically removed from the ring. #7752
 * [BUGFIX] Alertmanager: Reject the global `mattermost_webhook_url_file` setting in per-tenant configs, consistent with every other global `*_file` setting. #7768
 * [BUGFIX] Alertmanager: Tighten per-tenant config validation to reject additional file-based settings. #7767
+* [BUGFIX] Query Frontend: Fix per-tenant results cache TTL for out-of-order results fallback order. It now falls back to `results_cache_ttl` before falling back to the global cache backend TTL.
 
 ## 1.21.1 2026-06-04
 

docs/configuration/config-file-reference.md

@@ -4737,8 +4737,9 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 
 # Per-tenant TTL for cached query results that overlap with the out-of-order
 # time window. These results may still receive out-of-order samples, so they
-# typically use a shorter TTL. 0 (default) means use the global cache backend
-# TTL configuration.
+# typically use a shorter TTL. 0 (default) means fall back to
+# frontend.results-cache-ttl, and if that is also 0, use the global cache
+# backend TTL configuration.
 # CLI flag: -frontend.out-of-order-results-cache-ttl
 [out_of_order_results_cache_ttl: <duration> | default = 0s]
 

pkg/chunk/cache/mock.go

@@ -11,11 +11,19 @@ type MockCache struct {
 	sync.Mutex
 	cache   map[string][]byte
 	lastTTL time.Duration
+	// DefaultTTL, when set, is applied when a Store call passes a TTL of 0, mirroring
+	// the global default validity of real cache backends (Memcached/Redis/FIFO).
+	DefaultTTL time.Duration
 }
 
+// Store records the resolved TTL. Mirroring real cache backends (Memcached/Redis/FIFO),
+// a TTL of 0 falls back to the configured default validity.
 func (m *MockCache) Store(_ context.Context, keys []string, bufs [][]byte, ttl time.Duration) {
 	m.Lock()
 	defer m.Unlock()
+	if ttl == 0 {
+		ttl = m.DefaultTTL
+	}
 	m.lastTTL = ttl
 	for i := range keys {
 		m.cache[keys[i]] = bufs[i]

pkg/querier/tripperware/queryrange/results_cache.go

@@ -782,17 +782,23 @@ func (s resultsCache) get(ctx context.Context, key string, ttl time.Duration) ([
 // getTTLForExtents calculates the appropriate TTL for given extents based on whether
 // they overlap with the out-of-order time window.
 func (s resultsCache) getTTLForExtents(tenantIDs []string, extents []tripperware.Extent) time.Duration {
-	var resultsCacheTTL, outOfOrderCacheTTL time.Duration
-	if len(tenantIDs) > 0 {
-		// Use smallest non-zero TTL to respect the most restrictive tenant's cache policy
-		resultsCacheTTL = validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, s.limits.ResultsCacheTTL)
-		outOfOrderCacheTTL = validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, s.limits.OutOfOrderResultsCacheTTL)
+	if len(tenantIDs) == 0 {
+		return 0
 	}
 
 	if s.extentsOverlapOutOfOrderWindow(extents, tenantIDs) {
-		return outOfOrderCacheTTL
+		// Use smallest non-zero TTL to respect the most restrictive tenant's cache policy.
+		// The out-of-order TTL is resolved per-tenant before aggregating: if a tenant does
+		// not explicitly set out_of_order_results_cache_ttl (0), it falls back to that
+		// tenant's results_cache_ttl, and only then to the global cache backend TTL (0).
+		return validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, func(userID string) time.Duration {
+			if ttl := s.limits.OutOfOrderResultsCacheTTL(userID); ttl > 0 {
+				return ttl
+			}
+			return s.limits.ResultsCacheTTL(userID)
+		})
 	}
-	return resultsCacheTTL
+	return validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, s.limits.ResultsCacheTTL)
 }
 
 // extentsOverlapOutOfOrderWindow checks if any extent overlaps with the out-of-order time window.

pkg/querier/tripperware/queryrange/results_cache_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -1877,62 +1878,110 @@ func TestResultsCachePutTTLSelection(t *testing.T) {
 	oneHourAgo := now.Add(-1 * time.Hour).UnixMilli()
 	twoHoursAgo := now.Add(-2 * time.Hour).UnixMilli()
 
+	// globalDefaultTTL is the cache backend's configured default validity, applied
+	// when the results cache passes a TTL of 0. A distinct, non-zero value lets us
+	// assert that the "fall back to global default" cases really land on it.
+	const globalDefaultTTL = 90 * time.Minute
+
 	tests := []struct {
-		name               string
-		extents            []tripperware.Extent
-		resultsCacheTTL    time.Duration
-		outOfOrderCacheTTL time.Duration
-		outOfOrderWindow   time.Duration
-		expectedTTL        time.Duration
+		name                      string
+		tenantIDs                 []string
+		extents                   []tripperware.Extent
+		resultsCacheTTL           map[string]time.Duration
+		outOfOrderResultsCacheTTL map[string]time.Duration
+		outOfOrderWindow          map[string]time.Duration
+		expectedTTL               time.Duration
 	}{
 		{
-			name: "old data uses results_cache_ttl",
+			name:      "old data uses results_cache_ttl",
+			tenantIDs: []string{"tenant-a"},
 			extents: []tripperware.Extent{
 				{Start: twoHoursAgo, End: twoHoursAgo + 1000}, // 2 hours ago, no overlap
 			},
-			resultsCacheTTL:    24 * time.Hour,
-			outOfOrderCacheTTL: 5 * time.Minute,
-			outOfOrderWindow:   1 * time.Hour,
-			expectedTTL:        24 * time.Hour,
+			resultsCacheTTL:           map[string]time.Duration{"tenant-a": 24 * time.Hour},
+			outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 5 * time.Minute},
+			outOfOrderWindow:          map[string]time.Duration{"tenant-a": 1 * time.Hour},
+			expectedTTL:               24 * time.Hour,
 		},
 		{
-			name: "recent data uses out_of_order_results_cache_ttl",
+			name:      "recent data uses out_of_order_results_cache_ttl",
+			tenantIDs: []string{"tenant-a"},
 			extents: []tripperware.Extent{
 				{Start: twoHoursAgo, End: oneHourAgo}, // overlaps with 1h window
 			},
-			resultsCacheTTL:    24 * time.Hour,
-			outOfOrderCacheTTL: 5 * time.Minute,
-			outOfOrderWindow:   1 * time.Hour,
-			expectedTTL:        5 * time.Minute,
+			resultsCacheTTL:           map[string]time.Duration{"tenant-a": 24 * time.Hour},
+			outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 5 * time.Minute},
+			outOfOrderWindow:          map[string]time.Duration{"tenant-a": 1 * time.Hour},
+			expectedTTL:               5 * time.Minute,
 		},
 		{
-			name: "zero out-of-order window uses results_cache_ttl",
+			name:      "zero out-of-order window uses results_cache_ttl",
+			tenantIDs: []string{"tenant-a"},
 			extents: []tripperware.Extent{
 				{Start: twoHoursAgo, End: oneHourAgo},
 			},
-			resultsCacheTTL:    12 * time.Hour,
-			outOfOrderCacheTTL: 5 * time.Minute,
-			outOfOrderWindow:   0, // no out-of-order support
-			expectedTTL:        12 * time.Hour,
+			resultsCacheTTL:           map[string]time.Duration{"tenant-a": 12 * time.Hour},
+			outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 5 * time.Minute},
+			outOfOrderWindow:          map[string]time.Duration{"tenant-a": 0}, // no out-of-order support
+			expectedTTL:               12 * time.Hour,
 		},
 		{
-			name: "zero TTLs use backend defaults",
+			name:      "recent data falls back to results_cache_ttl when out_of_order_results_cache_ttl is unset",
+			tenantIDs: []string{"tenant-a"},
+			extents: []tripperware.Extent{
+				{Start: twoHoursAgo, End: oneHourAgo}, // overlaps with 1h window
+			},
+			resultsCacheTTL:           map[string]time.Duration{"tenant-a": 24 * time.Hour},
+			outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 0}, // not set, should fall back to results_cache_ttl
+			outOfOrderWindow:          map[string]time.Duration{"tenant-a": 1 * time.Hour},
+			expectedTTL:               24 * time.Hour,
+		},
+		{
+			name:      "recent data uses backend default when both TTLs are unset",
+			tenantIDs: []string{"tenant-a"},
+			extents: []tripperware.Extent{
+				{Start: twoHoursAgo, End: oneHourAgo}, // overlaps with 1h window
+			},
+			resultsCacheTTL:           map[string]time.Duration{"tenant-a": 0},
+			outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 0},
+			outOfOrderWindow:          map[string]time.Duration{"tenant-a": 1 * time.Hour},
+			expectedTTL:               globalDefaultTTL, // TTL of 0 falls back to the backend default
+		},
+		{
+			name:      "zero TTLs use backend defaults",
+			tenantIDs: []string{"tenant-a"},
 			extents: []tripperware.Extent{
 				{Start: twoHoursAgo, End: twoHoursAgo + 1000},
 			},
-			resultsCacheTTL:    0,
-			outOfOrderCacheTTL: 0,
-			outOfOrderWindow:   0,
-			expectedTTL:        0, // backend default
+			resultsCacheTTL:           map[string]time.Duration{"tenant-a": 0},
+			outOfOrderResultsCacheTTL: map[string]time.Duration{"tenant-a": 0},
+			outOfOrderWindow:          map[string]time.Duration{"tenant-a": 0},
+			expectedTTL:               globalDefaultTTL, // TTL of 0 falls back to the backend default
+		},
+		{
+			// Federated query: tenant "a" leaves its OOO TTL unset, so it falls back to
+			// its own results_cache_ttl (30m); tenant "b" has 1h. The most restrictive is
+			// 30m. The fallback must be resolved per-tenant *before* aggregating across
+			// tenants — aggregating the raw configs first would instead yield 1h.
+			name:      "recent data resolves fallback per-tenant before aggregating",
+			tenantIDs: []string{"a", "b"},
+			extents: []tripperware.Extent{
+				{Start: twoHoursAgo, End: oneHourAgo}, // overlaps with 1h window
+			},
+			resultsCacheTTL:           map[string]time.Duration{"a": 30 * time.Minute, "b": 24 * time.Hour},
+			outOfOrderResultsCacheTTL: map[string]time.Duration{"a": 0, "b": 1 * time.Hour},
+			outOfOrderWindow:          map[string]time.Duration{"a": 1 * time.Hour, "b": 1 * time.Hour},
+			expectedTTL:               30 * time.Minute,
 		},
 	}
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			mockCache := cache.NewMockCache()
-			limits := mockLimits{
+			mockCache.(*cache.MockCache).DefaultTTL = globalDefaultTTL
+			limits := perTenantLimits{
 				resultsCacheTTL:           tc.resultsCacheTTL,
-				outOfOrderResultsCacheTTL: tc.outOfOrderCacheTTL,
+				outOfOrderResultsCacheTTL: tc.outOfOrderResultsCacheTTL,
 				outOfOrderWindow:          tc.outOfOrderWindow,
 			}
 
@@ -1956,15 +2005,35 @@ func TestResultsCachePutTTLSelection(t *testing.T) {
 			rc := rm.Wrap(nil).(*resultsCache)
 			rc.now = func() time.Time { return now }
 
-			ctx := user.InjectOrgID(context.Background(), "tenant-a")
-			tenantIDs, _ := users.TenantIDs(ctx)
-			rc.put(ctx, "test-key", tc.extents, tenantIDs)
+			ctx := user.InjectOrgID(context.Background(), strings.Join(tc.tenantIDs, "|"))
+			rc.put(ctx, "test-key", tc.extents, tc.tenantIDs)
 
 			assert.Equal(t, tc.expectedTTL, mockCache.(*cache.MockCache).GetLastTTL())
 		})
 	}
 }
 
+// perTenantLimits returns different TTL/window values per tenant so we can verify
+// that the out-of-order TTL fallback is resolved per-tenant before aggregating.
+type perTenantLimits struct {
+	mockLimits
+	resultsCacheTTL           map[string]time.Duration
+	outOfOrderResultsCacheTTL map[string]time.Duration
+	outOfOrderWindow          map[string]time.Duration
+}
+
+func (m perTenantLimits) ResultsCacheTTL(userID string) time.Duration {
+	return m.resultsCacheTTL[userID]
+}
+
+func (m perTenantLimits) OutOfOrderResultsCacheTTL(userID string) time.Duration {
+	return m.outOfOrderResultsCacheTTL[userID]
+}
+
+func (m perTenantLimits) OutOfOrderTimeWindow(userID string) model.Duration {
+	return model.Duration(m.outOfOrderWindow[userID])
+}
+
 type mockResolver struct {
 	tenantIDs []string
 	err       error

pkg/util/validation/limits.go

@@ -343,9 +343,10 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	_ = l.MaxCacheFreshness.Set("1m")
 	f.Int64Var(&l.MaxQueryResponseSize, "frontend.max-query-response-size", 0, "The maximum total uncompressed query response size. If the query was sharded the limit is applied to the total response size of all shards. This limit is enforced in query-frontend for `query` and `query_range` APIs. 0 to disable.")
 	f.Var(&l.MaxCacheFreshness, "frontend.max-cache-freshness", "Most recent allowed cacheable result per-tenant, to prevent caching very recent results that might still be in flux.")
-	// ResultsCacheTTL and OutOfOrderResultsCacheTTL default to 0 (use global cache config expiration)
+	// ResultsCacheTTL and OutOfOrderResultsCacheTTL default to 0. ResultsCacheTTL falls back to the
+	// global cache config expiration; OutOfOrderResultsCacheTTL falls back to ResultsCacheTTL first.
 	f.Var(&l.ResultsCacheTTL, "frontend.results-cache-ttl", "Per-tenant TTL for cached query results in the cache backend (Memcached/Redis/FIFO). This is the standard TTL for results that do not overlap with the out-of-order time window. 0 (default) means use the global cache backend TTL configuration.")
-	f.Var(&l.OutOfOrderResultsCacheTTL, "frontend.out-of-order-results-cache-ttl", "Per-tenant TTL for cached query results that overlap with the out-of-order time window. These results may still receive out-of-order samples, so they typically use a shorter TTL. 0 (default) means use the global cache backend TTL configuration.")
+	f.Var(&l.OutOfOrderResultsCacheTTL, "frontend.out-of-order-results-cache-ttl", "Per-tenant TTL for cached query results that overlap with the out-of-order time window. These results may still receive out-of-order samples, so they typically use a shorter TTL. 0 (default) means fall back to frontend.results-cache-ttl, and if that is also 0, use the global cache backend TTL configuration.")
 	f.Float64Var(&l.MaxQueriersPerTenant, "frontend.max-queriers-per-tenant", 0, "Maximum number of queriers that can handle requests for a single tenant. If set to 0 or value higher than number of available queriers, *all* queriers will handle requests for the tenant. If the value is < 1, it will be treated as a percentage and the gets a percentage of the total queriers. Each frontend (or query-scheduler, if used) will select the same set of queriers for the same tenant (given that all queriers are connected to all frontends / query-schedulers). This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL.")
 	f.IntVar(&l.QueryVerticalShardSize, "frontend.query-vertical-shard-size", 0, "[Experimental] Number of shards to use when distributing shardable PromQL queries.")
 	f.BoolVar(&l.QueryPriority.Enabled, "frontend.query-priority.enabled", false, "Whether queries are assigned with priorities.")

schemas/cortex-config-schema.json

@@ -5975,7 +5975,7 @@
         },
         "out_of_order_results_cache_ttl": {
           "default": "0s",
-          "description": "Per-tenant TTL for cached query results that overlap with the out-of-order time window. These results may still receive out-of-order samples, so they typically use a shorter TTL. 0 (default) means use the global cache backend TTL configuration.",
+          "description": "Per-tenant TTL for cached query results that overlap with the out-of-order time window. These results may still receive out-of-order samples, so they typically use a shorter TTL. 0 (default) means fall back to frontend.results-cache-ttl, and if that is also 0, use the global cache backend TTL configuration.",
           "type": "string",
           "x-cli-flag": "frontend.out-of-order-results-cache-ttl",
           "x-format": "duration"