Distributor: stop counting metadata towards the ingestion rate limit

CHANGELOG.md

@@ -4,6 +4,7 @@
 * [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7160
 * [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446
 * [CHANGE] HA Tracker: Move `-distributor.ha-tracker.failover-timeout` from a global config to a per-tenant runtime config. The flag name and default value (30s) remain the same. #7481
+* [CHANGE] Distributor: Metric metadata no longer counts towards `-distributor.ingestion-rate-limit`, which is documented as a limit in samples per second. This applies to both remote write 1.0 and 2.0, because the limit is enforced on the shared push path. Remote write 2.0 is affected most, since it attaches metadata to every series and so consumed roughly twice the configured limit for the same data, but 1.0 senders are affected too, as Prometheus sends metadata by default (`metadata_config.send`, batched up to 2000 entries per request). Metadata-only requests are therefore no longer rate limited at all, and metadata volume remains bounded only by `-ingester.max-metadata-per-user` and `-ingester.max-metadata-per-metric`. Tenants near their limit will see fewer 429s, and the 429 message no longer reports a metadata count. The ingester instance ingestion rate used by `-ingester.instance-limits.max-ingestion-rate` also stops counting metadata, for consistency. #7779
 * [FEATURE] Parquet: Support sharded parquet file conversion and querying. #7610
 * [FEATURE] Parquet Converter: Add experimental `-parquet-converter.max-num-columns` flag to automatically shard parquet files when the number of columns exceeds the configured limit. This prevents failures when a TSDB block has more unique label names than the parquet library's column limit (32767). #7624
 * [FEATURE] Distributor: Add experimental `-distributor.num-query-workers` flag to use a goroutine worker pool for query fan-out calls to ingesters. Reuses pre-grown goroutine stacks to eliminate the `runtime.copystack` overhead (~8% CPU) observed on rulers with wide ingester fan-out. Falls back to spawning a new goroutine when no worker is available. #7623

pkg/distributor/distributor.go

@@ -860,18 +860,28 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co
 	}
 
 	totalSamples := validatedFloatSamples + validatedHistogramSamples
-	totalN := totalSamples + validatedExemplars + len(validatedMetadata)
+	// Metadata does not count towards the ingestion rate limit. The limit is documented as
+	// "samples per second" (-distributor.ingestion-rate-limit), and metadata is not sample
+	// data. Metadata volume is bounded separately by -ingester.max-metadata-per-user and
+	// -ingester.max-metadata-per-metric.
+	//
+	// This matters most for remote write 2.0, which attaches metadata to every series, so
+	// counting it made a tenant consume roughly twice the configured limit for the same data.
+	totalN := totalSamples + validatedExemplars
 	if !d.ingestionRateLimiter.AllowN(now, userID, totalN) {
 		d.validateMetrics.DiscardedSamples.WithLabelValues(validation.RateLimited, userID).Add(float64(totalSamples))
 		d.validateMetrics.DiscardedExemplars.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedExemplars))
+		// The whole request is rejected, so any metadata it carried is discarded too, even
+		// though metadata did not contribute to exceeding the limit.
 		d.validateMetrics.DiscardedMetadata.WithLabelValues(validation.RateLimited, userID).Add(float64(len(validatedMetadata)))
 		// Return a 429 here to tell the client it is going too fast.
 		// Client may discard the data or slow down and re-send.
 		// Prometheus v2.26 added a remote-write option 'retry_on_http_429'.
-		return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (%v) exceeded while adding %d samples and %d metadata", d.ingestionRateLimiter.Limit(now, userID), totalSamples, len(validatedMetadata))
+		return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (%v) exceeded while adding %d samples", d.ingestionRateLimiter.Limit(now, userID), totalSamples)
 	}
 
-	// totalN included samples and metadata. Ingester follows this pattern when computing its ingestion rate.
+	// totalN counts samples and exemplars, but not metadata. Ingester follows this pattern
+	// when computing its ingestion rate.
 	d.ingestionRate.Add(int64(totalN))
 
 	var nativeHistogramErr error

pkg/distributor/distributor_test.go

@@ -726,11 +726,15 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) {
 			ingestionBurstSize:    10,
 			pushes: []testPush{
 				{samples: 4, expectedError: nil},
+				// Metadata does not consume any of the rate budget.
+				{metadata: 1, expectedError: nil},
+				{samples: 6, expectedError: nil},
+				// The budget is exhausted by the 10 samples above, so the samples in this
+				// request are rejected even though its metadata costs nothing.
+				{samples: 4, metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 4 samples")},
+				{samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 1 samples")},
+				// Metadata only requests are still accepted once the sample budget is gone.
 				{metadata: 1, expectedError: nil},
-				{samples: 6, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 6 samples and 0 metadata")},
-				{samples: 4, metadata: 1, expectedError: nil},
-				{samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 1 samples and 0 metadata")},
-				{metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 0 samples and 1 metadata")},
 			},
 		},
 		"global strategy: limit should be evenly shared across distributors": {
@@ -741,10 +745,10 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) {
 			pushes: []testPush{
 				{samples: 2, expectedError: nil},
 				{samples: 1, expectedError: nil},
-				{samples: 2, metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 2 samples and 1 metadata")},
-				{samples: 2, expectedError: nil},
-				{samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples and 0 metadata")},
-				{metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 0 samples and 1 metadata")},
+				{samples: 2, metadata: 1, expectedError: nil},
+				{samples: 2, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 2 samples")},
+				{samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples")},
+				{metadata: 1, expectedError: nil},
 			},
 		},
 		"global strategy: burst should set to each distributor": {
@@ -755,10 +759,27 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) {
 			pushes: []testPush{
 				{samples: 10, expectedError: nil},
 				{samples: 5, expectedError: nil},
-				{samples: 5, metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 5 samples and 1 metadata")},
-				{samples: 5, expectedError: nil},
-				{samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples and 0 metadata")},
-				{metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 0 samples and 1 metadata")},
+				{samples: 5, metadata: 1, expectedError: nil},
+				{samples: 5, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 5 samples")},
+				{samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples")},
+				{metadata: 1, expectedError: nil},
+			},
+		},
+		"metadata does not count towards the ingestion rate limit": {
+			distributors:          2,
+			ingestionRateStrategy: validation.LocalIngestionRateStrategy,
+			ingestionRate:         10,
+			ingestionBurstSize:    10,
+			pushes: []testPush{
+				// Metadata alone is never rate limited, so these are accepted even though
+				// each request carries far more metadata than the configured limit. Metadata
+				// volume is bounded by the max-metadata-per-user and max-metadata-per-metric
+				// limits instead.
+				{metadata: 20, expectedError: nil},
+				{metadata: 20, expectedError: nil},
+				// The full sample budget is still intact afterwards.
+				{samples: 10, expectedError: nil},
+				{samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 1 samples")},
 			},
 		},
 	}
@@ -923,8 +944,11 @@ func TestDistributor_PushIngestionRateLimiter_Histograms(t *testing.T) {
 			nativeHistogramIngestionBurstSize: 10,
 			pushes: []testPush{
 				{samples: 4, nhSamples: 4, metadata: 4, expectedError: nil},
-				{samples: 4, nhSamples: 4, metadata: 4, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 8 samples and 4 metadata")},
-				{samples: 3, nhSamples: 3, metadata: 2, expectedError: nil},
+				// The 4 metadata in each request no longer consume budget, so this one now
+				// fits: 8 samples against the 12 remaining of the burst.
+				{samples: 4, nhSamples: 4, metadata: 4, expectedError: nil},
+				// Only 4 of the burst is left, so these 6 samples are rejected.
+				{samples: 3, nhSamples: 3, metadata: 2, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 6 samples")},
 			},
 		},
 	}

pkg/ingester/ingester.go

@@ -1399,7 +1399,9 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 
 	// Given metadata is a best-effort approach, and we don't halt on errors
 	// process it before samples. Otherwise, we risk returning an error before ingestion.
-	ingestedMetadata := i.pushMetadata(ctx, userID, req.GetMetadata())
+	// The ingested count is deliberately discarded: metadata does not count towards the
+	// ingestion rate, and pushMetadata already records its own metrics.
+	i.pushMetadata(ctx, userID, req.GetMetadata())
 
 	reasonCounter := newLabelSetReasonCounters()
 
@@ -1806,8 +1808,9 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 		}
 	}
 
-	// Distributor counts both samples, metadata and histograms, so for consistency ingester does the same.
-	i.ingestionRate.Add(int64(succeededSamplesCount + succeededHistogramsCount + ingestedMetadata))
+	// Distributor counts samples and histograms but not metadata, so for consistency ingester
+	// does the same.
+	i.ingestionRate.Add(int64(succeededSamplesCount + succeededHistogramsCount))
 
 	switch req.Source {
 	case cortexpb.RULE: