From ed2135f59c3931205b842ed596b5639e29610f88 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Tue, 18 Aug 2026 00:05:04 -0700 Subject: [PATCH] Distributor: stop counting metadata towards the ingestion rate limit The -distributor.ingestion-rate-limit flag is documented as a limit in samples per second, but metadata was folded into the same budget as samples and exemplars. Metadata is not sample data, and its volume is already bounded separately by -ingester.max-metadata-per-user and -ingester.max-metadata-per-metric. This mattered most for Prometheus Remote Write 2.0, which attaches metadata to every series. A metric family with N series contributed N metadata entries to the budget, so a tenant consumed roughly twice its configured limit for the same data. The ingester instance ingestion rate, used by -ingester.instance-limits.max-ingestion-rate, stops counting metadata for the same reason. Its comment already stated that it mirrored the distributor. The 429 message no longer reports a metadata count, since metadata no longer contributes to exceeding the limit. Metadata carried by a rejected request is still counted as discarded. Upstream Prometheus takes the same position on the sender side: metadata is not a sample and is not counted against max_samples_per_send (prometheus/prometheus#14407). Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- CHANGELOG.md | 1 + pkg/distributor/distributor.go | 16 +++++++-- pkg/distributor/distributor_test.go | 52 +++++++++++++++++++++-------- pkg/ingester/ingester.go | 9 +++-- 4 files changed, 58 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abd8bcc7927..b92d997f3d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7160 * [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446 * [CHANGE] HA Tracker: Move `-distributor.ha-tracker.failover-timeout` from a global config to a per-tenant runtime config. The flag name and default value (30s) remain the same. #7481 +* [CHANGE] Distributor: Metric metadata no longer counts towards `-distributor.ingestion-rate-limit`, which is documented as a limit in samples per second. This applies to both remote write 1.0 and 2.0, because the limit is enforced on the shared push path. Remote write 2.0 is affected most, since it attaches metadata to every series and so consumed roughly twice the configured limit for the same data, but 1.0 senders are affected too, as Prometheus sends metadata by default (`metadata_config.send`, batched up to 2000 entries per request). Metadata-only requests are therefore no longer rate limited at all, and metadata volume remains bounded only by `-ingester.max-metadata-per-user` and `-ingester.max-metadata-per-metric`. Tenants near their limit will see fewer 429s, and the 429 message no longer reports a metadata count. The ingester instance ingestion rate used by `-ingester.instance-limits.max-ingestion-rate` also stops counting metadata, for consistency. #7779 * [FEATURE] Parquet: Support sharded parquet file conversion and querying. #7610 * [FEATURE] Parquet Converter: Add experimental `-parquet-converter.max-num-columns` flag to automatically shard parquet files when the number of columns exceeds the configured limit. This prevents failures when a TSDB block has more unique label names than the parquet library's column limit (32767). #7624 * [FEATURE] Distributor: Add experimental `-distributor.num-query-workers` flag to use a goroutine worker pool for query fan-out calls to ingesters. Reuses pre-grown goroutine stacks to eliminate the `runtime.copystack` overhead (~8% CPU) observed on rulers with wide ingester fan-out. Falls back to spawning a new goroutine when no worker is available. #7623 diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index edf9b750b87..b443b0df07c 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -860,18 +860,28 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co } totalSamples := validatedFloatSamples + validatedHistogramSamples - totalN := totalSamples + validatedExemplars + len(validatedMetadata) + // Metadata does not count towards the ingestion rate limit. The limit is documented as + // "samples per second" (-distributor.ingestion-rate-limit), and metadata is not sample + // data. Metadata volume is bounded separately by -ingester.max-metadata-per-user and + // -ingester.max-metadata-per-metric. + // + // This matters most for remote write 2.0, which attaches metadata to every series, so + // counting it made a tenant consume roughly twice the configured limit for the same data. + totalN := totalSamples + validatedExemplars if !d.ingestionRateLimiter.AllowN(now, userID, totalN) { d.validateMetrics.DiscardedSamples.WithLabelValues(validation.RateLimited, userID).Add(float64(totalSamples)) d.validateMetrics.DiscardedExemplars.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedExemplars)) + // The whole request is rejected, so any metadata it carried is discarded too, even + // though metadata did not contribute to exceeding the limit. d.validateMetrics.DiscardedMetadata.WithLabelValues(validation.RateLimited, userID).Add(float64(len(validatedMetadata))) // Return a 429 here to tell the client it is going too fast. // Client may discard the data or slow down and re-send. // Prometheus v2.26 added a remote-write option 'retry_on_http_429'. - return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (%v) exceeded while adding %d samples and %d metadata", d.ingestionRateLimiter.Limit(now, userID), totalSamples, len(validatedMetadata)) + return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (%v) exceeded while adding %d samples", d.ingestionRateLimiter.Limit(now, userID), totalSamples) } - // totalN included samples and metadata. Ingester follows this pattern when computing its ingestion rate. + // totalN counts samples and exemplars, but not metadata. Ingester follows this pattern + // when computing its ingestion rate. d.ingestionRate.Add(int64(totalN)) var nativeHistogramErr error diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 2cb0b1522f3..0ebd8ad1a77 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -726,11 +726,15 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { ingestionBurstSize: 10, pushes: []testPush{ {samples: 4, expectedError: nil}, + // Metadata does not consume any of the rate budget. + {metadata: 1, expectedError: nil}, + {samples: 6, expectedError: nil}, + // The budget is exhausted by the 10 samples above, so the samples in this + // request are rejected even though its metadata costs nothing. + {samples: 4, metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 4 samples")}, + {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 1 samples")}, + // Metadata only requests are still accepted once the sample budget is gone. {metadata: 1, expectedError: nil}, - {samples: 6, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 6 samples and 0 metadata")}, - {samples: 4, metadata: 1, expectedError: nil}, - {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 1 samples and 0 metadata")}, - {metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 0 samples and 1 metadata")}, }, }, "global strategy: limit should be evenly shared across distributors": { @@ -741,10 +745,10 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { pushes: []testPush{ {samples: 2, expectedError: nil}, {samples: 1, expectedError: nil}, - {samples: 2, metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 2 samples and 1 metadata")}, - {samples: 2, expectedError: nil}, - {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples and 0 metadata")}, - {metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 0 samples and 1 metadata")}, + {samples: 2, metadata: 1, expectedError: nil}, + {samples: 2, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 2 samples")}, + {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples")}, + {metadata: 1, expectedError: nil}, }, }, "global strategy: burst should set to each distributor": { @@ -755,10 +759,27 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { pushes: []testPush{ {samples: 10, expectedError: nil}, {samples: 5, expectedError: nil}, - {samples: 5, metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 5 samples and 1 metadata")}, - {samples: 5, expectedError: nil}, - {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples and 0 metadata")}, - {metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 0 samples and 1 metadata")}, + {samples: 5, metadata: 1, expectedError: nil}, + {samples: 5, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 5 samples")}, + {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples")}, + {metadata: 1, expectedError: nil}, + }, + }, + "metadata does not count towards the ingestion rate limit": { + distributors: 2, + ingestionRateStrategy: validation.LocalIngestionRateStrategy, + ingestionRate: 10, + ingestionBurstSize: 10, + pushes: []testPush{ + // Metadata alone is never rate limited, so these are accepted even though + // each request carries far more metadata than the configured limit. Metadata + // volume is bounded by the max-metadata-per-user and max-metadata-per-metric + // limits instead. + {metadata: 20, expectedError: nil}, + {metadata: 20, expectedError: nil}, + // The full sample budget is still intact afterwards. + {samples: 10, expectedError: nil}, + {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 1 samples")}, }, }, } @@ -923,8 +944,11 @@ func TestDistributor_PushIngestionRateLimiter_Histograms(t *testing.T) { nativeHistogramIngestionBurstSize: 10, pushes: []testPush{ {samples: 4, nhSamples: 4, metadata: 4, expectedError: nil}, - {samples: 4, nhSamples: 4, metadata: 4, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 8 samples and 4 metadata")}, - {samples: 3, nhSamples: 3, metadata: 2, expectedError: nil}, + // The 4 metadata in each request no longer consume budget, so this one now + // fits: 8 samples against the 12 remaining of the burst. + {samples: 4, nhSamples: 4, metadata: 4, expectedError: nil}, + // Only 4 of the burst is left, so these 6 samples are rejected. + {samples: 3, nhSamples: 3, metadata: 2, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 6 samples")}, }, }, } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 3a6c03449c6..05bb97357ea 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1399,7 +1399,9 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte // Given metadata is a best-effort approach, and we don't halt on errors // process it before samples. Otherwise, we risk returning an error before ingestion. - ingestedMetadata := i.pushMetadata(ctx, userID, req.GetMetadata()) + // The ingested count is deliberately discarded: metadata does not count towards the + // ingestion rate, and pushMetadata already records its own metrics. + i.pushMetadata(ctx, userID, req.GetMetadata()) reasonCounter := newLabelSetReasonCounters() @@ -1806,8 +1808,9 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte } } - // Distributor counts both samples, metadata and histograms, so for consistency ingester does the same. - i.ingestionRate.Add(int64(succeededSamplesCount + succeededHistogramsCount + ingestedMetadata)) + // Distributor counts samples and histograms but not metadata, so for consistency ingester + // does the same. + i.ingestionRate.Add(int64(succeededSamplesCount + succeededHistogramsCount)) switch req.Source { case cortexpb.RULE: