diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go index b2fdd382178..1ef2b8cbc9f 100644 --- a/pkg/receive/handler.go +++ b/pkg/receive/handler.go @@ -924,15 +924,16 @@ func (h *Handler) distributeTimeseriesToReplicas( for tsIndex, ts := range timeseries { var tenant = tenantHTTP - if h.splitTenantLabelName != "" { + // Only extract tenant from split label when running as a router. + // Ingestors should use the tenant determined by the router. + if h.splitTenantLabelName != "" && h.receiverMode != IngestorOnly { lbls := labelpb.ZLabelsToPromLabels(ts.Labels) tenantLabel := lbls.Get(h.splitTenantLabelName) if tenantLabel != "" { tenant = h.splitTenantLabelName + ":" + tenantLabel - } else { - tenant = h.options.DefaultTenantID } + // If tenant label doesn't exist, keep the tenant from HTTP header (tenantHTTP) } for _, rn := range replicas { @@ -1017,11 +1018,14 @@ func (h *Handler) sendLocalWrite( tenantSeriesMapping := map[string][]prompb.TimeSeries{} for _, ts := range trackedSeries.timeSeries { var tenant = tenantHTTP - if h.splitTenantLabelName != "" { + // Only extract tenant from split label when running as a router. + // Ingestors should use the tenant determined by the router. + if h.splitTenantLabelName != "" && h.receiverMode != IngestorOnly { lbls := labelpb.ZLabelsToPromLabels(ts.Labels) if tnt := lbls.Get(h.splitTenantLabelName); tnt != "" { - tenant = tnt + tenant = h.splitTenantLabelName + ":" + tnt } + // If tenant label doesn't exist, keep the tenant from HTTP header (tenantHTTP) } tenantSeriesMapping[tenant] = append(tenantSeriesMapping[tenant], ts) } diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index c938a4f0407..f5a5609b389 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -1052,8 +1052,8 @@ func TestReceiveExtractsTenant(t *testing.T) { Timeseries: []prompb.TimeSeries{ { Labels: []prompb.Label{ - {Name: tenantLabelName, Value: "tenant-1"}, {Name: "aa", Value: "bb"}, + {Name: tenantLabelName, Value: "tenant-1"}, }, Samples: []prompb.Sample{ {Value: 1, Timestamp: time.Now().UnixMilli()}, @@ -1063,7 +1063,7 @@ func TestReceiveExtractsTenant(t *testing.T) { }) })) - testutil.Ok(t, i.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"prometheus_tsdb_blocks_loaded"}, e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tenant", "tenant-1")), e2emon.WaitMissingMetrics())) + testutil.Ok(t, i.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"prometheus_tsdb_blocks_loaded"}, e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tenant", tenantLabelName+":tenant-1")), e2emon.WaitMissingMetrics())) }) t.Run("tenant label is extracted from one series, default is used for the other one", func(t *testing.T) { @@ -1072,8 +1072,8 @@ func TestReceiveExtractsTenant(t *testing.T) { Timeseries: []prompb.TimeSeries{ { Labels: []prompb.Label{ - {Name: tenantLabelName, Value: "tenant-2"}, {Name: "aa", Value: "bb"}, + {Name: tenantLabelName, Value: "tenant-2"}, }, Samples: []prompb.Sample{ {Value: 1, Timestamp: time.Now().UnixMilli()}, @@ -1101,8 +1101,8 @@ func TestReceiveExtractsTenant(t *testing.T) { Timeseries: []prompb.TimeSeries{ { Labels: []prompb.Label{ - {Name: tenantLabelName, Value: "tenant-3"}, {Name: "aa", Value: "bb"}, + {Name: tenantLabelName, Value: "tenant-3"}, }, Samples: []prompb.Sample{ {Value: 1, Timestamp: time.Now().UnixMilli()}, @@ -1149,7 +1149,7 @@ func TestReceiveExtractsTenant(t *testing.T) { })) testutil.Ok(t, i.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"prometheus_tsdb_blocks_loaded"}, e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tenant", "http-tenant")), e2emon.WaitMissingMetrics())) - testutil.Ok(t, i.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"prometheus_tsdb_blocks_loaded"}, e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tenant", "tenant-3")), e2emon.WaitMissingMetrics())) + testutil.Ok(t, i.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"prometheus_tsdb_blocks_loaded"}, e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tenant", tenantLabelName+":tenant-3")), e2emon.WaitMissingMetrics())) }) }