Trt 2389 test summaries

pkg/jobrunaggregator/jobrunaggregatorapi/types_row_test_summary.go

@@ -0,0 +1,20 @@
+package jobrunaggregatorapi
+
+import (
+	"cloud.google.com/go/civil"
+)
+
+// TestSummaryByPeriodRow represents aggregated test results for a specific suite and release over a time period.
+// This data structure corresponds to the suite_summary_by_period.sql query results.
+type TestSummaryByPeriodRow struct {
+	Release           string     `bigquery:"release"`
+	TestName          string     `bigquery:"test_name"`
+	TotalTestCount    int64      `bigquery:"total_test_count"`
+	TotalFailureCount int64      `bigquery:"total_failure_count"`
+	TotalFlakeCount   int64      `bigquery:"total_flake_count"`
+	FailureRate       float64    `bigquery:"failure_rate"`
+	AvgDurationMs     float64    `bigquery:"avg_duration_ms"`
+	PeriodStart       civil.Date `bigquery:"period_start"`
+	PeriodEnd         civil.Date `bigquery:"period_end"`
+	DaysWithData      int64      `bigquery:"days_with_data"`
+}

pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client.go

@@ -66,6 +66,14 @@ type CIDataClient interface {
 
 	// ListReleases lists all releases from the new release table
 	ListReleases(ctx context.Context) ([]jobrunaggregatorapi.ReleaseRow, error)
+
+	// ListTestSummaryByPeriod retrieves aggregated test results for a specific suite and release over a time period.
+	// Parameters:
+	//   - suiteName: The test suite to query (e.g., 'conformance')
+	//   - releaseName: The release version (e.g., '4.15')
+	//   - daysBack: Number of days to look back from current date
+	//   - minTestCount: Minimum number of test executions required to include a test in results
+	ListTestSummaryByPeriod(ctx context.Context, suiteName, releaseName string, daysBack, minTestCount int) ([]jobrunaggregatorapi.TestSummaryByPeriodRow, error)
 }
 
 type ciDataClient struct {
@@ -1095,3 +1103,71 @@ func (c *ciDataClient) ListAllKnownAlerts(ctx context.Context) ([]*jobrunaggrega
 
 	return allKnownAlerts, nil
 }
+
+func (c *ciDataClient) ListTestSummaryByPeriod(ctx context.Context, suiteName, releaseName string, daysBack, minTestCount int) ([]jobrunaggregatorapi.TestSummaryByPeriodRow, error) {
+	// Query to summarize test results for a specific suite and release over a time period
+	// Groups by release and test_name only (no infrastructure dimensions)
+	// Calculates total test_count, failure_count, flake_count, failure_rate, and avg_duration_ms
+	// Filters results to only include tests with sufficient test runs
+	queryString := c.dataCoordinates.SubstituteDataSetLocation(`
+SELECT
+  release,
+  test_name,
+  SUM(test_count) AS total_test_count,
+  SUM(failure_count) AS total_failure_count,
+  SUM(flake_count) AS total_flake_count,
+  SAFE_DIVIDE(SUM(failure_count), SUM(test_count)) AS failure_rate,
+  AVG(avg_duration_ms) AS avg_duration_ms,
+  MIN(date) AS period_start,
+  MAX(date) AS period_end,
+  COUNT(DISTINCT date) AS days_with_data
+FROM
+  DATA_SET_LOCATION.TestsSummaryByDate
+WHERE
+  suite = @suite_name
+  AND release = @release_name
+  AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL @days_back DAY)
+  AND date <= CURRENT_DATE()
+GROUP BY
+  release,
+  test_name
+HAVING
+  SUM(test_count) > @min_test_count
+ORDER BY
+  release,
+  total_failure_count DESC,
+  test_name
+`)
+
+	query := c.client.Query(queryString)
+	query.Labels = map[string]string{
+		bigQueryLabelKeyApp:   bigQueryLabelValueApp,
+		bigQueryLabelKeyQuery: bigQueryLabelValueTestSummaryByPeriod,
+	}
+	query.QueryConfig.Parameters = []bigquery.QueryParameter{
+		{Name: "suite_name", Value: suiteName},
+		{Name: "release_name", Value: releaseName},
+		{Name: "days_back", Value: daysBack},
+		{Name: "min_test_count", Value: minTestCount},
+	}
+
+	rows, err := query.Read(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query generic test summary by period with %q: %w", queryString, err)
+	}
+
+	results := []jobrunaggregatorapi.TestSummaryByPeriodRow{}
+	for {
+		row := &jobrunaggregatorapi.TestSummaryByPeriodRow{}
+		err = rows.Next(row)
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, *row)
+	}
+
+	return results, nil
+}

pkg/jobrunaggregator/jobrunaggregatorlib/retrying_ci_data_client.go

@@ -185,6 +185,16 @@ func (c *retryingCIDataClient) ListAllKnownAlerts(ctx context.Context) ([]*jobru
 	return ret, err
 }
 
+func (c *retryingCIDataClient) ListTestSummaryByPeriod(ctx context.Context, suiteName, releaseName string, daysBack, minTestCount int) ([]jobrunaggregatorapi.TestSummaryByPeriodRow, error) {
+	var ret []jobrunaggregatorapi.TestSummaryByPeriodRow
+	err := retry.OnError(slowBackoff, isReadQuotaError, func() error {
+		var innerErr error
+		ret, innerErr = c.delegate.ListTestSummaryByPeriod(ctx, suiteName, releaseName, daysBack, minTestCount)
+		return innerErr
+	})
+	return ret, err
+}
+
 var slowBackoff = wait.Backoff{
 	Steps:    4,
 	Duration: 10 * time.Second,

pkg/jobrunaggregator/jobrunaggregatorlib/util.go

@@ -54,6 +54,7 @@ const (
 	bigQueryLabelValueAllReleases              = "aggregator-all-releases"
 	bigQueryLabelValueReleaseTags              = "aggregator-release-tags"
 	bigQueryLabelValueJobRunIDsSinceTime       = "aggregator-job-run-ids-since-time"
+	bigQueryLabelValueTestSummaryByPeriod      = "aggregator-test-summary-by-period"
 )
 
 var (

pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/analyzer.go

@@ -24,8 +24,6 @@ type JobRunHistoricalDataAnalyzerOptions struct {
 
 func (o *JobRunHistoricalDataAnalyzerOptions) Run(ctx context.Context) error {
 
-	var newHistoricalData []jobrunaggregatorapi.HistoricalData
-
 	// targetRelease will either be what the caller specified on the CLI, or the most recent release.
 	// previousRelease will be the one prior to targetRelease.
 	var targetRelease, previousRelease string
@@ -44,6 +42,14 @@ func (o *JobRunHistoricalDataAnalyzerOptions) Run(ctx context.Context) error {
 	}
 	fmt.Printf("Using target release: %s, previous release: %s\n", targetRelease, previousRelease)
 
+	// For tests data type, we don't do comparison - just fetch and write directly
+	if o.dataType == "tests" {
+		return o.runTestsDataType(ctx, targetRelease)
+	}
+
+	// For other data types (alerts, disruptions), continue with comparison logic
+	var newHistoricalData []jobrunaggregatorapi.HistoricalData
+
 	currentHistoricalData, err := readHistoricalDataFile(o.currentFile, o.dataType)
 	if err != nil {
 		return err
@@ -91,6 +97,40 @@ func (o *JobRunHistoricalDataAnalyzerOptions) Run(ctx context.Context) error {
 	return nil
 }
 
+func (o *JobRunHistoricalDataAnalyzerOptions) runTestsDataType(ctx context.Context, release string) error {
+	// Hardcoded parameters for test summary query
+	const (
+		suiteName    = "openshift-tests"
+		daysBack     = 30
+		minTestCount = 100
+	)
+
+	fmt.Printf("Fetching test data for release %s, suite %s, last %d days, min %d test runs\n",
+		release, suiteName, daysBack, minTestCount)
+
+	testSummaries, err := o.ciDataClient.ListTestSummaryByPeriod(ctx, suiteName, release, daysBack, minTestCount)
+	if err != nil {
+		return fmt.Errorf("failed to list test summary by period: %w", err)
+	}
+
+	if len(testSummaries) == 0 {
+		return fmt.Errorf("no test data found for suite %s, release %s", suiteName, release)
+	}
+
+	// Write the test summaries directly to the output file as JSON
+	out, err := formatTestOutput(testSummaries)
+	if err != nil {
+		return fmt.Errorf("error formatting test output: %w", err)
+	}
+
+	if err := os.WriteFile(o.outputFile, out, 0644); err != nil {
+		return fmt.Errorf("failed to write output file: %w", err)
+	}
+
+	fmt.Printf("Successfully fetched %d test results and wrote to %s\n", len(testSummaries), o.outputFile)
+	return nil
+}
+
 func (o *JobRunHistoricalDataAnalyzerOptions) getAlertData(ctx context.Context) ([]jobrunaggregatorapi.HistoricalData, error) {
 	var allKnownAlerts []*jobrunaggregatorapi.KnownAlertRow
 	var newHistoricalData []*jobrunaggregatorapi.AlertHistoricalDataRow

pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/cmd.go

@@ -26,7 +26,7 @@ type JobRunHistoricalDataAnalyzerFlags struct {
 	PreviousRelease string
 }
 
-var supportedDataTypes = sets.New[string]("alerts", "disruptions")
+var supportedDataTypes = sets.New[string]("alerts", "disruptions", "tests")
 
 func NewJobRunHistoricalDataAnalyzerFlags() *JobRunHistoricalDataAnalyzerFlags {
 	return &JobRunHistoricalDataAnalyzerFlags{
@@ -60,15 +60,16 @@ func (f *JobRunHistoricalDataAnalyzerFlags) Validate() error {
 		return fmt.Errorf("must provide supported datatype %v", sets.List(supportedDataTypes))
 	}
 
-	if f.CurrentFile == "" {
+	// For tests data type, we don't need --current since we don't do comparison
+	if f.DataType != "tests" && f.CurrentFile == "" {
 		return fmt.Errorf("must provide --current [file_path] flag to compare against")
 	}
 
 	if f.Leeway < 0 {
 		return fmt.Errorf("leeway percent must be above 0")
 	}
 
-	if f.TargetRelease != "" && f.PreviousRelease == "" {
+	if f.TargetRelease != "" && f.PreviousRelease == "" && f.DataType != "tests" {
 		return fmt.Errorf("must specify --previous-release with --target-release")
 	}
 

pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/util.go

@@ -166,3 +166,20 @@ func formatOutput(data []parsedJobData, format string) ([]byte, error) {
 		return nil, fmt.Errorf("invalid output format (%s)", format)
 	}
 }
+
+func formatTestOutput(data []jobrunaggregatorapi.TestSummaryByPeriodRow) ([]byte, error) {
+	if len(data) == 0 {
+		return nil, nil
+	}
+	// Sort by release, failure count desc, test name
+	sort.SliceStable(data, func(i, j int) bool {
+		if data[i].Release != data[j].Release {
+			return data[i].Release < data[j].Release
+		}
+		if data[i].TotalFailureCount != data[j].TotalFailureCount {
+			return data[i].TotalFailureCount > data[j].TotalFailureCount
+		}
+		return data[i].TestName < data[j].TestName
+	})
+	return json.MarshalIndent(data, "", "  ")
+}