perf: push filters into test report subqueries (~830x improvement)

pkg/api/tests.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/api/iterator"
+	"gorm.io/gorm"
 
 	apitype "github.com/openshift/sippy/pkg/apis/api"
 	bq "github.com/openshift/sippy/pkg/bigquery"
@@ -469,7 +470,29 @@ func BuildTestsResults(dbc *db.DB, release, period string, collapse, includeOver
 	if collapse {
 		rawQuery = rawQuery.Select(`suite_name,name,jira_component,jira_component_id,` + query.QueryTestSummer).Group("suite_name,name,jira_component,jira_component_id")
 	} else {
-		rawQuery = query.TestsByNURPAndStandardDeviation(dbc, release, table)
+		// Split the raw filter so name filters push into both stats and
+		// passRates subqueries (for index use), while variant filters only
+		// push into passRates to preserve cross-variant stats semantics.
+		var subqueryFilters []query.SubqueryFilter
+		if rawFilter != nil {
+			variantFilter, nameFilter := rawFilter.Split([]string{"variants"})
+			if len(nameFilter.Items) > 0 {
+				subqueryFilters = append(subqueryFilters, query.SubqueryFilter{
+					Apply: func(db *gorm.DB) *gorm.DB {
+						return nameFilter.ToSQL(db, apitype.Test{})
+					},
+				})
+			}
+			if len(variantFilter.Items) > 0 {
+				subqueryFilters = append(subqueryFilters, query.SubqueryFilter{
+					Apply: func(db *gorm.DB) *gorm.DB {
+						return variantFilter.ToSQL(db, apitype.Test{})
+					},
+					VariantOnly: true,
+				})
+			}
+		}
+		rawQuery = query.TestsByNURPAndStandardDeviation(dbc, release, table, subqueryFilters...)
 		variantSelect = "suite_name, variants," +
 			"delta_from_working_average, working_average, working_standard_deviation, " +
 			"delta_from_passing_average, passing_average, passing_standard_deviation, " +

pkg/db/query/test_queries.go

@@ -244,7 +244,20 @@ func LoadBugsForTest(dbc *db.DB, testName string, filterClosed bool) ([]models.B
 // flake_average shows the average flake percentage among all variants.
 // flake_standard_deviation shows the standard deviation of the flake percentage among variants. The number reflects how much flake percentage differs among variants.
 // delta_from_flake_average shows how much each variant differs from the flake_average. This can be used to identify outliers.
-func TestsByNURPAndStandardDeviation(dbc *db.DB, release, table string) *gorm.DB {
+// SubqueryFilter wraps a filter function with metadata about what it targets.
+// Variant-only filters are applied only to passRates, not to stats, because
+// the stats subquery computes cross-variant averages and standard deviations
+// that should not be skewed by variant exclusions.
+type SubqueryFilter struct {
+	Apply       func(*gorm.DB) *gorm.DB
+	VariantOnly bool
+}
+
+func isVariantFilter(f SubqueryFilter) bool {
+	return f.VariantOnly
+}
+
+func TestsByNURPAndStandardDeviation(dbc *db.DB, release, table string, subqueryFilters ...SubqueryFilter) *gorm.DB {
 	// 1. Create a virtual stats table. There is a single row for each test.
 	stats := dbc.DB.Table(table).
 		Select(`
@@ -264,6 +277,18 @@ func TestsByNURPAndStandardDeviation(dbc *db.DB, release, table string) *gorm.DB
 		Select(`id as test_id, suite_name as pass_rate_suite_name, variants as pass_rate_variants, `+QueryTestPercentages).
 		Where(`release = ?`, release)
 
+	// Apply filters to the subqueries so they can leverage indexes instead of
+	// scanning the entire matview for the release. Variant-specific filters
+	// only apply to passRates to preserve cross-variant stats semantics.
+	for _, f := range subqueryFilters {
+		if isVariantFilter(f) {
+			passRates = f.Apply(passRates)
+		} else {
+			stats = f.Apply(stats)
+			passRates = f.Apply(passRates)
+		}
+	}
+
 	// 3. Join the tables to produce test report. Each row represent one variant of a test and contains all stats, both unique to the specific variant and average across all variants.
 	return dbc.DB.
 		Table(table).