diff --git a/config/views.yaml b/config/views.yaml index 687035f5d..b229ae646 100644 --- a/config/views.yaml +++ b/config/views.yaml @@ -87,6 +87,97 @@ component_readiness: enabled: true automate_jira: enabled: true +- name: 4.22-main-mass-failure + base_release: + release: "4.21" + relative_start: ga-30d + relative_end: ga + sample_release: + release: "4.22" + relative_start: now-7d + relative_end: now + variant_options: + column_group_by: + Network: { } + Platform: { } + Topology: { } + db_group_by: + Architecture: { } + FeatureSet: { } + Installer: { } + Network: { } + Platform: { } + Suite: { } + Topology: { } + Upgrade: { } + LayeredProduct: { } + include_variants: + Architecture: + - amd64 + - arm64 + - multi + FeatureSet: + - default + - techpreview + Installer: + - ipi + - upi + - hypershift + JobTier: + - blocking + - informing + - standard + LayeredProduct: + - none + - virt + Network: + - ovn + OS: + - rhcos9 + Owner: + - eng + - service-delivery + Platform: + - aws + - azure + - gcp + - metal + - rosa + - vsphere + Topology: + - ha + - microshift + - external + CGroupMode: + - v2 + ContainerRuntime: + - runc + - crun + Upgrade: + - micro + - minor + - none + advanced_options: + minimum_failure: 3 + confidence: 95 + pity_factor: 5 + ignore_missing: false + ignore_disruption: true + flake_as_failure: false + pass_rate_required_new_tests: 95 + include_multi_release_analysis: true + key_test_names: + - "install should succeed: overall" + - "[sig-cluster-lifecycle] Cluster completes upgrade" + - "[Jira:\"Test Framework\"] there should not be mass test failures" + metrics: + enabled: true + regression_tracking: + enabled: true + prime_cache: + enabled: true + automate_jira: + enabled: false - name: 4.22-hypershift-candidates base_release: release: "4.21" diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go index 8dfcfad5b..253d20c77 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go @@ -358,6 +358,8 @@ type fallbackTestQueryReleasesGeneratorCacheKey struct { VariantDBGroupBy sets.String // CRTimeRoundingFactor is used by GetReleaseDatesFromBigQuery CRTimeRoundingFactor time.Duration + // KeyTestNames affects the BuildComponentReportQuery results via filtering logic + KeyTestNames []string } // getCacheKey creates a cache key using the generator properties that we want included for uniqueness in what @@ -370,6 +372,7 @@ func (f *fallbackTestQueryReleasesGenerator) getCacheKey() fallbackTestQueryRele BaseEnd: f.BaseEnd, VariantDBGroupBy: f.ReqOptions.VariantOption.DBGroupBy, CRTimeRoundingFactor: f.ReqOptions.CacheOption.CRTimeRoundingFactor, + KeyTestNames: f.ReqOptions.AdvancedOption.KeyTestNames, } } @@ -516,8 +519,9 @@ type fallbackTestQueryGeneratorCacheKey struct { BaseRelease string BaseStart time.Time BaseEnd time.Time - // IgnoreDisruption is the only field within AdvancedOption that is used here + // IgnoreDisruption and KeyTestNames are fields within AdvancedOption that affect the query IgnoreDisruption bool + KeyTestNames []string IncludeVariants map[string][]string VariantDBGroupBy sets.String // if we ever needed fallback on cross-compare views we should include fields for that, @@ -533,6 +537,7 @@ func (f *fallbackTestQueryGenerator) getCacheKey() fallbackTestQueryGeneratorCac BaseStart: f.BaseStart, BaseEnd: f.BaseEnd, IgnoreDisruption: f.ReqOptions.AdvancedOption.IgnoreDisruption, + KeyTestNames: f.ReqOptions.AdvancedOption.KeyTestNames, IncludeVariants: f.ReqOptions.VariantOption.IncludeVariants, VariantDBGroupBy: f.ReqOptions.VariantOption.DBGroupBy, } @@ -541,7 +546,7 @@ func (f *fallbackTestQueryGenerator) getCacheKey() fallbackTestQueryGeneratorCac func (f *fallbackTestQueryGenerator) getTestFallbackRelease(ctx context.Context) (bq.ReportTestStatus, []error) { commonQuery, groupByQuery, queryParameters := query.BuildComponentReportQuery( f.client, f.ReqOptions, f.allVariants, f.ReqOptions.VariantOption.IncludeVariants, - query.DefaultJunitTable, false) + query.DefaultJunitTable, false, f.ReqOptions.AdvancedOption.KeyTestNames...) before := time.Now() log.Infof("Starting Fallback (%s) QueryTestStatus", f.BaseRelease) errs := []error{} diff --git a/pkg/api/componentreadiness/query/querygenerators.go b/pkg/api/componentreadiness/query/querygenerators.go index 6e65aaa6e..bd675d640 100644 --- a/pkg/api/componentreadiness/query/querygenerators.go +++ b/pkg/api/componentreadiness/query/querygenerators.go @@ -135,9 +135,8 @@ func NewBaseQueryGenerator( func (b *baseQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTestStatus, []error) { - commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(b.client, b.ReqOptions, b.allVariants, b.ReqOptions.VariantOption.IncludeVariants, DefaultJunitTable, false) + commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(b.client, b.ReqOptions, b.allVariants, b.ReqOptions.VariantOption.IncludeVariants, DefaultJunitTable, false, b.ReqOptions.AdvancedOption.KeyTestNames...) - before := time.Now() errs := []error{} baseString := commonQuery + ` AND jv_Release.variant_value = @BaseRelease` baseQuery := b.client.Query(ctx, bqlabel.CRJunitBase, baseString+groupByQuery) @@ -164,8 +163,6 @@ func (b *baseQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTest errs = append(errs, baseErrs...) } - log.Infof("Base QueryTestStatus completed in %s with %d base results from db", time.Since(before), len(baseStatus)) - return bq.ReportTestStatus{BaseStatus: baseStatus}, errs } @@ -207,9 +204,8 @@ func NewSampleQueryGenerator( } func (s *sampleQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTestStatus, []error) { - commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(s.client, s.ReqOptions, s.allVariants, s.IncludeVariants, s.JunitTable, true) + commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(s.client, s.ReqOptions, s.allVariants, s.IncludeVariants, s.JunitTable, true, s.ReqOptions.AdvancedOption.KeyTestNames...) - before := time.Now() errs := []error{} sampleString := commonQuery // Only set sample release when PR and payload options are not set @@ -273,12 +269,28 @@ func (s *sampleQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTe errs = append(errs, sampleErrs...) } - log.Infof("Sample QueryTestStatus completed in %s with %d sample results db", time.Since(before), len(sampleStatus)) - return bq.ReportTestStatus{SampleStatus: sampleStatus}, errs } +// buildPriorityCaseStatement generates a SQL CASE statement that assigns priority based on test position in the list. +// Lower index = higher priority. This is used to ensure when multiple key tests appear in the same job, +// only the highest priority one is counted. +func buildPriorityCaseStatement(keyTestNames []string) string { + var caseStatements []string + for i, testName := range keyTestNames { + // Escape single quotes in test name for SQL + escapedTestName := strings.ReplaceAll(testName, "'", "''") + caseStatements = append(caseStatements, fmt.Sprintf("WHEN test_name = '%s' THEN %d", escapedTestName, i)) + } + // Add a default case to handle any unexpected tests (should not happen due to IN UNNEST filter) + caseStatements = append(caseStatements, fmt.Sprintf("ELSE %d", len(keyTestNames))) + return strings.Join(caseStatements, "\n\t\t\t\t\t\t\t") +} + // BuildComponentReportQuery returns the common query for the higher level summary component summary. +// If keyTestNames is provided, when any of these tests fail in a job, all other test failures +// in that job are excluded from regression analysis. Only the highest priority (earliest in the list) +// key test will be included for each affected job. func BuildComponentReportQuery( client *bqcachedclient.Client, reqOptions reqopts.RequestOptions, @@ -286,6 +298,7 @@ func BuildComponentReportQuery( includeVariants map[string][]string, junitTable string, isSample bool, + keyTestNames ...string, ) (string, string, []bigquery.QueryParameter) { // Parts of the query, including the columns returned, are dynamic, based on the list of variants we're told to work with. // Variants will be returned as columns with names like: variant_[VariantName] @@ -294,7 +307,7 @@ func BuildComponentReportQuery( joinVariants := "" groupByVariants := "" for _, v := range sortedKeys(allJobVariants.Variants) { - joinVariants += fmt.Sprintf("LEFT JOIN %s.job_variants jv_%s ON variant_registry_job_name = jv_%s.job_name AND jv_%s.variant_name = '%s'\n", + joinVariants += fmt.Sprintf("LEFT JOIN %s.job_variants jv_%s ON junit_data.variant_registry_job_name = jv_%s.job_name AND jv_%s.variant_name = '%s'\n", client.Dataset, v, v, v, v) } for _, v := range reqOptions.VariantOption.DBGroupBy.List() { @@ -313,33 +326,95 @@ func BuildComponentReportQuery( // TODO: last_failure here explicitly uses success_val not adjusted_success_val, this ensures we // show the last time the test failed, not flaked. if you enable the flakes as failures feature (which is // non default today), the last failure time will be wrong which can impact things like failed fix detection. - queryString := fmt.Sprintf(`WITH latest_component_mapping AS ( - SELECT * - FROM %s.component_mapping cm - WHERE created_at = ( - SELECT MAX(created_at) - FROM %s.component_mapping)) + // Build the WITH clause - add jobs_with_failed_key_tests CTE if keyTestNames is provided + withClause := "" + if len(keyTestNames) > 0 { + // Create a CTE that identifies the highest priority (lowest index) key test in each job + // This ensures when multiple key tests appear in the same job, only the highest priority one is used + withClause = fmt.Sprintf(`WITH key_test_priorities AS ( + SELECT + prowjob_build_id, + test_name, + -- Find the index/priority of each test (lower index = higher priority) + CASE + %s + END AS test_priority + FROM %s.%s AS junit + WHERE modified_time >= DATETIME(@From) + AND modified_time < DATETIME(@To) + AND test_name IN UNNEST(@KeyTestNames) + AND success_val = 0 + ), + jobs_with_highest_priority_test AS ( + SELECT + prowjob_build_id, + test_name + FROM key_test_priorities + WHERE test_priority = ( + SELECT MIN(test_priority) + FROM key_test_priorities ep2 + WHERE ep2.prowjob_build_id = key_test_priorities.prowjob_build_id + ) + ), + latest_component_mapping AS ( + SELECT * + FROM %s.component_mapping cm + WHERE created_at = ( + SELECT MAX(created_at) + FROM %s.component_mapping))`, + buildPriorityCaseStatement(keyTestNames), client.Dataset, junitTable, client.Dataset, client.Dataset) + } else { + withClause = fmt.Sprintf(`WITH latest_component_mapping AS ( + SELECT * + FROM %s.component_mapping cm + WHERE created_at = ( + SELECT MAX(created_at) + FROM %s.component_mapping))`, + client.Dataset, client.Dataset) + } + + queryString := fmt.Sprintf(`%s SELECT - ANY_VALUE(test_name HAVING MAX prowjob_start) AS test_name, - ANY_VALUE(testsuite HAVING MAX prowjob_start) AS test_suite, + ANY_VALUE(junit_data.test_name HAVING MAX junit_data.prowjob_start) AS test_name, + ANY_VALUE(junit_data.testsuite HAVING MAX junit_data.prowjob_start) AS test_suite, cm.id as test_id, %s COUNT(cm.id) AS total_count, - SUM(adjusted_success_val) AS success_count, - SUM(adjusted_flake_count) AS flake_count, - MAX(CASE WHEN success_val = 0 THEN prowjob_start ELSE NULL END) AS last_failure, + SUM(junit_data.adjusted_success_val) AS success_count, + SUM(junit_data.adjusted_flake_count) AS flake_count, + MAX(CASE WHEN junit_data.success_val = 0 THEN junit_data.prowjob_start ELSE NULL END) AS last_failure, ANY_VALUE(cm.component) AS component, ANY_VALUE(cm.capabilities) AS capabilities, - FROM (%s) - INNER JOIN latest_component_mapping cm ON testsuite = cm.suite AND test_name = cm.name + FROM (%s) AS junit_data + INNER JOIN latest_component_mapping cm ON junit_data.testsuite = cm.suite AND junit_data.test_name = cm.name `, - client.Dataset, client.Dataset, selectVariants, fmt.Sprintf(dedupedJunitTable, jobNameQueryPortion, client.Dataset, junitTable, client.Dataset, client.Dataset, jobRunAnnotationToIgnore)) + withClause, selectVariants, fmt.Sprintf(dedupedJunitTable, jobNameQueryPortion, client.Dataset, junitTable, client.Dataset, client.Dataset, jobRunAnnotationToIgnore)) queryString += joinVariants queryString += `WHERE cm.staff_approved_obsolete = false AND - (variant_registry_job_name LIKE 'periodic-%%' OR variant_registry_job_name LIKE 'release-%%' OR variant_registry_job_name LIKE 'aggregator-%%')` + (junit_data.variant_registry_job_name LIKE 'periodic-%%' OR junit_data.variant_registry_job_name LIKE 'release-%%' OR junit_data.variant_registry_job_name LIKE 'aggregator-%%')` commonParams := []bigquery.QueryParameter{} + + // Add filtering logic for key tests with priority + // Only include the highest priority test from each job, and exclude all other tests from those jobs + if len(keyTestNames) > 0 { + queryString += ` + AND ( + -- Include tests from jobs that don't have any failed key tests + junit_data.prowjob_build_id NOT IN (SELECT prowjob_build_id FROM jobs_with_highest_priority_test) + -- Or include only the highest priority key test from jobs that have them + OR EXISTS ( + SELECT 1 FROM jobs_with_highest_priority_test j + WHERE j.prowjob_build_id = junit_data.prowjob_build_id + AND j.test_name = junit_data.test_name + ) + )` + commonParams = append(commonParams, bigquery.QueryParameter{ + Name: "KeyTestNames", + Value: keyTestNames, + }) + } if reqOptions.AdvancedOption.IgnoreDisruption { queryString += ` AND NOT 'Disruption' in UNNEST(capabilities)` } @@ -627,6 +702,7 @@ func FetchTestStatusResults(ctx context.Context, query *bigquery.Query) (map[str status[testIDStr] = testStatus } + return status, errs } diff --git a/pkg/api/componentreadiness/query/querygenerators_test.go b/pkg/api/componentreadiness/query/querygenerators_test.go new file mode 100644 index 000000000..37fa56410 --- /dev/null +++ b/pkg/api/componentreadiness/query/querygenerators_test.go @@ -0,0 +1,310 @@ +package query + +import ( + "strings" + "testing" + + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/util/sets" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildComponentReportQuery_ExclusiveTestFiltering(t *testing.T) { + mockClient := &bqcachedclient.Client{ + Dataset: "test_dataset", + } + + allJobVariants := crtest.JobVariants{ + Variants: map[string][]string{ + "Platform": {"aws", "gcp"}, + "Network": {"sdn", "ovn"}, + }, + } + + baseReqOptions := reqopts.RequestOptions{ + VariantOption: reqopts.Variants{ + ColumnGroupBy: sets.NewString("Platform"), + DBGroupBy: sets.NewString("Network"), + IncludeVariants: map[string][]string{}, + }, + AdvancedOption: reqopts.Advanced{ + IgnoreDisruption: true, + }, + } + + includeVariants := map[string][]string{} + + tests := []struct { + name string + keyTestNames []string + expectedCTE bool + expectedFilter bool + expectedParam bool + expectedCTEContent string + }{ + { + name: "No key tests - no filtering", + keyTestNames: nil, + expectedCTE: false, + expectedFilter: false, + expectedParam: false, + }, + { + name: "Empty key tests - no filtering", + keyTestNames: []string{}, + expectedCTE: false, + expectedFilter: false, + expectedParam: false, + }, + { + name: "With key tests - filtering applied", + keyTestNames: []string{ + "[sig-cluster-lifecycle] Cluster completes upgrade", + "install should succeed: overall", + }, + expectedCTE: true, + expectedFilter: true, + expectedParam: true, + expectedCTEContent: "jobs_with_highest_priority_test", + }, + { + name: "Single key test - filtering applied", + keyTestNames: []string{ + "install should succeed: overall", + }, + expectedCTE: true, + expectedFilter: true, + expectedParam: true, + expectedCTEContent: "jobs_with_highest_priority_test", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reqOptions := baseReqOptions + reqOptions.AdvancedOption.KeyTestNames = tt.keyTestNames + + commonQuery, _, queryParams := BuildComponentReportQuery( + mockClient, + reqOptions, + allJobVariants, + includeVariants, + DefaultJunitTable, + false, + tt.keyTestNames..., + ) + + // Check if CTE is present when expected + if tt.expectedCTE { + assert.Contains(t, commonQuery, "jobs_with_highest_priority_test", + "Query should contain jobs_with_highest_priority_test CTE") + assert.Contains(t, commonQuery, "key_test_priorities", + "Query should contain exclusive_test_priorities CTE for priority calculation") + assert.Contains(t, commonQuery, "AND success_val = 0", + "CTE should only identify jobs where key tests FAILED (success_val = 0)") + assert.Contains(t, commonQuery, "test_name IN UNNEST(@KeyTestNames)", + "CTE should filter by key test names") + assert.Contains(t, commonQuery, "test_priority", + "CTE should calculate test priority based on list order") + } else { + assert.NotContains(t, commonQuery, "jobs_with_highest_priority_test", + "Query should not contain jobs_with_highest_priority_test CTE when no key tests") + assert.NotContains(t, commonQuery, "key_test_priorities", + "Query should not contain exclusive_test_priorities CTE when no key tests") + } + + // Check if filtering WHERE clause is present when expected + if tt.expectedFilter { + assert.Contains(t, commonQuery, "junit_data.prowjob_build_id NOT IN (SELECT prowjob_build_id FROM jobs_with_highest_priority_test)", + "Query should include tests from jobs without failed key tests") + assert.Contains(t, commonQuery, "EXISTS", + "Query should use EXISTS to match highest priority test from jobs with key tests") + assert.Contains(t, commonQuery, "j.prowjob_build_id = junit_data.prowjob_build_id", + "Query should match both prowjob_build_id and test_name in EXISTS clause") + } + + // Check if query parameter is present when expected + if tt.expectedParam { + foundParam := false + for _, param := range queryParams { + if param.Name == "KeyTestNames" { + foundParam = true + assert.Equal(t, tt.keyTestNames, param.Value, + "KeyTestNames parameter should match input") + break + } + } + assert.True(t, foundParam, "KeyTestNames parameter should be present in query parameters") + } else { + for _, param := range queryParams { + assert.NotEqual(t, "KeyTestNames", param.Name, + "KeyTestNames parameter should not be present when no key tests") + } + } + + // Verify CTE content structure if CTE is expected + if tt.expectedCTEContent != "" { + // Extract the CTE section + parts := strings.Split(commonQuery, "latest_component_mapping") + require.Greater(t, len(parts), 1, "Query should contain latest_component_mapping CTE") + + cteSection := parts[0] + assert.Contains(t, cteSection, tt.expectedCTEContent, + "Query should contain expected CTE") + + // Verify the CTE selects prowjob_build_id and test_name for priority-based filtering + // The new structure identifies the highest priority test per job + normalizedCTE := strings.ReplaceAll(strings.ReplaceAll(cteSection, "\t", " "), "\n", " ") + assert.Contains(t, normalizedCTE, "prowjob_build_id", + "CTE should select prowjob_build_id") + assert.Contains(t, normalizedCTE, "test_name", + "CTE should select test_name to identify the highest priority test") + } + }) + } +} + +func TestBuildComponentReportQuery_ExclusiveTestLogic(t *testing.T) { + // This test verifies the specific logic: we only exclude OTHER tests from jobs + // where key tests FAILED (not just present) + mockClient := &bqcachedclient.Client{ + Dataset: "test_dataset", + } + + allJobVariants := crtest.JobVariants{ + Variants: map[string][]string{ + "Platform": {"aws"}, + }, + } + + reqOptions := reqopts.RequestOptions{ + VariantOption: reqopts.Variants{ + ColumnGroupBy: sets.NewString("Platform"), + DBGroupBy: sets.NewString(), + IncludeVariants: map[string][]string{}, + }, + AdvancedOption: reqopts.Advanced{ + KeyTestNames: []string{"install should succeed: overall"}, + }, + } + + commonQuery, _, _ := BuildComponentReportQuery( + mockClient, + reqOptions, + allJobVariants, + map[string][]string{}, + DefaultJunitTable, + false, + "install should succeed: overall", + ) + + // The query should: + // 1. Create CTEs that identify the highest priority test in each job + assert.Contains(t, commonQuery, "WITH key_test_priorities AS", + "Should create CTE for calculating test priorities") + assert.Contains(t, commonQuery, "jobs_with_highest_priority_test AS", + "Should create CTE for identifying highest priority test per job") + + // 2. The CTE should check success_val = 0 (failure) + cteEnd := strings.Index(commonQuery, "latest_component_mapping") + require.Greater(t, cteEnd, 0, "Should contain latest_component_mapping CTE") + + cteSection := commonQuery[:cteEnd] + assert.Contains(t, cteSection, "success_val = 0", + "CTE should only match FAILED key tests (success_val = 0), not all instances") + + // 3. Priority-based filtering: only include highest priority test from jobs with key test failures + assert.Contains(t, commonQuery, "test_priority", + "Should calculate test priority based on list order") + assert.Contains(t, commonQuery, "junit_data.prowjob_build_id NOT IN (SELECT prowjob_build_id FROM jobs_with_highest_priority_test)", + "Should include tests from jobs without failed key tests") + assert.Contains(t, commonQuery, "EXISTS", + "Should use EXISTS to match the highest priority test from jobs with key test failures") + assert.Contains(t, commonQuery, "j.prowjob_build_id = junit_data.prowjob_build_id", + "Should match prowjob_build_id in EXISTS clause") + assert.Contains(t, commonQuery, "j.test_name = junit_data.test_name", + "Should match test_name in EXISTS clause") + + // 4. Verify the logic correctly filters based on priority + // Normalize whitespace for easier parsing + normalizedQuery := strings.ReplaceAll(strings.ReplaceAll(commonQuery, "\t", " "), "\n", " ") + normalizedQuery = strings.Join(strings.Fields(normalizedQuery), " ") // Collapse all whitespace + + // The query should contain the priority-based filtering logic using EXISTS + assert.Contains(t, normalizedQuery, "jobs_with_highest_priority_test j", + "Should use jobs_with_highest_priority_test in EXISTS subquery") +} + +func TestBuildComponentReportQuery_WithAndWithoutExclusiveTests(t *testing.T) { + // This test compares queries with and without key tests to ensure + // the base query structure is the same, only the filtering differs + mockClient := &bqcachedclient.Client{ + Dataset: "test_dataset", + } + + allJobVariants := crtest.JobVariants{ + Variants: map[string][]string{ + "Platform": {"aws"}, + }, + } + + baseReqOptions := reqopts.RequestOptions{ + VariantOption: reqopts.Variants{ + ColumnGroupBy: sets.NewString("Platform"), + DBGroupBy: sets.NewString(), + IncludeVariants: map[string][]string{}, + }, + AdvancedOption: reqopts.Advanced{}, + } + + // Query without key tests + queryWithout, _, paramsWithout := BuildComponentReportQuery( + mockClient, + baseReqOptions, + allJobVariants, + map[string][]string{}, + DefaultJunitTable, + false, + ) + + // Query with key tests + reqOptionsWithExclusive := baseReqOptions + reqOptionsWithExclusive.AdvancedOption.KeyTestNames = []string{"install should succeed: overall"} + + queryWith, _, paramsWith := BuildComponentReportQuery( + mockClient, + reqOptionsWithExclusive, + allJobVariants, + map[string][]string{}, + DefaultJunitTable, + false, + "install should succeed: overall", + ) + + // Both should have the component_mapping CTE + assert.Contains(t, queryWithout, "latest_component_mapping", + "Query without key tests should have component mapping CTE") + assert.Contains(t, queryWith, "latest_component_mapping", + "Query with key tests should have component mapping CTE") + + // Only the query with key tests should have the filtering CTEs + assert.NotContains(t, queryWithout, "jobs_with_highest_priority_test", + "Query without key tests should not have filtering CTE") + assert.NotContains(t, queryWithout, "key_test_priorities", + "Query without key tests should not have priority calculation CTE") + assert.Contains(t, queryWith, "jobs_with_highest_priority_test", + "Query with key tests should have filtering CTE") + assert.Contains(t, queryWith, "key_test_priorities", + "Query with key tests should have priority calculation CTE") + + // Check parameters + assert.Len(t, paramsWithout, 0, "Query without key tests should have no extra parameters") + assert.Len(t, paramsWith, 1, "Query with key tests should have 1 parameter") + if len(paramsWith) > 0 { + assert.Equal(t, "KeyTestNames", paramsWith[0].Name, + "Parameter should be named KeyTestNames") + } +} diff --git a/pkg/apis/api/componentreport/reqopts/types.go b/pkg/apis/api/componentreport/reqopts/types.go index 04567d180..90247f6c0 100644 --- a/pkg/apis/api/componentreport/reqopts/types.go +++ b/pkg/apis/api/componentreport/reqopts/types.go @@ -107,4 +107,9 @@ type Advanced struct { IgnoreDisruption bool `json:"ignore_disruption" yaml:"ignore_disruption"` FlakeAsFailure bool `json:"flake_as_failure" yaml:"flake_as_failure"` IncludeMultiReleaseAnalysis bool `json:"include_multi_release_analysis" yaml:"include_multi_release_analysis"` + // KeyTestNames contains test names that, when they fail in a job, cause all other test failures + // in that job to be excluded from regression analysis. This is used to filter out mass failures + // caused by fundamental infrastructure issues (e.g., install failures, upgrade failures). + // When multiple key tests fail in the same job, only the highest priority (earliest in list) test is included. + KeyTestNames []string `json:"key_test_names,omitempty" yaml:"key_test_names,omitempty"` } diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 93bde9cb1..2e4b6dbb3 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -993,6 +993,7 @@ func (s *Server) getComponentReportFromRequest(req *http.Request) (componentrepo options, warnings, err := utils.ParseComponentReportRequest(s.views.ComponentReadiness, allReleases, req, allJobVariants, s.crTimeRoundingFactor, s.config.ComponentReadinessConfig.VariantJunitTableOverrides) + if err != nil { return componentreport.ComponentReport{}, err } @@ -1048,6 +1049,7 @@ func (s *Server) jsonComponentReportTestDetailsFromBigQuery(w http.ResponseWrite reqOptions, _, err := utils.ParseComponentReportRequest(s.views.ComponentReadiness, allReleases, req, allJobVariants, s.crTimeRoundingFactor, s.config.ComponentReadinessConfig.VariantJunitTableOverrides) + if err != nil { failureResponse(w, http.StatusBadRequest, err.Error()) return