Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ tests:
TEST_SUITE: ocp-compliance-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- always_run: true
- always_run: false
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need 4.18 on master at all?
Should we just delete it as well as 4.19 after making 4.20 the default for tests?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keeping it (as only run on-demand) allows us to run CI tests against a supported version of OCP from a PR.
I think it would be useful to keep the range of OCP versions available to test while we're supporting running on them. What do you think? I think the downsides are A: the extra files in this directory (too many / confusing for the wider team to approach the config?) and B: the long /test ? list (but that is already un-usably long with only the oldest+latest versions).

Copy link
Contributor

@msugakov msugakov Nov 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, /test ? goes out of hand and that's the main reason for me. Whenever I ask Konflux to /retest <something>, OSCI replies with longer and longer list.

Also, we don't cover all versions so this looks inconsistent and as an engineer not involved in OSCI I would doubt before attempting anything on 4.18.

Finally, these files add mental overhead which means more expensive maintenance.

If I have a case to test my change on older OCP version, I'd spin up an Infra cluster and test my change there, without e2e tests. In addition, will trust e2e tests on the "newest" and the "oldest" versions to do their part and provide safety from regressions.

Therefore, if there's no active/persistent situation to use them, I'd vote to delete instead of having them "in case of need" because the need may never materialize.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as an engineer not involved in OSCI I would doubt before attempting anything on 4.18.

Finally, these files add mental overhead which means more expensive maintenance.

I think these are reason enough to remove the extra version tests.

will trust e2e tests on the "newest" and the "oldest" versions to do their part and provide safety from regressions.

👍

as: nongroovy-e2e-tests
optional: true
steps:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ resources:
memory: 4000Mi
test_binary_build_commands: .openshift-ci/dispatch.sh test-binary-build-commands
tests:
- as: qa-e2e-tests
- always_run: false
as: qa-e2e-tests
optional: true
run_if_changed: ^((generated|image|operator|pkg|make|deploy|scripts)/.*|Makefile|status\.sh|[A-Z_]+_VERSION)
steps:
env:
COLLECTION_METHOD: core_bpf
Expand All @@ -36,7 +36,6 @@ tests:
- always_run: false
as: scanner-v4-install-tests
optional: true
run_if_changed: ^(scanner|image|central/localscanner|sensor/kubernetes/localscanner|operator|roxctl|deploy|proto|scripts/ci|tests/e2e)/.*
steps:
env:
COLLECTION_METHOD: core_bpf
Expand All @@ -54,9 +53,9 @@ tests:
TEST_SUITE: ocp-qa-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-crun-e2e
timeout: 5h0m0s
- as: operator-e2e-tests
- always_run: false
as: operator-e2e-tests
optional: true
run_if_changed: ^((generated|image|operator|pkg|make|deploy|scripts)/.*|Makefile|status\.sh|[A-Z_]+_VERSION)
steps:
env:
OCP_VERSION: ocp/fast-4.19
Expand All @@ -73,9 +72,9 @@ tests:
TEST_SUITE: ocp-sensor-integration-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: ui-e2e-tests
- always_run: false
as: ui-e2e-tests
optional: true
run_if_changed: ^(ui|proto/(storage|api)|central/.+?/service|pkg/features/list.go|tests/e2e/lib.sh)
steps:
env:
COLLECTION_METHOD: core_bpf
Expand All @@ -94,17 +93,17 @@ tests:
TEST_SUITE: ocp-qa-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: compliance-e2e-tests
- always_run: false
as: compliance-e2e-tests
optional: true
run_if_changed: ^(compliance|central\/compliance.*)
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/fast-4.19
TEST_SUITE: ocp-compliance-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- always_run: true
- always_run: false
as: nongroovy-e2e-tests
optional: true
steps:
Expand All @@ -114,59 +113,6 @@ tests:
TEST_SUITE: ocp-nongroovy-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-qa-e2e-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/fast-4.19
TEST_SUITE: ocp-qa-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-operator-e2e-tests
postsubmit: true
steps:
env:
OCP_VERSION: ocp/fast-4.19
TEST_SUITE: ocp-operator-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-scanner-v4-install-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/fast-4.19
TEST_SUITE: ocp-scanner-v4-install-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-ui-e2e-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/fast-4.19
TEST_SUITE: ocp-ui-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-nongroovy-e2e-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/fast-4.19
TEST_SUITE: ocp-nongroovy-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-compliance-e2e-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/fast-4.19
TEST_SUITE: ocp-compliance-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
zz_generated_metadata:
branch: master
org: stackrox
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ resources:
memory: 4000Mi
test_binary_build_commands: .openshift-ci/dispatch.sh test-binary-build-commands
tests:
- always_run: false
as: qa-e2e-tests
- as: qa-e2e-tests
optional: true
run_if_changed: ^((generated|image|operator|pkg|make|deploy|scripts)/.*|Makefile|status\.sh|[A-Z_]+_VERSION)
steps:
env:
COLLECTION_METHOD: core_bpf
Expand All @@ -36,6 +36,7 @@ tests:
- always_run: false
as: scanner-v4-install-tests
optional: true
run_if_changed: ^(scanner|image|central/localscanner|sensor/kubernetes/localscanner|operator|roxctl|deploy|proto|scripts/ci|tests/e2e)/.*
steps:
env:
COLLECTION_METHOD: core_bpf
Expand All @@ -53,9 +54,9 @@ tests:
TEST_SUITE: ocp-qa-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-crun-e2e
timeout: 5h0m0s
- always_run: false
as: operator-e2e-tests
- as: operator-e2e-tests
optional: true
run_if_changed: ^((generated|image|operator|pkg|make|deploy|scripts)/.*|Makefile|status\.sh|[A-Z_]+_VERSION)
steps:
env:
OCP_VERSION: ocp/candidate-4.20
Expand All @@ -72,9 +73,9 @@ tests:
TEST_SUITE: ocp-sensor-integration-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- always_run: false
as: ui-e2e-tests
- as: ui-e2e-tests
optional: true
run_if_changed: ^(ui|proto/(storage|api)|central/.+?/service|pkg/features/list.go|tests/e2e/lib.sh)
steps:
env:
COLLECTION_METHOD: core_bpf
Expand All @@ -93,17 +94,17 @@ tests:
TEST_SUITE: ocp-qa-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- always_run: false
as: compliance-e2e-tests
- as: compliance-e2e-tests
optional: true
run_if_changed: ^(compliance|central\/compliance.*)
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/candidate-4.20
TEST_SUITE: ocp-compliance-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- always_run: false
- always_run: true
as: nongroovy-e2e-tests
optional: true
steps:
Expand All @@ -113,6 +114,59 @@ tests:
TEST_SUITE: ocp-nongroovy-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-qa-e2e-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/candidate-4.20
TEST_SUITE: ocp-qa-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-operator-e2e-tests
postsubmit: true
steps:
env:
OCP_VERSION: ocp/candidate-4.20
TEST_SUITE: ocp-operator-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-scanner-v4-install-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/candidate-4.20
TEST_SUITE: ocp-scanner-v4-install-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-ui-e2e-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/candidate-4.20
TEST_SUITE: ocp-ui-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-nongroovy-e2e-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/candidate-4.20
TEST_SUITE: ocp-nongroovy-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
- as: merge-compliance-e2e-tests
postsubmit: true
steps:
env:
COLLECTION_METHOD: core_bpf
OCP_VERSION: ocp/candidate-4.20
TEST_SUITE: ocp-compliance-e2e-tests
workflow: stackrox-automation-flavors-ocp-4-e2e
timeout: 5h0m0s
zz_generated_metadata:
branch: master
org: stackrox
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -860,18 +860,18 @@ postsubmits:
skip_cloning: true
timeout: 5h0m0s
labels:
ci-operator.openshift.io/variant: ocp-4-19
ci-operator.openshift.io/variant: ocp-4-20
ci.openshift.io/generator: prowgen
max_concurrency: 6
name: branch-ci-stackrox-stackrox-master-ocp-4-19-merge-compliance-e2e-tests
name: branch-ci-stackrox-stackrox-master-ocp-4-20-merge-compliance-e2e-tests
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --report-credentials-file=/etc/report/credentials
- --target=merge-compliance-e2e-tests
- --variant=ocp-4-19
- --variant=ocp-4-20
command:
- ci-operator
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
Expand Down Expand Up @@ -914,18 +914,18 @@ postsubmits:
skip_cloning: true
timeout: 5h0m0s
labels:
ci-operator.openshift.io/variant: ocp-4-19
ci-operator.openshift.io/variant: ocp-4-20
ci.openshift.io/generator: prowgen
max_concurrency: 6
name: branch-ci-stackrox-stackrox-master-ocp-4-19-merge-nongroovy-e2e-tests
name: branch-ci-stackrox-stackrox-master-ocp-4-20-merge-nongroovy-e2e-tests
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --report-credentials-file=/etc/report/credentials
- --target=merge-nongroovy-e2e-tests
- --variant=ocp-4-19
- --variant=ocp-4-20
command:
- ci-operator
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
Expand Down Expand Up @@ -968,18 +968,18 @@ postsubmits:
skip_cloning: true
timeout: 5h0m0s
labels:
ci-operator.openshift.io/variant: ocp-4-19
ci-operator.openshift.io/variant: ocp-4-20
ci.openshift.io/generator: prowgen
max_concurrency: 6
name: branch-ci-stackrox-stackrox-master-ocp-4-19-merge-operator-e2e-tests
name: branch-ci-stackrox-stackrox-master-ocp-4-20-merge-operator-e2e-tests
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --report-credentials-file=/etc/report/credentials
- --target=merge-operator-e2e-tests
- --variant=ocp-4-19
- --variant=ocp-4-20
command:
- ci-operator
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
Expand Down Expand Up @@ -1022,18 +1022,18 @@ postsubmits:
skip_cloning: true
timeout: 5h0m0s
labels:
ci-operator.openshift.io/variant: ocp-4-19
ci-operator.openshift.io/variant: ocp-4-20
ci.openshift.io/generator: prowgen
max_concurrency: 6
name: branch-ci-stackrox-stackrox-master-ocp-4-19-merge-qa-e2e-tests
name: branch-ci-stackrox-stackrox-master-ocp-4-20-merge-qa-e2e-tests
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --report-credentials-file=/etc/report/credentials
- --target=merge-qa-e2e-tests
- --variant=ocp-4-19
- --variant=ocp-4-20
command:
- ci-operator
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
Expand Down Expand Up @@ -1076,18 +1076,18 @@ postsubmits:
skip_cloning: true
timeout: 5h0m0s
labels:
ci-operator.openshift.io/variant: ocp-4-19
ci-operator.openshift.io/variant: ocp-4-20
ci.openshift.io/generator: prowgen
max_concurrency: 6
name: branch-ci-stackrox-stackrox-master-ocp-4-19-merge-scanner-v4-install-tests
name: branch-ci-stackrox-stackrox-master-ocp-4-20-merge-scanner-v4-install-tests
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --report-credentials-file=/etc/report/credentials
- --target=merge-scanner-v4-install-tests
- --variant=ocp-4-19
- --variant=ocp-4-20
command:
- ci-operator
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
Expand Down Expand Up @@ -1130,18 +1130,18 @@ postsubmits:
skip_cloning: true
timeout: 5h0m0s
labels:
ci-operator.openshift.io/variant: ocp-4-19
ci-operator.openshift.io/variant: ocp-4-20
ci.openshift.io/generator: prowgen
max_concurrency: 6
name: branch-ci-stackrox-stackrox-master-ocp-4-19-merge-ui-e2e-tests
name: branch-ci-stackrox-stackrox-master-ocp-4-20-merge-ui-e2e-tests
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --report-credentials-file=/etc/report/credentials
- --target=merge-ui-e2e-tests
- --variant=ocp-4-19
- --variant=ocp-4-20
command:
- ci-operator
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
Expand Down
Loading