Skip to content

Commit 60f677b

Browse files
pmengelbertpmengelbert
authored
test: catch broken scanner (#540)
* Do not remove failed images in e2e tests This commit introduces an easier-to-use way of setting additional args on the manager for the e2e tests. Along with this change, `--scanner-arg=--delete-scan-failed-images=false` is the default, meaning that tests will not delete images for which the scan failed. This is to address #536, where a failing scanner was not caught by the e2e tests. I have verified locally that a failing scanner will cause at least one test to fail. Signed-off-by: Peter Engelbert <[email protected]> * Ensure failing scanner causes at least one e2e test failure Adds a new e2e test Signed-off-by: Peter Engelbert <[email protected]> * Rename tests to sort logs Multiple tests were assigned the same name, and because the test_logs are placed in directories named after the tests, the log directories were getting cluttered with results from multiple tests. This commit gives each test a unique name, and therefore a unique log directory. Signed-off-by: Peter Engelbert <[email protected]> * Tidy code in imagecollector_controller Signed-off-by: Peter Engelbert <[email protected]> Signed-off-by: Peter Engelbert <[email protected]>
1 parent 3a37497 commit 60f677b

File tree

18 files changed

+141
-15
lines changed

18 files changed

+141
-15
lines changed

‎controllers/imagecollector/imagecollector_controller.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,10 +306,14 @@ func (r *Reconciler) createImageJob(ctx context.Context, req ctrl.Request, argsC
306306
}
307307

308308
if !scanDisabled {
309+
deleteFailedString := strconv.FormatBool(*deleteScanFailedImages)
310+
scanFailedArg := fmt.Sprintf("--delete-scan-failed-images=%s", deleteFailedString)
311+
scannerArgs = append(scannerArgs, scanFailedArg)
312+
309313
scannerContainer := corev1.Container{
310314
Name: "trivy-scanner",
311315
Image: *scannerImage,
312-
Args: append(scannerArgs, "delete-scan-failed-images="+strconv.FormatBool(*deleteScanFailedImages)),
316+
Args: scannerArgs,
313317
VolumeMounts: []corev1.VolumeMount{
314318
{MountPath: "/run/eraser.sh/shared-data", Name: "shared-data"},
315319
},

‎test/e2e/tests/collector_disable_scan/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ func TestMain(m *testing.M) {
4040
"--set", util.CollectorImageTag.Set(collectorImage.Tag),
4141
"--set", util.ManagerImageRepo.Set(managerImage.Repo),
4242
"--set", util.ManagerImageTag.Set(managerImage.Tag),
43-
"--set", `controllerManager.additionalArgs={--job-cleanup-on-success-delay=1m}`),
43+
"--set", util.ManagerAdditionalArgs.Set("--job-cleanup-on-success-delay=1m").String(),
44+
),
4445
).Finish(
4546
envfuncs.DestroyKindCluster(util.KindClusterName),
4647
)

‎test/e2e/tests/collector_ensure_scan/eraser_test.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
package e2e
5+
6+
import (
7+
"context"
8+
"testing"
9+
10+
"github.com/Azure/eraser/test/e2e/util"
11+
12+
"sigs.k8s.io/e2e-framework/pkg/envconf"
13+
"sigs.k8s.io/e2e-framework/pkg/features"
14+
)
15+
16+
func TestEnsureScannerFunctions(t *testing.T) {
17+
collectScanErasePipelineFeat := features.New("Collector pods should run automatically, trigger the scanner, then the eraser pods. Manifest deployment test.").
18+
Assess("Vulnerable Image successfully deleted from all nodes", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
19+
ctxT, cancel := context.WithTimeout(ctx, util.Timeout)
20+
defer cancel()
21+
util.CheckImageRemoved(ctxT, t, util.GetClusterNodes(t), util.Alpine)
22+
23+
return ctx
24+
}).
25+
Assess("Get logs", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
26+
if err := util.GetManagerLogs(ctx, cfg, t); err != nil {
27+
t.Error("error getting manager logs", err)
28+
}
29+
30+
if err := util.GetPodLogs(ctx, cfg, t, false); err != nil {
31+
t.Error("error getting manager logs", err)
32+
}
33+
34+
return ctx
35+
}).
36+
Feature()
37+
38+
util.Testenv.Test(t, collectScanErasePipelineFeat)
39+
}

‎test/e2e/tests/collector_ensure_scan/main_test.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
package e2e
5+
6+
import (
7+
"os"
8+
"testing"
9+
10+
eraserv1alpha1 "github.com/Azure/eraser/api/v1alpha1"
11+
"github.com/Azure/eraser/test/e2e/util"
12+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
13+
"k8s.io/client-go/kubernetes/scheme"
14+
"sigs.k8s.io/e2e-framework/pkg/env"
15+
"sigs.k8s.io/e2e-framework/pkg/envconf"
16+
"sigs.k8s.io/e2e-framework/pkg/envfuncs"
17+
)
18+
19+
func TestMain(m *testing.M) {
20+
utilruntime.Must(eraserv1alpha1.AddToScheme(scheme.Scheme))
21+
22+
eraserImage := util.ParsedImages.EraserImage
23+
managerImage := util.ParsedImages.ManagerImage
24+
collectorImage := util.ParsedImages.CollectorImage
25+
scannerImage := util.ParsedImages.ScannerImage
26+
27+
util.Testenv = env.NewWithConfig(envconf.New())
28+
// Create KinD Cluster
29+
util.Testenv.Setup(
30+
envfuncs.CreateKindClusterWithConfig(util.KindClusterName, util.NodeVersion, "../../kind-config.yaml"),
31+
envfuncs.CreateNamespace(util.TestNamespace),
32+
envfuncs.LoadDockerImageToCluster(util.KindClusterName, util.ManagerImage),
33+
envfuncs.LoadDockerImageToCluster(util.KindClusterName, util.Image),
34+
envfuncs.LoadDockerImageToCluster(util.KindClusterName, util.CollectorImage),
35+
envfuncs.LoadDockerImageToCluster(util.KindClusterName, util.VulnerableImage),
36+
envfuncs.LoadDockerImageToCluster(util.KindClusterName, util.NonVulnerableImage),
37+
envfuncs.LoadDockerImageToCluster(util.KindClusterName, util.ScannerImage),
38+
util.DeployEraserHelm(util.TestNamespace,
39+
"--set", util.ScannerImageRepo.Set(scannerImage.Repo),
40+
"--set", util.ScannerImageTag.Set(scannerImage.Tag),
41+
"--set", util.EraserImageRepo.Set(eraserImage.Repo),
42+
"--set", util.EraserImageTag.Set(eraserImage.Tag),
43+
"--set", util.CollectorImageRepo.Set(collectorImage.Repo),
44+
"--set", util.CollectorImageTag.Set(collectorImage.Tag),
45+
"--set", util.ManagerImageRepo.Set(managerImage.Repo),
46+
"--set", util.ManagerImageTag.Set(managerImage.Tag),
47+
"--set", util.ManagerAdditionalArgs.Set("--job-cleanup-on-success-delay=2m").String(),
48+
),
49+
).Finish(
50+
envfuncs.DestroyKindCluster(util.KindClusterName),
51+
)
52+
os.Exit(util.Testenv.Run(m))
53+
}

‎test/e2e/tests/collector_skip_excluded/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ func TestMain(m *testing.M) {
4646
"--set", util.CollectorImageTag.Set(collectorImage.Tag),
4747
"--set", util.ManagerImageRepo.Set(managerImage.Repo),
4848
"--set", util.ManagerImageTag.Set(managerImage.Tag),
49-
"--set", `controllerManager.additionalArgs={--job-cleanup-on-success-delay=2m}`),
49+
"--set", util.ManagerAdditionalArgs.Set("--job-cleanup-on-success-delay=2m").String(),
50+
),
5051
).Finish(
5152
envfuncs.DestroyKindCluster(util.KindClusterName),
5253
)

‎test/e2e/tests/helm_pull_secret/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ func TestMain(m *testing.M) {
4343
"--set", util.ManagerImageRepo.Set(managerImage.Repo),
4444
"--set", util.ManagerImageTag.Set(managerImage.Tag),
4545
"--set-json", util.ImagePullSecrets.Set(util.ImagePullSecretJSON),
46-
"--set", `controllerManager.additionalArgs={--job-cleanup-on-success-delay=2m}`),
46+
"--set", util.ManagerAdditionalArgs.Set("--job-cleanup-on-success-delay=2m").String(),
47+
),
4748
).Finish(
4849
envfuncs.DestroyKindCluster(util.KindClusterName),
4950
)

‎test/e2e/tests/imagelist_alias/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ func TestMain(m *testing.M) {
3737
"--set", util.EraserImageTag.Set(eraserImage.Tag),
3838
"--set", util.ManagerImageRepo.Set(managerImage.Repo),
3939
"--set", util.ManagerImageTag.Set(managerImage.Tag),
40-
"--set", `controllerManager.additionalArgs={--job-cleanup-on-success-delay=1m}`),
40+
"--set", util.ManagerAdditionalArgs.Set("--job-cleanup-on-success-delay=1m").String(),
41+
),
4142
).Finish(
4243
envfuncs.DestroyKindCluster(util.KindClusterName),
4344
)

‎test/e2e/tests/imagelist_exclusion_list/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ func TestMain(m *testing.M) {
3636
"--set", util.EraserImageTag.Set(eraserImage.Tag),
3737
"--set", util.ManagerImageRepo.Set(managerImage.Repo),
3838
"--set", util.ManagerImageTag.Set(managerImage.Tag),
39-
"--set", `controllerManager.additionalArgs={--job-cleanup-on-success-delay=1m}`),
39+
"--set", util.ManagerAdditionalArgs.Set("--job-cleanup-on-success-delay=1m").String(),
40+
),
4041
).Finish(
4142
envfuncs.DestroyKindCluster(util.KindClusterName),
4243
)

‎test/e2e/tests/imagelist_include_nodes/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ func TestMain(m *testing.M) {
3636
"--set", util.EraserImageTag.Set(eraserImage.Tag),
3737
"--set", util.ManagerImageRepo.Set(managerImage.Repo),
3838
"--set", util.ManagerImageTag.Set(managerImage.Tag),
39-
"--set", `controllerManager.additionalArgs={--filter-nodes=include, --job-cleanup-on-success-delay=1m}`),
39+
"--set", util.ManagerAdditionalArgs.Set("--filter-nodes=include").Set("--job-cleanup-on-success-delay=1m").String(),
40+
),
4041
).Finish(
4142
envfuncs.DestroyKindCluster(util.KindClusterName),
4243
)

‎test/e2e/tests/imagelist_prune_images/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ func TestMain(m *testing.M) {
3636
"--set", util.EraserImageTag.Set(eraserImage.Tag),
3737
"--set", util.ManagerImageRepo.Set(managerImage.Repo),
3838
"--set", util.ManagerImageTag.Set(managerImage.Tag),
39-
"--set", `controllerManager.additionalArgs={--job-cleanup-on-success-delay=1m}`),
39+
"--set", util.ManagerAdditionalArgs.Set("--job-cleanup-on-success-delay=1m").String(),
40+
),
4041
).Finish(
4142
envfuncs.DestroyKindCluster(util.KindClusterName),
4243
)

0 commit comments

Comments
 (0)