diff --git a/internal/provider/kubernetes/kubernetes_test.go b/internal/provider/kubernetes/kubernetes_test.go index 28b820a8fac..88a7d1f1b83 100644 --- a/internal/provider/kubernetes/kubernetes_test.go +++ b/internal/provider/kubernetes/kubernetes_test.go @@ -9,13 +9,11 @@ package kubernetes import ( "context" - "fmt" "os" "path/filepath" "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -25,6 +23,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -62,6 +61,7 @@ func TestProvider(t *testing.T) { // Setup and start the kube provider. svr, err := config.New(os.Stdout, os.Stderr) + require.NoError(t, err) // Disable webhook server for provider test to avoid non-existent cert errors svr.EnvoyGateway.Provider.Kubernetes.TopologyInjector = &egv1a1.EnvoyGatewayTopologyInjector{Disable: ptr.To(true)} @@ -130,7 +130,7 @@ func testGatewayClassController(ctx context.Context, t *testing.T, provider *Pro require.Eventually(t, func() bool { return cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc) == nil }, defaultWait, defaultTick) - assert.Equal(t, gc.ObjectMeta.Generation, int64(1)) + require.Equal(t, gc.ObjectMeta.Generation, int64(1)) } func testGatewayClassAcceptedStatus(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { @@ -143,19 +143,7 @@ func testGatewayClassAcceptedStatus(ctx context.Context, t *testing.T, provider require.NoError(t, cli.Delete(ctx, gc)) }() - require.Eventually(t, func() bool { - if err := cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc); err != nil { - return false - } - - for _, cond := range gc.Status.Conditions { - if cond.Type == string(gwapiv1.GatewayClassConditionStatusAccepted) && cond.Status == metav1.ConditionTrue { - return true - } - } - - return false - }, defaultWait, defaultTick) + requireGatewayClassAccepted(t, cli, gc) require.Eventually(t, func() bool { return resources.GatewayAPIResources.Len() != 0 @@ -164,7 +152,7 @@ func testGatewayClassAcceptedStatus(ctx context.Context, t *testing.T, provider // Even though no gateways exist, the controller loads the empty resource map // to support gateway deletions. require.Eventually(t, func() bool { - _, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + _, ok := getGatewayClassFromResources(resources, gc.Name) return ok }, defaultWait, defaultTick) } @@ -196,33 +184,20 @@ func testGatewayClassWithParamRef(ctx context.Context, t *testing.T, provider *P require.NoError(t, cli.Delete(ctx, gc)) }() - // Ensure the GatewayClass reports "Ready". - require.Eventually(t, func() bool { - if err := cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc); err != nil { - return false - } - - for _, cond := range gc.Status.Conditions { - if cond.Type == string(gwapiv1.GatewayClassConditionStatusAccepted) && cond.Status == metav1.ConditionTrue { - return true - } - } - - return false - }, defaultWait, defaultTick) + requireGatewayClassAccepted(t, cli, gc) require.Eventually(t, func() bool { return resources.GatewayAPIResources.Len() != 0 }, defaultWait, defaultTick) require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } if res.EnvoyProxyForGatewayClass != nil { - assert.Equal(t, res.EnvoyProxyForGatewayClass.Spec, ep.Spec) + require.Equal(t, res.EnvoyProxyForGatewayClass.Spec, ep.Spec) return true } @@ -230,15 +205,10 @@ func testGatewayClassWithParamRef(ctx context.Context, t *testing.T, provider *P }, defaultWait, defaultTick) } -func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { - cli := provider.manager.GetClient() - - gc := test.GetGatewayClass("gc-scheduled-status-test", egv1a1.GatewayControllerName, nil) - require.NoError(t, cli.Create(ctx, gc)) - - // Ensure the GatewayClass reports "Ready". +func requireGatewayClassAccepted(t *testing.T, cli client.Client, gc *gwapiv1.GatewayClass) { + // Ensure the GatewayClass reports "Accepted". require.Eventually(t, func() bool { - if err := cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc); err != nil { + if err := cli.Get(t.Context(), types.NamespacedName{Name: gc.Name}, gc); err != nil { return false } @@ -249,7 +219,16 @@ func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Pro } return false - }, defaultWait, defaultTick) + }, defaultWait, defaultTick, " timed out waiting for GatewayClass %s to report Accepted=True condition", gc.Name) +} + +func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { + cli := provider.manager.GetClient() + + gc := test.GetGatewayClass("gc-scheduled-status-test", egv1a1.GatewayControllerName, nil) + require.NoError(t, cli.Create(ctx, gc)) + + requireGatewayClassAccepted(t, cli, gc) defer func() { require.NoError(t, cli.Delete(ctx, gc)) @@ -331,22 +310,21 @@ func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Pro require.NoError(t, cli.Create(ctx, deploy)) require.NoError(t, cli.Create(ctx, svc)) - // Ensure the Gateway reports "Scheduled". + // Ensure the Gateway reports "Accepted". require.Eventually(t, func() bool { if err := cli.Get(ctx, utils.NamespacedName(gw), gw); err != nil { return false } for _, cond := range gw.Status.Conditions { - fmt.Printf("Condition: %v\n", cond) if cond.Type == string(gwapiv1.GatewayConditionAccepted) && cond.Status == metav1.ConditionTrue { return true } } - // Scheduled=True condition not found. + t.Logf("Accepted=True condition not found in Gateway %s/%s conditions: %+v", gw.Namespace, gw.Name, gw.Status.Conditions) return false - }, defaultWait, defaultTick) + }, defaultWait, defaultTick, " timed out waiting for Gateway %s to report Accepted=True condition", utils.NamespacedName(gw)) defer func() { require.NoError(t, cli.Delete(ctx, gw)) @@ -358,14 +336,14 @@ func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Pro // Ensure the number of Gateways in the Gateway resource table is as expected. require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } return res != nil && len(res.Gateways) == 1 }, defaultWait, defaultTick) - // Ensure the gatewayclass has been finalized. + // Ensure the GatewayClass has been finalized. require.Eventually(t, func() bool { err := cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc) return err == nil && slices.Contains(gc.Finalizers, gatewayClassFinalizer) @@ -378,13 +356,13 @@ func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Pro }, defaultWait, defaultTick) res := resources.GetResourcesByGatewayClass(gc.Name) - assert.NotNil(t, res) + require.NotNil(t, res) // Only check if the spec is equal // The watchable map will not store a resource // with an updated status if the spec has not changed // to eliminate this endless loop: // reconcile->store->translate->update-status->reconcile - assert.Equal(t, gw.Spec, res.Gateways[0].Spec) + require.Equal(t, gw.Spec, res.Gateways[0].Spec) } func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { @@ -393,20 +371,7 @@ func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resour gc := test.GetGatewayClass("httproute-test", egv1a1.GatewayControllerName, nil) require.NoError(t, cli.Create(ctx, gc)) - // Ensure the GatewayClass reports ready. - require.Eventually(t, func() bool { - if err := cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc); err != nil { - return false - } - - for _, cond := range gc.Status.Conditions { - if cond.Type == string(gwapiv1.GatewayClassConditionStatusAccepted) && cond.Status == metav1.ConditionTrue { - return true - } - } - - return false - }, defaultWait, defaultTick) + requireGatewayClassAccepted(t, cli, gc) defer func() { require.NoError(t, cli.Delete(ctx, gc)) @@ -887,7 +852,8 @@ func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resour }, } - for _, testCase := range testCases { + for i := range testCases { + testCase := testCases[i] t.Run(testCase.name, func(t *testing.T) { require.NoError(t, cli.Create(ctx, &testCase.route)) defer func() { @@ -905,7 +871,7 @@ func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resour }, defaultWait, defaultTick) require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -913,12 +879,12 @@ func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resour }, defaultWait, defaultTick) res := resources.GetResourcesByGatewayClass(gc.Name) - assert.NotNil(t, res) - assert.Equal(t, &testCase.route, res.HTTPRoutes[0]) + require.NotNil(t, res) + require.Contains(t, res.HTTPRoutes, &testCase.route) // Ensure the HTTPRoute Namespace is in the Namespace resource map. require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -937,7 +903,7 @@ func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resour return true } - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -1036,7 +1002,8 @@ func testTLSRoute(ctx context.Context, t *testing.T, provider *Provider, resourc }, } - for _, testCase := range testCases { + for i := range testCases { + testCase := testCases[i] t.Run(testCase.name, func(t *testing.T) { require.NoError(t, cli.Create(ctx, &testCase.route)) defer func() { @@ -1054,7 +1021,7 @@ func testTLSRoute(ctx context.Context, t *testing.T, provider *Provider, resourc }, defaultWait, defaultTick) require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -1062,12 +1029,12 @@ func testTLSRoute(ctx context.Context, t *testing.T, provider *Provider, resourc }, defaultWait, defaultTick) res := resources.GetResourcesByGatewayClass(gc.Name) - assert.NotNil(t, res) - assert.Equal(t, &testCase.route, res.TLSRoutes[0]) + require.NotNil(t, res) + require.Contains(t, res.TLSRoutes, &testCase.route) // Ensure the HTTPRoute Namespace is in the Namespace resource map. require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -1081,7 +1048,7 @@ func testTLSRoute(ctx context.Context, t *testing.T, provider *Provider, resourc // Ensure the Service is in the resource map. require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -1218,7 +1185,7 @@ func testServiceCleanupForMultipleRoutes(ctx context.Context, t *testing.T, prov // Check that the Service is present in the resource map require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -1233,7 +1200,7 @@ func testServiceCleanupForMultipleRoutes(ctx context.Context, t *testing.T, prov // Delete the TLSRoute, and check if the Service is still present require.NoError(t, cli.Delete(ctx, &tlsRoute)) require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -1248,7 +1215,7 @@ func testServiceCleanupForMultipleRoutes(ctx context.Context, t *testing.T, prov // Delete the HTTPRoute, and check if the Service is also removed require.NoError(t, cli.Delete(ctx, &httpRoute)) require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gc.Name) + res, ok := getGatewayClassFromResources(resources, gc.Name) if !ok { return false } @@ -1317,7 +1284,7 @@ func TestNamespacedProvider(t *testing.T) { // Ensure only 2 gateways are reconciled gatewayList := &gwapiv1.GatewayList{} require.NoError(t, cli.List(ctx, gatewayList)) - assert.Equal(t, len(gatewayList.Items), 2) + require.Equal(t, len(gatewayList.Items), 2) // Stop the kube provider. defer func() { @@ -1377,19 +1344,7 @@ func TestNamespaceSelectorProvider(t *testing.T) { gc := test.GetGatewayClass(gcName, egv1a1.GatewayControllerName, nil) require.NoError(t, cli.Create(ctx, gc)) - require.Eventually(t, func() bool { - if err := cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc); err != nil { - return false - } - - for _, cond := range gc.Status.Conditions { - if cond.Type == string(gwapiv1.GatewayClassConditionStatusAccepted) && cond.Status == metav1.ConditionTrue { - return true - } - } - - return false - }, defaultWait, defaultTick) + requireGatewayClassAccepted(t, cli, gc) defer func() { require.NoError(t, cli.Delete(ctx, gc)) @@ -1580,7 +1535,7 @@ func TestNamespaceSelectorProvider(t *testing.T) { }) } -func waitUntilGatewayClassResourcesAreReady(resources *message.ProviderResources, gatewayClassName string) (*resource.Resources, bool) { +func getGatewayClassFromResources(resources *message.ProviderResources, gatewayClassName string) (*resource.Resources, bool) { res := resources.GetResourcesByGatewayClass(gatewayClassName) if res == nil { return nil, false @@ -1591,7 +1546,7 @@ func waitUntilGatewayClassResourcesAreReady(resources *message.ProviderResources func requireResourceReady(t *testing.T, resources *message.ProviderResources, gatewayClassName string, cmpFunc func(r *resource.Resources) bool) { require.Eventually(t, func() bool { - res, ok := waitUntilGatewayClassResourcesAreReady(resources, gatewayClassName) + res, ok := getGatewayClassFromResources(resources, gatewayClassName) if !ok { return false } diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 843ead2e77a..4f52e3d2c5a 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -1,10 +1,10 @@ # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. # To know the available versions check: # - https://github.com/kubernetes-sigs/controller-tools/blob/main/envtest-releases.yaml -ENVTEST_K8S_VERSION ?= 1.29.5 +ENVTEST_K8S_VERSION ?= 1.33.0 # Need run cel validation across multiple versions of k8s -# TODO: zhaohuabing update kubebuilder assets to 1.33.0 when available -ENVTEST_K8S_VERSIONS ?= 1.29.5 1.30.3 1.31.0 1.32.0 +# TODO: update kubebuilder assets to 1.34.0 when available +ENVTEST_K8S_VERSIONS ?= 1.30.3 1.31.0 1.32.0 1.33.0 # GATEWAY_API_VERSION refers to the version of Gateway API CRDs. # For more details, see https://gateway-api.sigs.k8s.io/guides/getting-started/#installing-gateway-api