implement user-server(service-proxy), provide user a https server to access managed cluster service on the hub side.

Signed-off-by: xuezhaojun <[email protected]>

Author: xuezhaojun

.github/workflows/go-presubmit.yml

@@ -90,25 +90,11 @@ jobs:
         run: curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]
-      - name: Prepare OCM testing environment
-        run: |
-          clusteradm init --output-join-command-file join.sh --wait
-          sh -c "$(cat join.sh) loopback --force-internal-endpoint-lookup"
-          clusteradm accept --clusters loopback --wait 30
-          kubectl wait --for=condition=ManagedClusterConditionAvailable managedcluster/loopback
+        with:
+          cluster_name: e2e
       - name: Build image
-        run: |
-          make images
-          kind load docker-image quay.io/open-cluster-management/cluster-proxy:latest  --name chart-testing
-      - name: Install latest cluster-proxy
-        run: |
-          helm install \
-             -n open-cluster-management-addon --create-namespace \
-             cluster-proxy charts/cluster-proxy/ \
-             --set tag=latest --set installByPlacement.placementName=default
-      - name: Build&Run e2e test
-        run: |
-          kubectl wait --for=condition=ProxyServerDeployed=true managedproxyconfiguration cluster-proxy --timeout=60s
-          kubectl wait --for=condition=Available deployment/cluster-proxy --timeout=60s -n open-cluster-management-addon
-          kubectl port-forward -n open-cluster-management-addon services/proxy-entrypoint 8090:8090 &
-          make test-e2e
+        run: make images
+      - name: Setup environment
+        run: make setup-env-for-e2e && make deploy-cluster-proxy-e2e
+      - name: Run e2e tests
+        run: make test-e2e

Makefile

@@ -3,11 +3,15 @@ IMG ?= controller:latest
 IMAGE_REGISTRY_NAME ?= quay.io/open-cluster-management
 IMAGE_NAME = cluster-proxy
 IMAGE_TAG ?= latest
-E2E_TEST_CLUSTER_NAME ?= loopback
+E2E_TEST_CLUSTER_NAME ?= e2e
 CONTAINER_ENGINE ?= docker
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:crdVersions={v1},allowDangerousTypes=true,generateEmbeddedObjectMeta=true"
 
+# Label filter for e2e tests (Ginkgo v2 label filter expression)
+# Examples: "install", "connectivity", "certificate && !rotation", etc.
+LABEL_FILTER ?=
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -56,11 +60,7 @@ fmt: ## Run go fmt against code.
 vet: ## Run go vet against code.
 	go vet ./...
 
-golint:
-	go install github.com/golangci/golangci-lint/cmd/[email protected]
-	golangci-lint run --timeout=3m ./...
-
-verify: fmt vet golint
+verify: fmt vet
 
 test: manifests generate fmt vet ## Run tests.
 	go test ./pkg/... -coverprofile cover.out
@@ -70,6 +70,7 @@ test: manifests generate fmt vet ## Run tests.
 build: generate fmt vet
 	go build -o bin/addon-manager cmd/addon-manager/main.go
 	go build -o bin/addon-agent cmd/addon-agent/main.go
+	go build -o bin/cluster-proxy cmd/cluster-proxy/main.go
 
 docker-build: test ## Build docker image with the manager.
 	$(CONTAINER_ENGINE) build -t ${IMG} .
@@ -81,7 +82,7 @@ docker-push: ## Push docker image with the manager.
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.18.0)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
@@ -117,6 +118,30 @@ images:
 		--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
 		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
 
+images-amd64:
+	$(CONTAINER_ENGINE) buildx build \
+		--platform linux/amd64 \
+		--load \
+		-f cmd/Dockerfile \
+		--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
+		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
+
+pure-image:
+	$(CONTAINER_ENGINE) build \
+		-f cmd/pure.Dockerfile \
+		--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
+		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
+
+pure-image-amd64:
+	$(CONTAINER_ENGINE) buildx build \
+		--platform linux/amd64 \
+		--load \
+		-f cmd/pure.Dockerfile \
+		--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
+		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
+
+## Integration Testing
+
 ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
 test-integration: manifests generate fmt vet
 	mkdir -p ${ENVTEST_ASSETS_DIR}
@@ -126,13 +151,126 @@ test-integration: manifests generate fmt vet
 		setup_envtest_env $(ENVTEST_ASSETS_DIR); \
 		go test ./test/integration/... -coverprofile cover.out
 
-e2e-job-image:
+## E2E Testing
+
+# Note: here we use internal service ns as the entrypointAddress. The test cluster should be registered to itself as a managed cluster.
+setup-env-for-e2e:
+	@echo "Setting up environment for e2e tests..."
+	./test/e2e/env/init.sh
+.PHONY: setup-env-for-e2e
+
+# load cluster-proxy image into kind cluster
+load-cluster-proxy-image-kind:
+	@echo "Loading cluster-proxy image into kind cluster..."
+	kind load docker-image $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) --name $(E2E_TEST_CLUSTER_NAME)
+.PHONY: load-cluster-proxy-image-kind
+
+# delete cluster-proxy image from kind cluster nodes
+delete-cluster-proxy-image-from-kind:
+	@echo "Deleting cluster-proxy image from kind cluster nodes..."
+	@for node in $$(kind get nodes --name $(E2E_TEST_CLUSTER_NAME) 2>/dev/null || echo ""); do \
+		if [ -n "$$node" ]; then \
+			docker exec $$node crictl rmi $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) 2>/dev/null || true; \
+		fi; \
+	done
+.PHONY: delete-cluster-proxy-image-from-kind
+
+deploy-cluster-proxy-e2e: delete-cluster-proxy-image-from-kind load-cluster-proxy-image-kind
+	@echo "Deploying cluster-proxy crds..."
+	kubectl apply -f charts/cluster-proxy/crds/managedproxyconfigurations.yaml
+	kubectl apply -f charts/cluster-proxy/crds/managedproxyserviceresolvers.yaml
+	@echo "Deploying cluster-proxy..."
+	helm install \
+	-n open-cluster-management-addon --create-namespace \
+	cluster-proxy charts/cluster-proxy \
+	--set registry=$(IMAGE_REGISTRY_NAME) \
+	--set image=$(IMAGE_NAME) \
+	--set tag=$(IMAGE_TAG) \
+	--set proxyServerImage=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME) \
+	--set proxyAgentImage=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME) \
+	--set proxyServer.entrypointAddress="proxy-entrypoint.open-cluster-management-addon.svc" \
+	--set proxyServer.port=8091
+	@echo "Cluster-proxy deployed successfully!"
+.PHONY: deploy-cluster-proxy-e2e
+
+# Build e2e test container image
+build-e2e-image:
+	@echo "Building e2e test container image..."
 	$(CONTAINER_ENGINE) build \
-		-f test/e2e/job/Dockerfile \
-		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e-job:$(IMAGE_TAG) .
+		-f test/e2e/Dockerfile \
+		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) .
+.PHONY: build-e2e-image
+
+# Load e2e image into kind cluster (for local testing)
+load-e2e-image-kind:
+	@echo "Loading e2e image into kind cluster..."
+	kind load docker-image $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) --name $(E2E_TEST_CLUSTER_NAME)
+.PHONY: load-e2e-image-kind
+
+# Delete e2e image from kind cluster nodes (for rapid iteration)
+delete-e2e-image-from-kind:
+	@echo "Deleting e2e image from kind cluster nodes..."
+	@for node in $$(kind get nodes --name $(E2E_TEST_CLUSTER_NAME) 2>/dev/null || echo ""); do \
+		if [ -n "$$node" ]; then \
+			docker exec $$node crictl rmi $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) 2>/dev/null || true; \
+		fi; \
+	done
+.PHONY: delete-e2e-image-from-kind
+
+# Run e2e tests in cluster using container image (Kubernetes-native approach)
+# Use LABEL_FILTER to run specific tests, e.g.: make test-e2e LABEL_FILTER="install"
+test-e2e: delete-e2e-image-from-kind build-e2e-image load-e2e-image-kind
+	@echo "Deleting existing e2e test job if present..."
+	@kubectl delete job cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found
+	@echo "Deploying e2e test job..."
+	@if [ -n "$(LABEL_FILTER)" ]; then \
+		echo "Running tests with label filter: $(LABEL_FILTER)"; \
+	fi
+	@sed -e '/name: LABEL_FILTER/{n;s|value: ""|value: "$(LABEL_FILTER)"|;}' \
+	     -e 's|image: quay.io/open-cluster-management/cluster-proxy-e2e:latest|image: $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG)|g' \
+	     test/e2e/env/job.yaml | kubectl apply -f -
+	@./test/e2e/env/wait-for-job.sh cluster-proxy-e2e open-cluster-management-addon 1200
+.PHONY: test-e2e
+
+# Rapid iteration workflow for e2e tests (cleans up everything first)
+# Use LABEL_FILTER to run specific tests, e.g.: make retest-e2e LABEL_FILTER="connectivity"
+retest-e2e: clean-e2e delete-e2e-image-from-kind build-e2e-image load-e2e-image-kind
+	@echo "Deleting existing e2e test job if present..."
+	@kubectl delete job cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found
+	@echo "Deploying e2e test job..."
+	@if [ -n "$(LABEL_FILTER)" ]; then \
+		echo "Running tests with label filter: $(LABEL_FILTER)"; \
+	fi
+	@sed -e '/name: LABEL_FILTER/{n;s|value: ""|value: "$(LABEL_FILTER)"|;}' \
+	     -e 's|image: quay.io/open-cluster-management/cluster-proxy-e2e:latest|image: $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG)|g' \
+	     test/e2e/env/job.yaml | kubectl apply -f -
+	@./test/e2e/env/wait-for-job.sh cluster-proxy-e2e open-cluster-management-addon 1200
+.PHONY: retest-e2e
 
-build-e2e:
-	go test -c -o bin/e2e ./test/e2e/
+# Clean up e2e test job and related resources
+clean-e2e:
+	@echo "Cleaning up e2e test resources..."
+	kubectl delete job/cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found=true
+	kubectl delete serviceaccount/cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found=true
+	kubectl delete clusterrolebinding/cluster-proxy-e2e --ignore-not-found=true
+	kubectl delete clusterrole/cluster-proxy-e2e --ignore-not-found=true
+.PHONY: clean-e2e
 
-test-e2e: build-e2e
-	./bin/e2e --test-cluster $(E2E_TEST_CLUSTER_NAME)
+# Quick verify of user-server
+# Example result:
+# {
+#   "kind": "APIVersions",
+#   "versions": [
+#     "v1"
+#   ],
+#   "serverAddressByClientCIDRs": [
+#     {
+#       "clientCIDR": "0.0.0.0/0",
+#       "serverAddress": "172.17.0.2:6443"
+#     }
+#   ]
+# }
+verify-user-server:
+	@echo "Verifying user-server..."
+	TOKEN=$$(kubectl create token default -n default) && POD=$$(kubectl get pods -n open-cluster-management-addon -l component=cluster-proxy-user --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') && kubectl debug -it $$POD -n open-cluster-management-addon --image=praqma/network-multitool -- sh -c "curl -k -H 'Authorization: Bearer $$TOKEN' https://cluster-proxy-user.open-cluster-management-addon.svc.cluster.local:9092/loopback/api"
+.PHONY: verify-user-server

charts/cluster-proxy/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: cluster-proxy
 description: A Helm chart for Cluster-Proxy OCM Addon
 type: application
-version: 0.8.0
+version: 0.9.0
 appVersion: 1.0.0