Skip to content

Commit 125fa92

Browse files
xuezhaojunxuezhaojun
committed
implement user-server(service-proxy), provide user a https server to access managed cluster service on the hub side.
Signed-off-by: xuezhaojun <[email protected]>
1 parent 7bcbdcf commit 125fa92

File tree

200 files changed

+23409
-1926
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

200 files changed

+23409
-1926
lines changed

‎.github/workflows/go-presubmit.yml‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -90,25 +90,13 @@ jobs:
9090
run: curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash
9191
- name: Create k8s Kind Cluster
9292
uses: helm/[email protected]
93-
- name: Prepare OCM testing environment
94-
run: |
95-
clusteradm init --output-join-command-file join.sh --wait
96-
sh -c "$(cat join.sh) loopback --force-internal-endpoint-lookup"
97-
clusteradm accept --clusters loopback --wait 30
98-
kubectl wait --for=condition=ManagedClusterConditionAvailable managedcluster/loopback
99-
- name: Build image
93+
with:
94+
cluster_name: e2e
95+
- name: Setup environment
96+
run: make setup-env-for-e2e && make deploy-cluster-proxy-e2e
97+
- name: Build main images
10098
run: |
10199
make images
102-
kind load docker-image quay.io/open-cluster-management/cluster-proxy:latest --name chart-testing
103-
- name: Install latest cluster-proxy
104-
run: |
105-
helm install \
106-
-n open-cluster-management-addon --create-namespace \
107-
cluster-proxy charts/cluster-proxy/ \
108-
--set tag=latest --set installByPlacement.placementName=default
109-
- name: Build&Run e2e test
110-
run: |
111-
kubectl wait --for=condition=ProxyServerDeployed=true managedproxyconfiguration cluster-proxy --timeout=60s
112-
kubectl wait --for=condition=Available deployment/cluster-proxy --timeout=60s -n open-cluster-management-addon
113-
kubectl port-forward -n open-cluster-management-addon services/proxy-entrypoint 8090:8090 &
114-
make test-e2e
100+
kind load docker-image quay.io/open-cluster-management/cluster-proxy:latest --name e2e
101+
- name: Run e2e tests
102+
run: make test-e2e

‎Makefile‎

Lines changed: 151 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,15 @@ IMG ?= controller:latest
33
IMAGE_REGISTRY_NAME ?= quay.io/open-cluster-management
44
IMAGE_NAME = cluster-proxy
55
IMAGE_TAG ?= latest
6-
E2E_TEST_CLUSTER_NAME ?= loopback
6+
E2E_TEST_CLUSTER_NAME ?= e2e
77
CONTAINER_ENGINE ?= docker
88
# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
99
CRD_OPTIONS ?= "crd:crdVersions={v1},allowDangerousTypes=true,generateEmbeddedObjectMeta=true"
1010

11+
# Label filter for e2e tests (Ginkgo v2 label filter expression)
12+
# Examples: "install", "connectivity", "certificate && !rotation", etc.
13+
LABEL_FILTER ?=
14+
1115
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
1216
ifeq (,$(shell go env GOBIN))
1317
GOBIN=$(shell go env GOPATH)/bin
@@ -70,6 +74,7 @@ test: manifests generate fmt vet ## Run tests.
7074
build: generate fmt vet
7175
go build -o bin/addon-manager cmd/addon-manager/main.go
7276
go build -o bin/addon-agent cmd/addon-agent/main.go
77+
go build -o bin/cluster-proxy cmd/cluster-proxy/main.go
7378

7479
docker-build: test ## Build docker image with the manager.
7580
$(CONTAINER_ENGINE) build -t ${IMG} .
@@ -81,7 +86,7 @@ docker-push: ## Push docker image with the manager.
8186

8287
CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
8388
controller-gen: ## Download controller-gen locally if necessary.
84-
$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.18.0)
89+
$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0)
8590

8691
KUSTOMIZE = $(shell pwd)/bin/kustomize
8792
kustomize: ## Download kustomize locally if necessary.
@@ -117,6 +122,30 @@ images:
117122
--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
118123
-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
119124

125+
images-amd64:
126+
$(CONTAINER_ENGINE) buildx build \
127+
--platform linux/amd64 \
128+
--load \
129+
-f cmd/Dockerfile \
130+
--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
131+
-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
132+
133+
pure-image:
134+
$(CONTAINER_ENGINE) build \
135+
-f cmd/pure.Dockerfile \
136+
--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
137+
-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
138+
139+
pure-image-amd64:
140+
$(CONTAINER_ENGINE) buildx build \
141+
--platform linux/amd64 \
142+
--load \
143+
-f cmd/pure.Dockerfile \
144+
--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
145+
-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
146+
147+
## Integration Testing
148+
120149
ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
121150
test-integration: manifests generate fmt vet
122151
mkdir -p ${ENVTEST_ASSETS_DIR}
@@ -126,13 +155,126 @@ test-integration: manifests generate fmt vet
126155
setup_envtest_env $(ENVTEST_ASSETS_DIR); \
127156
go test ./test/integration/... -coverprofile cover.out
128157

129-
e2e-job-image:
158+
## E2E Testing
159+
160+
# Note: here we use internal service ns as the entrypointAddress. The test cluster should be registered to itself as a managed cluster.
161+
setup-env-for-e2e:
162+
@echo "Setting up environment for e2e tests..."
163+
./test/e2e/env/init.sh
164+
.PHONY: setup-env-for-e2e
165+
166+
# load cluster-proxy image into kind cluster
167+
load-cluster-proxy-image-kind:
168+
@echo "Loading cluster-proxy image into kind cluster..."
169+
kind load docker-image $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) --name $(E2E_TEST_CLUSTER_NAME)
170+
.PHONY: load-cluster-proxy-image-kind
171+
172+
# delete cluster-proxy image from kind cluster nodes
173+
delete-cluster-proxy-image-from-kind:
174+
@echo "Deleting cluster-proxy image from kind cluster nodes..."
175+
@for node in $$(kind get nodes --name $(E2E_TEST_CLUSTER_NAME) 2>/dev/null || echo ""); do \
176+
if [ -n "$$node" ]; then \
177+
docker exec $$node crictl rmi $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) 2>/dev/null || true; \
178+
fi; \
179+
done
180+
.PHONY: delete-cluster-proxy-image-from-kind
181+
182+
deploy-cluster-proxy-e2e: delete-cluster-proxy-image-from-kind load-cluster-proxy-image-kind
183+
@echo "Deploying cluster-proxy crds..."
184+
kubectl apply -f charts/cluster-proxy/crds/managedproxyconfigurations.yaml
185+
kubectl apply -f charts/cluster-proxy/crds/managedproxyserviceresolvers.yaml
186+
@echo "Deploying cluster-proxy..."
187+
helm install \
188+
-n open-cluster-management-addon --create-namespace \
189+
cluster-proxy charts/cluster-proxy \
190+
--set registry=$(IMAGE_REGISTRY_NAME) \
191+
--set image=$(IMAGE_NAME) \
192+
--set tag=$(IMAGE_TAG) \
193+
--set proxyServerImage=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME) \
194+
--set proxyAgentImage=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME) \
195+
--set proxyServer.entrypointAddress="proxy-entrypoint.open-cluster-management-addon.svc" \
196+
--set proxyServer.port=8091
197+
@echo "Cluster-proxy deployed successfully!"
198+
.PHONY: deploy-cluster-proxy-e2e
199+
200+
# Build e2e test container image
201+
build-e2e-image:
202+
@echo "Building e2e test container image..."
130203
$(CONTAINER_ENGINE) build \
131-
-f test/e2e/job/Dockerfile \
132-
-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e-job:$(IMAGE_TAG) .
204+
-f test/e2e/Dockerfile \
205+
-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) .
206+
.PHONY: build-e2e-image
207+
208+
# Load e2e image into kind cluster (for local testing)
209+
load-e2e-image-kind:
210+
@echo "Loading e2e image into kind cluster..."
211+
kind load docker-image $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) --name $(E2E_TEST_CLUSTER_NAME)
212+
.PHONY: load-e2e-image-kind
213+
214+
# Delete e2e image from kind cluster nodes (for rapid iteration)
215+
delete-e2e-image-from-kind:
216+
@echo "Deleting e2e image from kind cluster nodes..."
217+
@for node in $$(kind get nodes --name $(E2E_TEST_CLUSTER_NAME) 2>/dev/null || echo ""); do \
218+
if [ -n "$$node" ]; then \
219+
docker exec $$node crictl rmi $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) 2>/dev/null || true; \
220+
fi; \
221+
done
222+
.PHONY: delete-e2e-image-from-kind
223+
224+
# Run e2e tests in cluster using container image (Kubernetes-native approach)
225+
# Use LABEL_FILTER to run specific tests, e.g.: make test-e2e LABEL_FILTER="install"
226+
test-e2e: delete-e2e-image-from-kind build-e2e-image load-e2e-image-kind
227+
@echo "Deleting existing e2e test job if present..."
228+
@kubectl delete job cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found
229+
@echo "Deploying e2e test job..."
230+
@if [ -n "$(LABEL_FILTER)" ]; then \
231+
echo "Running tests with label filter: $(LABEL_FILTER)"; \
232+
fi
233+
@sed -e '/name: LABEL_FILTER/{n;s|value: ""|value: "$(LABEL_FILTER)"|;}' \
234+
-e 's|image: quay.io/open-cluster-management/cluster-proxy-e2e:latest|image: $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG)|g' \
235+
test/e2e/env/job.yaml | kubectl apply -f -
236+
@./test/e2e/env/wait-for-job.sh cluster-proxy-e2e open-cluster-management-addon 1200
237+
.PHONY: test-e2e
238+
239+
# Rapid iteration workflow for e2e tests (cleans up everything first)
240+
# Use LABEL_FILTER to run specific tests, e.g.: make retest-e2e LABEL_FILTER="connectivity"
241+
retest-e2e: clean-e2e delete-e2e-image-from-kind build-e2e-image load-e2e-image-kind
242+
@echo "Deleting existing e2e test job if present..."
243+
@kubectl delete job cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found
244+
@echo "Deploying e2e test job..."
245+
@if [ -n "$(LABEL_FILTER)" ]; then \
246+
echo "Running tests with label filter: $(LABEL_FILTER)"; \
247+
fi
248+
@sed -e '/name: LABEL_FILTER/{n;s|value: ""|value: "$(LABEL_FILTER)"|;}' \
249+
-e 's|image: quay.io/open-cluster-management/cluster-proxy-e2e:latest|image: $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG)|g' \
250+
test/e2e/env/job.yaml | kubectl apply -f -
251+
@./test/e2e/env/wait-for-job.sh cluster-proxy-e2e open-cluster-management-addon 1200
252+
.PHONY: retest-e2e
133253

134-
build-e2e:
135-
go test -c -o bin/e2e ./test/e2e/
254+
# Clean up e2e test job and related resources
255+
clean-e2e:
256+
@echo "Cleaning up e2e test resources..."
257+
kubectl delete job/cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found=true
258+
kubectl delete serviceaccount/cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found=true
259+
kubectl delete clusterrolebinding/cluster-proxy-e2e --ignore-not-found=true
260+
kubectl delete clusterrole/cluster-proxy-e2e --ignore-not-found=true
261+
.PHONY: clean-e2e
136262

137-
test-e2e: build-e2e
138-
./bin/e2e --test-cluster $(E2E_TEST_CLUSTER_NAME)
263+
# Quick verify of user-server
264+
# Example result:
265+
# {
266+
# "kind": "APIVersions",
267+
# "versions": [
268+
# "v1"
269+
# ],
270+
# "serverAddressByClientCIDRs": [
271+
# {
272+
# "clientCIDR": "0.0.0.0/0",
273+
# "serverAddress": "172.17.0.2:6443"
274+
# }
275+
# ]
276+
# }
277+
verify-user-server:
278+
@echo "Verifying user-server..."
279+
TOKEN=$$(kubectl create token default -n default) && POD=$$(kubectl get pods -n open-cluster-management-addon -l component=cluster-proxy-user --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') && kubectl debug -it $$POD -n open-cluster-management-addon --image=praqma/network-multitool -- sh -c "curl -k -H 'Authorization: Bearer $$TOKEN' https://cluster-proxy-user.open-cluster-management-addon.svc.cluster.local:9092/loopback/api"
280+
.PHONY: verify-user-server

‎charts/cluster-proxy/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@ apiVersion: v2
22
name: cluster-proxy
33
description: A Helm chart for Cluster-Proxy OCM Addon
44
type: application
5-
version: 0.8.0
5+
version: 0.9.0
66
appVersion: 1.0.0

‎charts/cluster-proxy/README.md‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# Cluster Proxy Helm Chart
2+
3+
This Helm chart installs the Cluster Proxy addon for Open Cluster Management (OCM), which enables accessing services in isolated managed clusters through reverse proxy tunnels.
4+
5+
## Prerequisites
6+
7+
- Kubernetes cluster
8+
- Helm 3.x
9+
- Open Cluster Management (OCM) installed
10+
11+
## Installation
12+
13+
```bash
14+
helm install cluster-proxy ./charts/cluster-proxy \
15+
--namespace open-cluster-management-addon \
16+
--create-namespace
17+
```
18+
19+
## Configuration
20+
21+
### Values
22+
23+
| Parameter | Description | Default |
24+
|-----------|-------------|---------|
25+
| `registry` | Image registry | `quay.io/open-cluster-management` |
26+
| `image` | Image name | `cluster-proxy` |
27+
| `tag` | Image tag | Chart version |
28+
| `replicas` | Number of replicas | `1` |
29+
| `spokeAddonNamespace` | Namespace for spoke addon | `open-cluster-management-cluster-proxy` |
30+
| `proxyServerImage` | Proxy server image | `quay.io/open-cluster-management/cluster-proxy` |
31+
| `proxyAgentImage` | Proxy agent image | `quay.io/open-cluster-management/cluster-proxy` |
32+
| `proxyServer.entrypointLoadBalancer` | Enable LoadBalancer for entrypoint | `false` |
33+
| `proxyServer.entrypointAddress` | Custom entrypoint address | `""` |
34+
| `proxyServer.port` | Proxy server port | `8091` |
35+
| `installByPlacement.placementName` | Placement name for installation | `""` |
36+
| `installByPlacement.placementNamespace` | Placement namespace | `""` |
37+
| `enableUserServer` | Enable user server deployment | `false` |
38+
39+
### User Server Configuration
40+
41+
The user server provides an API endpoint for managing cluster proxy connections. To enable it:
42+
43+
```bash
44+
helm install cluster-proxy ./charts/cluster-proxy \
45+
--set enableUserServer=true
46+
```
47+
48+
#### Important Prerequisites for User Server
49+
50+
**Before enabling the user server, you MUST create the following secret in the installation namespace:**
51+
52+
**cluster-proxy-user-serving-cert** - TLS certificate for the user server
53+
```yaml
54+
apiVersion: v1
55+
kind: Secret
56+
type: kubernetes.io/tls
57+
metadata:
58+
name: cluster-proxy-user-serving-cert
59+
namespace: <release-namespace>
60+
data:
61+
tls.crt: <base64-encoded-certificate>
62+
tls.key: <base64-encoded-private-key>
63+
```
64+
65+
**Automatically Created Secrets:**
66+
67+
The following secrets will be automatically created by the controller and do NOT need to be created manually:
68+
- **proxy-server-ca** - CA certificate for the proxy server
69+
- **proxy-client** - Client certificate for proxy authentication
70+
71+
**⚠️ Warning:** If the `cluster-proxy-user-serving-cert` secret is not present before installation, the user-server deployment will remain in **Pending** state and pods will fail to start.
72+
73+
To verify the secret is created:
74+
```bash
75+
kubectl get secret -n <release-namespace> cluster-proxy-user-serving-cert
76+
```
77+
78+
## Examples
79+
80+
### Basic Installation
81+
```bash
82+
helm install cluster-proxy ./charts/cluster-proxy
83+
```
84+
85+
### With User Server Enabled
86+
```bash
87+
# First, create the required secret
88+
kubectl create secret tls cluster-proxy-user-serving-cert \
89+
--cert=path/to/tls.crt \
90+
--key=path/to/tls.key \
91+
-n open-cluster-management-addon
92+
93+
# Then install with user server enabled
94+
# Note: proxy-server-ca and proxy-client secrets will be created automatically by the controller
95+
helm install cluster-proxy ./charts/cluster-proxy \
96+
--namespace open-cluster-management-addon \
97+
--set enableUserServer=true
98+
```
99+
100+
### Custom Image and Replicas
101+
```bash
102+
helm install cluster-proxy ./charts/cluster-proxy \
103+
--set image=my-custom-proxy \
104+
--set tag=v1.0.0 \
105+
--set replicas=3
106+
```
107+
108+
## Upgrading
109+
110+
```bash
111+
helm upgrade cluster-proxy ./charts/cluster-proxy
112+
```
113+
114+
## Uninstallation
115+
116+
```bash
117+
helm uninstall cluster-proxy
118+
```
119+
120+
## Troubleshooting
121+
122+
### User Server Pods Stuck in Pending
123+
124+
**Symptom:** After enabling `enableUserServer=true`, the deployment pods remain in Pending state.
125+
126+
**Solution:** Verify that the required secret exists in the namespace:
127+
```bash
128+
kubectl get secret -n <namespace> cluster-proxy-user-serving-cert
129+
```
130+
131+
If the secret is missing, create it:
132+
```bash
133+
kubectl create secret tls cluster-proxy-user-serving-cert \
134+
--cert=path/to/tls.crt \
135+
--key=path/to/tls.key \
136+
-n <namespace>
137+
```
138+
139+
Note: The `proxy-server-ca` and `proxy-client` secrets are created automatically by the controller and do not need manual creation.
140+
141+
### ImagePullBackOff Errors
142+
143+
**Solution:** Verify the image registry and credentials:
144+
```bash
145+
helm upgrade cluster-proxy ./charts/cluster-proxy \
146+
--set registry=<your-registry> \
147+
--set image=<your-image> \
148+
--set tag=<your-tag>
149+
```
150+
151+
## More Information
152+
153+
For more details about the Cluster Proxy project, visit the [GitHub repository](https://github.com/open-cluster-management-io/cluster-proxy).

0 commit comments

Comments
 (0)