fix sr shared bigip multiple cis instances (#3941)

Author: lavanya-f5

build-tools/Dockerfile.debian

@@ -13,7 +13,7 @@ COPY . .
 
 RUN $REPOPATH/build-tools/rel-build.sh
 
-FROM python:3.10-slim-buster
+FROM python:3.10-slim-bullseye
 
 ENV APPPATH /app
 

docs/RELEASE-NOTES.rst

@@ -14,11 +14,12 @@ Added Functionality
 
 Bug Fixes
 ````````````
+* Issue 3719 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3719>`_: Fix shared static routes override each other with multiple CIS instances writing to Common Partition
 * Issue 3852 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3852>`_: Improve logging when BIG-IP is not reachable during pod initialization
 
 Upgrade notes
 ``````````````
-
+* Upgrading to CIS 2.21, static routes are deleted and recreated with new description added to fix the issue(`Github#3719`) of static routes overriding each other with multiple CIS instances writing to Common Partition. This may cause a brief disruption in traffic while the routes are being recreated.
 2.20.1
 -------------
 

docs/config_examples/StaticRoute/README.md

@@ -72,5 +72,7 @@ In case static routes are not added, along with looking at CIS logs you can also
 
 In case static routes are configured with calico CNI, you can check the logs of CIS to see if the blockaffinities are being read properly. If not, you can check the permissions of the CIS service account to read blockaffinities. You can also check and verify that the blockaffinities are being created properly in the calico CNI.
 
-
+### FAQ
+* **Q: How to configure shared static routes with multiple CIS instances in different clusters using the same BIGIP?**
+  * A: Set `--shared-static-routes=true` and `--local-cluster-name=<clusterName>` in CIS deployment args. This will create static routes in /Common partition and uniquely manage them per cluster without overriding other cluster routes.
 

pkg/controller/node_poll_handler.go

@@ -181,6 +181,39 @@ func ciliumPodCidr(annotation map[string]string) string {
 	return ""
 }
 
+// setCISIdentifierForRoutes sets the CIS identifier for route section based on cluster configuration
+func (ctlr *Controller) setCISIdentifierForRoutes(routes *routeSection) {
+	var routeClusterName string
+	// Use local cluster name to create unique CIS identifier across clusters
+	if ctlr.multiClusterMode == SecondaryCIS {
+		// For secondary CIS, use the HA pair cluster name
+		// so that static routes are not overwritten from the HA pair during failover
+		routeClusterName = ctlr.multiClusterHandler.HAPairClusterName
+	} else {
+		routeClusterName = ctlr.multiClusterHandler.LocalClusterName
+	}
+	var nodeLabelSelector string
+	if clusterConfig, ok := ctlr.multiClusterHandler.ClusterConfigs[routeClusterName]; ok {
+		nodeLabelSelector = clusterConfig.nodeLabelSelector
+	}
+	if routeClusterName != "" {
+		routes.CISIdentifier = strings.TrimPrefix(ctlr.RequestHandler.PrimaryBigIPWorker.getPostManager().BIGIPURL, "https://") + "_" + routeClusterName
+		if nodeLabelSelector != "" {
+			routes.CISIdentifier += "_" + nodeLabelSelector
+		}
+		log.Infof("Using cluster-specific CIS identifier: %s (cluster: %s, nodeLabelSelector: %s)", routes.CISIdentifier, routeClusterName, nodeLabelSelector)
+	} else {
+		if nodeLabelSelector != "" {
+			routes.CISIdentifier = strings.TrimPrefix(ctlr.RequestHandler.PrimaryBigIPWorker.getPostManager().BIGIPURL, "https://")
+			routes.CISIdentifier += "_" + nodeLabelSelector
+		} else {
+			// Don't set CIS identifier when no cluster name or no nodelabelselctor is configured
+			routes.CISIdentifier = ""
+			log.Warningf("Local cluster name not set. Multiple CIS instances across clusters may still cause route conflicts with shared-static-routes writing to same BIGIP instance!")
+		}
+	}
+}
+
 func (ctlr *Controller) processStaticRouteUpdate(
 	nodes []interface{},
 ) {
@@ -193,7 +226,9 @@ func (ctlr *Controller) processStaticRouteUpdate(
 	}
 	log.Debugf("Processing Node Updates for static routes")
 	routes := routeSection{}
-	routes.CISIdentifier = ctlr.Partition + "_" + strings.TrimPrefix(ctlr.RequestHandler.PrimaryBigIPWorker.getPostManager().BIGIPURL, "https://")
+	// Set CIS identifier for routes
+	ctlr.setCISIdentifierForRoutes(&routes)
+
 	nodePodCIDRMap := ctlr.GetNodePodCIDRMap()
 	for _, obj := range nodes {
 		node := obj.(*v1.Node)
@@ -444,7 +479,9 @@ func (ctlr *Controller) processBlockAffinities(clusterName string) {
 	var baListInf []interface{}
 	baListInf = ctlr.getBlockAffinitiesFromAllClusters()
 	routes := routeSection{}
-	routes.CISIdentifier = ctlr.Partition + "_" + strings.TrimPrefix(ctlr.RequestHandler.PrimaryBigIPWorker.getPostManager().BIGIPURL, "https://")
+	// Set CIS identifier for routes
+	ctlr.setCISIdentifierForRoutes(&routes)
+
 	clusterConfig := ctlr.multiClusterHandler.getClusterConfig(clusterName)
 	for _, obj := range baListInf {
 		blockAffinity := obj.(*unstructured.Unstructured)

pkg/controller/node_poll_handler_test.go

@@ -97,7 +97,8 @@ var _ = Describe("Node Poller Handler", func() {
 
 	It("Nodes Update processing", func() {
 		mockCtlr.Partition = "test"
-		cisIdentifier := mockCtlr.Partition + "_127.0.0.1"
+		mockCtlr.multiClusterHandler.LocalClusterName = "localCluster"
+		cisIdentifier := "127.0.0.1_" + mockCtlr.multiClusterHandler.LocalClusterName
 		mockCtlr.setNodeInformer("")
 		mockCtlr.UseNodeInternal = true
 		namespace := "default"

requirements.txt

@@ -1,3 +1,3 @@
--e git+https://github.com/f5devcentral/f5-cccl.git@497c325211de2191afe1ffaef673df07f7bb7c26#egg=f5-cccl
--e git+https://github.com/f5devcentral/f5-ctlr-agent.git@a717f8843f4dced916efda1edeebc8675ef0e675#egg=f5-ctlr-agent
+-e git+https://github.com/f5devcentral/f5-cccl.git@6dfb9005bb7de35652773db26e1239b84611f8ab#egg=f5-cccl
+-e git+https://github.com/f5devcentral/f5-ctlr-agent.git@fee3f82f5948fc45e9bbc980d3d299ae7e5bfa1c#egg=f5-ctlr-agent
 -e git+https://github.com/F5Networks/f5-icontrol-rest-python.git@3fee4a4599e903cce1abfe232e6ffb74d7085b64#egg=f5-icontrol-rest