Simulator: speed up bundle loading

Tune the client side rate limit parameter to speed up bundle loading time.
- Set QPS and Burst to 100 by default.
- Add flags to allow override the values.

Author: bk201

cmd/simulator.go

@@ -8,14 +8,12 @@ import (
 	"time"
 
 	"github.com/mitchellh/go-homedir"
+	wranglerunstructured "github.com/rancher/wrangler/pkg/unstructured"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
-
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
-	wranglerunstructured "github.com/rancher/wrangler/pkg/unstructured"
-
 	"github.com/rancher/support-bundle-kit/pkg/simulator/apiserver"
 	"github.com/rancher/support-bundle-kit/pkg/simulator/certs"
 	"github.com/rancher/support-bundle-kit/pkg/simulator/etcd"
@@ -24,10 +22,12 @@ import (
 )
 
 var (
-	simHome    string
-	bundlePath string
-	resetHome  bool
-	skipLoad   bool
+	simHome     string
+	bundlePath  string
+	resetHome   bool
+	skipLoad    bool
+	clientQPS   float32
+	clientBurst int
 )
 
 var simulatorCmd = &cobra.Command{
@@ -50,7 +50,7 @@ support bundle contents using native k8s tooling like kubectl`,
 		ctx, cancel := context.WithCancel(context.TODO())
 		defer cancel()
 
-		a := apiserver.APIServerConfig{}
+		a := apiserver.NewAPIServerConfig(clientQPS, clientBurst)
 
 		generatedCerts, err := certs.GenerateCerts([]string{"localhost"}, simHome)
 		if err != nil {
@@ -105,6 +105,7 @@ support bundle contents using native k8s tooling like kubectl`,
 		}
 
 		if !skipLoad {
+			start := time.Now()
 			err = o.CreateUnstructuredClusterObjects()
 
 			if err != nil {
@@ -124,6 +125,8 @@ support bundle contents using native k8s tooling like kubectl`,
 			// ignore the error creation
 			_ = o.CreatedFailedObjectsList()
 			logrus.Info("All resources loaded successfully")
+
+			logrus.Infof("Time to load all objects: %s seconds", time.Since(start))
 		}
 
 		err = eg.Wait()
@@ -145,6 +148,8 @@ func init() {
 	simulatorCmd.PersistentFlags().StringVar(&bundlePath, "bundle-path", ".", "location to support bundle. default is .")
 	simulatorCmd.PersistentFlags().BoolVar(&resetHome, "reset", false, "reset sim-home, will clear the contents and start a clean etcd + apiserver instance")
 	simulatorCmd.PersistentFlags().BoolVar(&skipLoad, "skip-load", false, "skip load / re-load of bundle. this will ensure current etcd contents are only accessible")
+	simulatorCmd.PersistentFlags().Float32Var(&clientQPS, "client-qps", apiserver.DefaultClientQPS, "QPS for the kubernete client to push objects to the api server")
+	simulatorCmd.PersistentFlags().IntVar(&clientBurst, "client-burst", apiserver.DefaultClientBurst, "Burst for the kubernete client to push objects to the api server")
 }
 
 // GetServiceClusterIP will return the service cluster IP from the support bundle

pkg/simulator/apiserver/apiserver.go

@@ -7,32 +7,45 @@ import (
 	"net"
 	"time"
 
-	"github.com/rancher/support-bundle-kit/pkg/simulator/certs"
-	"github.com/rancher/support-bundle-kit/pkg/simulator/etcd"
+	"github.com/sirupsen/logrus"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	kubeconfig "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+
+	"github.com/rancher/support-bundle-kit/pkg/simulator/certs"
+	"github.com/rancher/support-bundle-kit/pkg/simulator/etcd"
 )
 
 const (
 	DefaultServiceClusterIP = "10.53.0.1"
+	DefaultClientQPS        = 100
+	DefaultClientBurst      = 100
 )
 
 type APIServerConfig struct {
 	Certs      *certs.CertInfo
 	Etcd       *etcd.EtcdConfig
 	KubeConfig string
 	Config     *rest.Config
+	QPS        float32
+	Burst      int
 }
 
 const (
 	APIVersionsSupported = "v1=true,api/beta=true,api/alpha=false"
 )
 
+func NewAPIServerConfig(qps float32, burst int) APIServerConfig {
+	return APIServerConfig{
+		QPS:   qps,
+		Burst: burst,
+	}
+}
+
 // RunAPIServer will bootstrap an API server with only core resources enabled
 // No additional controllers will be scheduled
 func (a *APIServerConfig) RunAPIServer(ctx context.Context, serviceClusterIP string) error {
@@ -125,6 +138,10 @@ func (a *APIServerConfig) GenerateKubeConfig(path string) error {
 		return err
 	}
 
+	config.QPS = a.QPS
+	config.Burst = a.Burst
+	logrus.Infof("Client will be configured with QPS: %f, Burst: %d", config.QPS, config.Burst)
+
 	a.Config = config
 
 	return nil

pkg/simulator/apiserver/apiserver_test.go

@@ -2,13 +2,14 @@ package apiserver
 
 import (
 	"context"
-	"github.com/rancher/support-bundle-kit/pkg/simulator/certs"
-	"github.com/rancher/support-bundle-kit/pkg/simulator/etcd"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
+
+	"github.com/rancher/support-bundle-kit/pkg/simulator/certs"
+	"github.com/rancher/support-bundle-kit/pkg/simulator/etcd"
 )
 
 func TestRunAPIServer(t *testing.T) {
@@ -22,7 +23,7 @@ func TestRunAPIServer(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	a := APIServerConfig{}
+	a := NewAPIServerConfig(DefaultClientQPS, DefaultClientBurst)
 
 	generatedCerts, err := certs.GenerateCerts([]string{"localhost", "127.0.0.1"}, dir)
 	if err != nil {

pkg/simulator/crd/crd_test.go

@@ -2,15 +2,17 @@ package crd
 
 import (
 	"context"
-	"github.com/rancher/support-bundle-kit/pkg/simulator/apiserver"
-	"github.com/rancher/support-bundle-kit/pkg/simulator/certs"
-	"github.com/rancher/support-bundle-kit/pkg/simulator/etcd"
-	"golang.org/x/sync/errgroup"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/rancher/support-bundle-kit/pkg/simulator/apiserver"
+	"github.com/rancher/support-bundle-kit/pkg/simulator/certs"
+	"github.com/rancher/support-bundle-kit/pkg/simulator/etcd"
 )
 
 func TestWriteFiles(t *testing.T) {
@@ -36,7 +38,7 @@ func TestInstallCRD(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	a := apiserver.APIServerConfig{}
+	a := apiserver.NewAPIServerConfig(apiserver.DefaultClientQPS, apiserver.DefaultClientBurst)
 
 	generatedCerts, err := certs.GenerateCerts([]string{"localhost", "127.0.0.1"}, dir)
 	if err != nil {

pkg/simulator/objects/apply.go

@@ -3,6 +3,7 @@ package objects
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	wranglerunstructured "github.com/rancher/wrangler/pkg/unstructured"
@@ -54,7 +55,8 @@ var (
 		"PodSecurityPolicy": true,
 	}
 
-	cacheMap = make(map[schema.GroupVersionKind]*meta.RESTMapping)
+	cacheMap  = make(map[schema.GroupVersionKind]*meta.RESTMapping)
+	cacheLock = new(sync.Mutex)
 )
 
 // NewObjectManager is a wrapper around apply and support bundle path
@@ -94,6 +96,11 @@ func (o *ObjectManager) CreateUnstructuredClusterObjects() error {
 		return err
 	}
 
+	// TODO: check all CRDs are created
+	logrus.Info("Sleeping for 5 seconds before applying cluster objects")
+	time.Sleep(5 * time.Second)
+	logrus.Info("Sleeping done")
+
 	progressMgr = NewProgressManager("Step 1/4: Cluster objects")
 	err = o.ApplyObjects(clusterObjs, true, nil, progressMgr.progress)
 
@@ -271,6 +278,8 @@ func objectHousekeeping(obj *unstructured.Unstructured) error {
 // wrapper to lookup GVR for usage with dynamic client
 func findGVR(gvk schema.GroupVersionKind, cfg *rest.Config) (*meta.RESTMapping, error) {
 
+	cacheLock.Lock()
+	defer cacheLock.Unlock()
 	existingMapping, ok := cacheMap[gvk]
 	if ok {
 		return existingMapping, nil

pkg/simulator/objects/progress.go

@@ -25,6 +25,13 @@ func (p *ProgressManager) progress(current, total int) {
 
 	if current == total {
 		fmt.Printf("\n")
-		fmt.Printf("Time to load all objects: %s seconds\n\n", time.Since(p.start))
+		elapsed := time.Since(p.start)
+		elapsedInSecond := int(elapsed.Seconds())
+
+		objsPerSecond := total
+		if elapsedInSecond != 0 {
+			objsPerSecond = total / elapsedInSecond
+		}
+		fmt.Printf("Time to load all objects: %s seconds. (%d objects/s)\n\n", elapsed, objsPerSecond)
 	}
 }