feature: create secret for agent password

Author: filipcirtog

.evergreen-tasks.yml

@@ -585,6 +585,11 @@ tasks:
     commands:
       - func: "e2e_test"
 
+  - name: e2e_sharded_cluster_scram_sha_256_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
   - name: e2e_sharded_cluster_scram_sha_1_user_connectivity
     tags: [ "patch-run" ]
     commands:

.evergreen.yml

@@ -710,6 +710,7 @@ task_groups:
       - e2e_replica_set_scram_x509_ic_manual_certs
       - e2e_sharded_cluster_scram_sha_1_upgrade
       - e2e_sharded_cluster_scram_sha_256_user_connectivity
+      - e2e_sharded_cluster_scram_sha_256_switch_project
       - e2e_sharded_cluster_scram_sha_1_user_connectivity
       - e2e_sharded_cluster_scram_x509_ic_manual_certs
       - e2e_sharded_cluster_external_access

controllers/om/automation_config.go

@@ -1,19 +1,28 @@
 package om
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/spf13/cast"
 	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/oidc"
+	"github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/secret"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/generate"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/maputil"
 )
 
+// The constants for the authentication secret
+const agentAuthenticationSecretSuffix = "-agent-auth-secret"
+const autoPwdSecretKey = "automation-agent-password"
+
 // AutomationConfig maintains the raw map in the Deployment field
 // and constructs structs to make use of go's type safety
 // Dev notes: actually, this object is just a wrapper for the `Deployment` object which is received from Ops Manager,
@@ -429,16 +438,63 @@ func (ac *AutomationConfig) EnsureKeyFileContents() error {
 // EnsurePassword makes sure that there is an Automation Agent password
 // that the agents will use to communicate with the deployments. The password
 // is returned, so it can be provided to the other agents
-func (ac *AutomationConfig) EnsurePassword() (string, error) {
-	if ac.Auth.AutoPwd == "" || ac.Auth.AutoPwd == util.InvalidAutomationAgentPassword {
-		automationAgentBackupPassword, err := generate.KeyFileContents()
+// EnsurePassword makes sure that there is an Automation Agent password
+// that the agents will use to communicate with the deployments. The password
+// is returned, so it can be provided to the other agents.
+func (ac *AutomationConfig) EnsurePassword(k8sClient secret.GetUpdateCreator, ctx context.Context, mdbNamespacedName *types.NamespacedName) (string, error) {
+	secretNamespacedName := client.ObjectKey{Name: mdbNamespacedName.Name + agentAuthenticationSecretSuffix, Namespace: mdbNamespacedName.Namespace}
+
+	var password string
+
+	data, err := secret.ReadStringData(ctx, k8sClient, secretNamespacedName)
+	if err == nil {
+		if val, ok := data[autoPwdSecretKey]; ok && len(val) > 0 {
+			password = val
+		}
+	} else if secret.SecretNotExist(err) {
+		if ac.Auth.AutoPwd != "" && ac.Auth.AutoPwd != util.InvalidAutomationAgentPassword {
+			password = ac.Auth.AutoPwd
+		}
+
+		err := EnsureEmptySecret(ctx, k8sClient, secretNamespacedName)
 		if err != nil {
 			return "", err
 		}
-		ac.Auth.AutoPwd = automationAgentBackupPassword
-		return automationAgentBackupPassword, nil
 	}
-	return ac.Auth.AutoPwd, nil
+
+	if password == "" {
+		generatedPassword, genErr := generate.KeyFileContents()
+		if genErr != nil {
+			return "", genErr
+		}
+		password = generatedPassword
+	}
+
+	ac.Auth.AutoPwd = password
+	err = secret.UpdateField(ctx, k8sClient, secretNamespacedName, autoPwdSecretKey, password)
+	if err != nil {
+		return "", fmt.Errorf("failed to update password field in shared secret %s/%s: %w", secretNamespacedName.Namespace, secretNamespacedName.Name, err)
+	}
+
+	return password, nil
+}
+
+func EnsureEmptySecret(ctx context.Context, k8sClient secret.GetUpdateCreator, secretNamespacedName types.NamespacedName) error {
+	dataFields := map[string]string{
+		autoPwdSecretKey: "",
+	}
+
+	emptySecret := secret.Builder().
+		SetName(secretNamespacedName.Name).
+		SetNamespace(secretNamespacedName.Namespace).
+		SetStringMapToData(dataFields).
+		Build()
+
+	if err := secret.CreateOrUpdateIfNeeded(ctx, k8sClient, emptySecret); err != nil {
+		return fmt.Errorf("failed to create or update empty secret %s/%s: %w", secretNamespacedName.Namespace, secretNamespacedName.Name, err)
+	}
+
+	return nil
 }
 
 func (ac *AutomationConfig) CanEnableX509ProjectAuthentication() (bool, string) {

controllers/operator/appdbreplicaset_controller.go

@@ -1667,7 +1667,7 @@ func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx contex
 		AutoPEMKeyFilePath: agentCertPath,
 		CAFilePath:         util.CAFilePathInContainer,
 	}
-	err = authentication.Configure(conn, opts, false, log)
+	err = authentication.Configure(r.client, ctx, &types.NamespacedName{Namespace: opsManager.Namespace, Name: opsManager.Name}, conn, opts, false, log)
 	if err != nil {
 		log.Errorf("Could not set Automation Authentication options in Ops/Cloud Manager for the Application Database. "+
 			"Application Database is always configured with authentication enabled, but this will not be "+

controllers/operator/authentication/authentication.go

@@ -1,13 +1,17 @@
 package authentication
 
 import (
+	"context"
+
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
+	"k8s.io/apimachinery/pkg/types"
 
 	mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb"
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/oidc"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
@@ -82,7 +86,7 @@ type UserOptions struct {
 
 // Configure will configure all the specified authentication Mechanisms. We need to ensure we wait for
 // the agents to reach ready state after each operation as prematurely updating the automation config can cause the agents to get stuck.
-func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error {
+func Configure(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error {
 	log.Infow("ensuring correct deployment mechanisms", "ProcessNames", opts.ProcessNames, "Mechanisms", opts.Mechanisms)
 
 	// In case we're recovering, we can push all changes at once, because the mechanism is triggered after 20min by default.
@@ -113,7 +117,7 @@ func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.Sug
 
 	// once we have made sure that the deployment authentication mechanism array contains the desired auth mechanism
 	// we can then configure the agent authentication.
-	if err := enableAgentAuthentication(conn, opts, log); err != nil {
+	if err := enableAgentAuthentication(client, ctx, mdbNamespacedName, conn, opts, log); err != nil {
 		return xerrors.Errorf("error enabling agent authentication: %w", err)
 	}
 	if err := waitForReadyStateIfNeeded(); err != nil {
@@ -151,7 +155,7 @@ func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.Sug
 
 // Disable disables all authentication mechanisms, and waits for the agents to reach goal state. It is still required to provide
 // automation agent username, password and keyfile contents to ensure a valid Automation Config.
-func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error {
+func Disable(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error {
 	ac, err := conn.ReadAutomationConfig()
 	if err != nil {
 		return xerrors.Errorf("error reading automation config: %w", err)
@@ -181,7 +185,7 @@ func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.Sugare
 		if err := ac.EnsureKeyFileContents(); err != nil {
 			return xerrors.Errorf("error ensuring keyfile contents: %w", err)
 		}
-		if _, err := ac.EnsurePassword(); err != nil {
+		if _, err := ac.EnsurePassword(client, ctx, mdbNamespacedName); err != nil {
 			return xerrors.Errorf("error ensuring agent password: %w", err)
 		}
 
@@ -258,7 +262,7 @@ func removeUnsupportedAgentMechanisms(conn om.Connection, opts Options, log *zap
 
 // enableAgentAuthentication determines which agent authentication mechanism should be configured
 // and enables it in Ops Manager
-func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error {
+func enableAgentAuthentication(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, log *zap.SugaredLogger) error {
 	ac, err := conn.ReadAutomationConfig()
 	if err != nil {
 		return xerrors.Errorf("error reading automation config: %w", err)
@@ -267,7 +271,7 @@ func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.Sugare
 	// we then configure the agent authentication for that type
 	mechanism := convertToMechanismOrPanic(opts.AgentMechanism, ac)
 
-	if err := ensureAgentAuthenticationIsConfigured(conn, opts, ac, mechanism, log); err != nil {
+	if err := ensureAgentAuthenticationIsConfigured(client, ctx, mdbNamespacedName, conn, opts, ac, mechanism, log); err != nil {
 		return xerrors.Errorf("error ensuring agent authentication is configured: %w", err)
 	}
 
@@ -365,14 +369,14 @@ func addOrRemoveAgentClientCertificate(conn om.Connection, opts Options, log *za
 }
 
 // ensureAgentAuthenticationIsConfigured will configure the agent authentication settings based on the desiredAgentAuthMechanism
-func ensureAgentAuthenticationIsConfigured(conn om.Connection, opts Options, ac *om.AutomationConfig, mechanism Mechanism, log *zap.SugaredLogger) error {
+func ensureAgentAuthenticationIsConfigured(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, ac *om.AutomationConfig, mechanism Mechanism, log *zap.SugaredLogger) error {
 	if mechanism.IsAgentAuthenticationConfigured(ac, opts) {
 		log.Infof("Agent authentication mechanism %s is already configured", mechanism.GetName())
 		return nil
 	}
 
 	log.Infof("Enabling %s agent authentication", mechanism.GetName())
-	return mechanism.EnableAgentAuthentication(conn, opts, log)
+	return mechanism.EnableAgentAuthentication(client, ctx, mdbNamespacedName, conn, opts, log)
 }
 
 // ensureDeploymentMechanisms configures the given AutomationConfig to allow deployments to

controllers/operator/authentication/authentication_mechanism.go

@@ -1,19 +1,22 @@
 package authentication
 
 import (
+	"context"
 	"slices"
 	"strings"
 
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
 // Mechanism is an interface that needs to be implemented for any Ops Manager authentication mechanism
 type Mechanism interface {
-	EnableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error
+	EnableAgentAuthentication(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, log *zap.SugaredLogger) error
 	DisableAgentAuthentication(conn om.Connection, log *zap.SugaredLogger) error
 	EnableDeploymentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error
 	DisableDeploymentAuthentication(conn om.Connection, log *zap.SugaredLogger) error

controllers/operator/authentication/configure_authentication_test.go

@@ -1,13 +1,16 @@
 package authentication
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
+	"github.com/mongodb/mongodb-kubernetes/controllers/operator/mock"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
@@ -17,6 +20,10 @@ func init() {
 }
 
 func TestConfigureScramSha256(t *testing.T) {
+	ctx := context.Background()
+	kubeClient, _ := mock.NewDefaultFakeClient()
+	mdbNamespacedName := &types.NamespacedName{Namespace: "test", Name: "test"}
+
 	dep := om.NewDeployment()
 	conn := om.NewMockedOmConnection(dep)
 
@@ -27,7 +34,7 @@ func TestConfigureScramSha256(t *testing.T) {
 		AgentMechanism:   "SCRAM",
 	}
 
-	if err := Configure(conn, opts, false, zap.S()); err != nil {
+	if err := Configure(kubeClient, ctx, mdbNamespacedName, conn, opts, false, zap.S()); err != nil {
 		t.Fatal(err)
 	}
 
@@ -41,6 +48,10 @@ func TestConfigureScramSha256(t *testing.T) {
 }
 
 func TestConfigureX509(t *testing.T) {
+	ctx := context.Background()
+	kubeClient, _ := mock.NewDefaultFakeClient()
+	mdbNamespacedName := &types.NamespacedName{Namespace: "test", Name: "test"}
+
 	dep := om.NewDeployment()
 	conn := om.NewMockedOmConnection(dep)
 
@@ -55,7 +66,7 @@ func TestConfigureX509(t *testing.T) {
 		},
 	}
 
-	if err := Configure(conn, opts, false, zap.S()); err != nil {
+	if err := Configure(kubeClient, ctx, mdbNamespacedName, conn, opts, false, zap.S()); err != nil {
 		t.Fatal(err)
 	}
 
@@ -69,6 +80,10 @@ func TestConfigureX509(t *testing.T) {
 }
 
 func TestConfigureScramSha1(t *testing.T) {
+	ctx := context.Background()
+	kubeClient, _ := mock.NewDefaultFakeClient()
+	mdbNamespacedName := &types.NamespacedName{Namespace: "test", Name: "test"}
+
 	dep := om.NewDeployment()
 	conn := om.NewMockedOmConnection(dep)
 
@@ -79,7 +94,7 @@ func TestConfigureScramSha1(t *testing.T) {
 		AgentMechanism:   "SCRAM-SHA-1",
 	}
 
-	if err := Configure(conn, opts, false, zap.S()); err != nil {
+	if err := Configure(kubeClient, ctx, mdbNamespacedName, conn, opts, false, zap.S()); err != nil {
 		t.Fatal(err)
 	}
 
@@ -91,6 +106,10 @@ func TestConfigureScramSha1(t *testing.T) {
 }
 
 func TestConfigureMultipleAuthenticationMechanisms(t *testing.T) {
+	ctx := context.Background()
+	kubeClient, _ := mock.NewDefaultFakeClient()
+	mdbNamespacedName := &types.NamespacedName{Namespace: "test", Name: "test"}
+
 	dep := om.NewDeployment()
 	conn := om.NewMockedOmConnection(dep)
 
@@ -104,7 +123,7 @@ func TestConfigureMultipleAuthenticationMechanisms(t *testing.T) {
 		},
 	}
 
-	if err := Configure(conn, opts, false, zap.S()); err != nil {
+	if err := Configure(kubeClient, ctx, mdbNamespacedName, conn, opts, false, zap.S()); err != nil {
 		t.Fatal(err)
 	}
 
@@ -124,6 +143,10 @@ func TestConfigureMultipleAuthenticationMechanisms(t *testing.T) {
 }
 
 func TestDisableAuthentication(t *testing.T) {
+	ctx := context.Background()
+	kubeClient, _ := mock.NewDefaultFakeClient()
+	mdbNamespacedName := &types.NamespacedName{Namespace: "test", Name: "test"}
+
 	dep := om.NewDeployment()
 	conn := om.NewMockedOmConnection(dep)
 
@@ -133,7 +156,7 @@ func TestDisableAuthentication(t *testing.T) {
 		return nil
 	}, zap.S())
 
-	if err := Disable(conn, Options{}, true, zap.S()); err != nil {
+	if err := Disable(kubeClient, ctx, mdbNamespacedName, conn, Options{}, true, zap.S()); err != nil {
 		t.Fatal(err)
 	}
 
@@ -212,7 +235,11 @@ func assertDeploymentMechanismsConfigured(t *testing.T, authMechanism Mechanism,
 }
 
 func assertAgentAuthenticationDisabled(t *testing.T, authMechanism Mechanism, conn om.Connection, opts Options) {
-	err := authMechanism.EnableAgentAuthentication(conn, opts, zap.S())
+	ctx := context.Background()
+	kubeClient, _ := mock.NewDefaultFakeClient()
+	mdbNamespacedName := &types.NamespacedName{Namespace: "test", Name: "test"}
+
+	err := authMechanism.EnableAgentAuthentication(kubeClient, ctx, mdbNamespacedName, conn, opts, zap.S())
 	require.NoError(t, err)
 
 	ac, err := conn.ReadAutomationConfig()

controllers/operator/authentication/ldap.go

@@ -1,10 +1,14 @@
 package authentication
 
 import (
+	"context"
+
 	"go.uber.org/zap"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/stringutil"
 )
@@ -15,7 +19,7 @@ func (l *ldapAuthMechanism) GetName() MechanismName {
 	return LDAPPlain
 }
 
-func (l *ldapAuthMechanism) EnableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error {
+func (l *ldapAuthMechanism) EnableAgentAuthentication(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, log *zap.SugaredLogger) error {
 	log.Info("Configuring LDAP authentication")
 	err := conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error {
 		if err := ac.EnsureKeyFileContents(); err != nil {