CLOUDP-328217: Automation agent password secret (#566)

# Summary

If a deployment is moved to a different project, the automation agent
password will be re-generated, triggering a password change in the
automation plan.

This will cause a deadlock in a sharded cluster due to the multiple
components requiring automation. However, it will not cause issues in
replicasets.

This is a blocker for migrating projects in sharded deployments.

## Proof of Work

For SCRAM (the only auth mechanism who re-generates a pwd), we now save
the automation agent's password in a secret. During migration, the
stored secret is utilized to preserve the password ,ensuring project
migration possible.

## Observed problems
For LDAP (Sharded + Replica), the following tests are failing, even
though the only modification made is updating the MongoDB resource's
project reference, with no other changes applied. To help further
investigation, I have commented out certain code in the tests (which can
make them fail) so the issue can be consistently reproduced. While the
deployment returns to the "running" state, the users are missing from
the automation configuration.

Additionally, certain flaky parts in the SHA1 and SHA256 tests, as well
as the LDAP and X509 project switch tests, have been commented
out/disabled due to the lack of support for project migrations. These
tests should be activated once project migration functionality has been
implemented.

## Checklist

- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [x] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: filipcirtog

.evergreen-tasks.yml

@@ -690,6 +690,46 @@ tasks:
     commands:
       - func: "e2e_test"
 
+  - name: e2e_sharded_cluster_scram_sha_256_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_sharded_cluster_scram_sha_1_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_sharded_cluster_x509_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_replica_set_scram_sha_256_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_replica_set_scram_sha_1_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_replica_set_x509_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_replica_set_ldap_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_sharded_cluster_ldap_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
   # TODO: not used in any variant
   - name: e2e_replica_set_scram_x509_internal_cluster
     tags: [ "patch-run" ]

.evergreen.yml

@@ -745,6 +745,15 @@ task_groups:
       - e2e_sharded_cluster_scram_sha_1_user_connectivity
       - e2e_sharded_cluster_scram_x509_ic_manual_certs
       - e2e_sharded_cluster_external_access
+      - e2e_replica_set_scram_sha_256_switch_project
+      - e2e_sharded_cluster_scram_sha_256_switch_project
+      - e2e_replica_set_scram_sha_1_switch_project
+      - e2e_sharded_cluster_scram_sha_1_switch_project
+      # TODO CLOUDP-349093 - Disabled these tests as they don't use the password secret, and project migrations aren't fully supported yet.  
+      # e2e_sharded_cluster_x509_switch_project
+      # e2e_replica_set_x509_switch_project
+      # e2e_replica_set_ldap_switch_project
+      # e2e_sharded_cluster_ldap_switch_project
       # e2e_auth_transitions_task_group
       - e2e_replica_set_scram_sha_and_x509
       - e2e_replica_set_x509_to_scram_transition

changelog/20251127_fix_backed_up_the_agent_password_in_a_secret_for_scram.md

@@ -0,0 +1,6 @@
+---
+kind: fix
+date: 2025-11-27
+---
+
+* Backed up the agent password in a secret for SCRAM authentication to prevent unnecessary password rotations.

controllers/om/automation_config.go

@@ -1,14 +1,20 @@
 package om
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/spf13/cast"
 	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/oidc"
+	"github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/secret"
+	"github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/util/constants"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/generate"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/maputil"
@@ -426,19 +432,57 @@ func (ac *AutomationConfig) EnsureKeyFileContents() error {
 	return nil
 }
 
+// AuthSecretName for a given mdbName (`mdbName`) returns the name of
+// the secret associated with it.
+func AuthSecretName(mdbName string) string {
+	return fmt.Sprintf("%s-agent-auth-secret", mdbName)
+}
+
 // EnsurePassword makes sure that there is an Automation Agent password
 // that the agents will use to communicate with the deployments. The password
 // is returned, so it can be provided to the other agents
-func (ac *AutomationConfig) EnsurePassword() (string, error) {
-	if ac.Auth.AutoPwd == "" || ac.Auth.AutoPwd == util.InvalidAutomationAgentPassword {
-		automationAgentBackupPassword, err := generate.KeyFileContents()
-		if err != nil {
-			return "", err
+// EnsurePassword makes sure that there is an Automation Agent password
+// that the agents will use to communicate with the deployments. The password
+// is returned, so it can be provided to the other agents.
+func (ac *AutomationConfig) EnsurePassword(ctx context.Context, k8sClient secret.GetUpdateCreator, mdbNamespacedName types.NamespacedName) (string, error) {
+	secretName := AuthSecretName(mdbNamespacedName.Name)
+	secretNamespacedName := client.ObjectKey{Name: secretName, Namespace: mdbNamespacedName.Namespace}
+	var password string
+
+	data, err := secret.ReadStringData(ctx, k8sClient, secretNamespacedName)
+	if err == nil {
+		if val, ok := data[constants.AutomationAgentAuthSecretKey]; ok && len(val) > 0 {
+			password = val
+		}
+	} else if secret.SecretNotExist(err) {
+		if ac.Auth.AutoPwd != "" && ac.Auth.AutoPwd != util.InvalidAutomationAgentPassword {
+			password = ac.Auth.AutoPwd
+		}
+	}
+
+	if password == "" {
+		generatedPassword, genErr := generate.KeyFileContents()
+		if genErr != nil {
+			return "", genErr
 		}
-		ac.Auth.AutoPwd = automationAgentBackupPassword
-		return automationAgentBackupPassword, nil
+		password = generatedPassword
+	}
+
+	dataFields := map[string]string{
+		constants.AutomationAgentAuthSecretKey: password,
+	}
+
+	passwordSecret := secret.Builder().
+		SetName(secretNamespacedName.Name).
+		SetNamespace(secretNamespacedName.Namespace).
+		SetStringMapToData(dataFields).
+		Build()
+
+	if err := secret.CreateOrUpdateIfNeeded(ctx, k8sClient, passwordSecret); err != nil {
+		return "", fmt.Errorf("failed to update password field in shared secret %s/%s: %w", secretNamespacedName.Namespace, secretNamespacedName.Name, err)
 	}
-	return ac.Auth.AutoPwd, nil
+	ac.Auth.AutoPwd = password
+	return password, nil
 }
 
 func (ac *AutomationConfig) CanEnableX509ProjectAuthentication() (bool, string) {

controllers/operator/appdbreplicaset_controller.go

@@ -1666,8 +1666,9 @@ func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx contex
 		AutoUser:           util.AutomationAgentUserName,
 		AutoPEMKeyFilePath: agentCertPath,
 		CAFilePath:         util.CAFilePathInContainer,
+		MongoDBResource:    types.NamespacedName{Namespace: opsManager.Namespace, Name: opsManager.Name},
 	}
-	err = authentication.Configure(conn, opts, false, log)
+	err = authentication.Configure(ctx, r.client, conn, opts, false, log)
 	if err != nil {
 		log.Errorf("Could not set Automation Authentication options in Ops/Cloud Manager for the Application Database. "+
 			"Application Database is always configured with authentication enabled, but this will not be "+

controllers/operator/authentication/authentication.go

@@ -1,13 +1,17 @@
 package authentication
 
 import (
+	"context"
+
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
+	"k8s.io/apimachinery/pkg/types"
 
 	mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb"
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/oidc"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
@@ -63,6 +67,8 @@ type Options struct {
 	AutoPwd string
 
 	AutoLdapGroupDN string
+
+	MongoDBResource types.NamespacedName
 }
 
 func Redact(o Options) Options {
@@ -82,7 +88,7 @@ type UserOptions struct {
 
 // Configure will configure all the specified authentication Mechanisms. We need to ensure we wait for
 // the agents to reach ready state after each operation as prematurely updating the automation config can cause the agents to get stuck.
-func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error {
+func Configure(ctx context.Context, client kubernetesClient.Client, conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error {
 	log.Infow("ensuring correct deployment mechanisms", "ProcessNames", opts.ProcessNames, "Mechanisms", opts.Mechanisms)
 
 	// In case we're recovering, we can push all changes at once, because the mechanism is triggered after 20min by default.
@@ -113,7 +119,7 @@ func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.Sug
 
 	// once we have made sure that the deployment authentication mechanism array contains the desired auth mechanism
 	// we can then configure the agent authentication.
-	if err := enableAgentAuthentication(conn, opts, log); err != nil {
+	if err := enableAgentAuthentication(ctx, client, conn, opts, log); err != nil {
 		return xerrors.Errorf("error enabling agent authentication: %w", err)
 	}
 	if err := waitForReadyStateIfNeeded(); err != nil {
@@ -151,7 +157,7 @@ func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.Sug
 
 // Disable disables all authentication mechanisms, and waits for the agents to reach goal state. It is still required to provide
 // automation agent username, password and keyfile contents to ensure a valid Automation Config.
-func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error {
+func Disable(ctx context.Context, client kubernetesClient.Client, conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error {
 	ac, err := conn.ReadAutomationConfig()
 	if err != nil {
 		return xerrors.Errorf("error reading automation config: %w", err)
@@ -181,7 +187,7 @@ func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.Sugare
 		if err := ac.EnsureKeyFileContents(); err != nil {
 			return xerrors.Errorf("error ensuring keyfile contents: %w", err)
 		}
-		if _, err := ac.EnsurePassword(); err != nil {
+		if _, err := ac.EnsurePassword(ctx, client, opts.MongoDBResource); err != nil {
 			return xerrors.Errorf("error ensuring agent password: %w", err)
 		}
 
@@ -258,7 +264,7 @@ func removeUnsupportedAgentMechanisms(conn om.Connection, opts Options, log *zap
 
 // enableAgentAuthentication determines which agent authentication mechanism should be configured
 // and enables it in Ops Manager
-func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error {
+func enableAgentAuthentication(ctx context.Context, client kubernetesClient.Client, conn om.Connection, opts Options, log *zap.SugaredLogger) error {
 	ac, err := conn.ReadAutomationConfig()
 	if err != nil {
 		return xerrors.Errorf("error reading automation config: %w", err)
@@ -267,7 +273,7 @@ func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.Sugare
 	// we then configure the agent authentication for that type
 	mechanism := convertToMechanismOrPanic(opts.AgentMechanism, ac)
 
-	if err := ensureAgentAuthenticationIsConfigured(conn, opts, ac, mechanism, log); err != nil {
+	if err := ensureAgentAuthenticationIsConfigured(ctx, client, conn, opts, ac, mechanism, log); err != nil {
 		return xerrors.Errorf("error ensuring agent authentication is configured: %w", err)
 	}
 
@@ -365,14 +371,14 @@ func addOrRemoveAgentClientCertificate(conn om.Connection, opts Options, log *za
 }
 
 // ensureAgentAuthenticationIsConfigured will configure the agent authentication settings based on the desiredAgentAuthMechanism
-func ensureAgentAuthenticationIsConfigured(conn om.Connection, opts Options, ac *om.AutomationConfig, mechanism Mechanism, log *zap.SugaredLogger) error {
+func ensureAgentAuthenticationIsConfigured(ctx context.Context, client kubernetesClient.Client, conn om.Connection, opts Options, ac *om.AutomationConfig, mechanism Mechanism, log *zap.SugaredLogger) error {
 	if mechanism.IsAgentAuthenticationConfigured(ac, opts) {
 		log.Infof("Agent authentication mechanism %s is already configured", mechanism.GetName())
 		return nil
 	}
 
 	log.Infof("Enabling %s agent authentication", mechanism.GetName())
-	return mechanism.EnableAgentAuthentication(conn, opts, log)
+	return mechanism.EnableAgentAuthentication(ctx, client, conn, opts, log)
 }
 
 // ensureDeploymentMechanisms configures the given AutomationConfig to allow deployments to

controllers/operator/authentication/authentication_mechanism.go

@@ -1,19 +1,21 @@
 package authentication
 
 import (
+	"context"
 	"slices"
 	"strings"
 
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
 // Mechanism is an interface that needs to be implemented for any Ops Manager authentication mechanism
 type Mechanism interface {
-	EnableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error
+	EnableAgentAuthentication(ctx context.Context, client kubernetesClient.Client, conn om.Connection, opts Options, log *zap.SugaredLogger) error
 	DisableAgentAuthentication(conn om.Connection, log *zap.SugaredLogger) error
 	EnableDeploymentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error
 	DisableDeploymentAuthentication(conn om.Connection, log *zap.SugaredLogger) error