orchestrator upgrade fix

deploy/charts/app-orch-tenant-controller/Chart.yaml

@@ -5,8 +5,8 @@
 apiVersion: v2
 description: Tenant Controller
 name: app-orch-tenant-controller
-version: 0.3.11
+version: 0.3.12-dev-bcea3f1f
 annotations:
   revision: ""
   created: ""
-appVersion: "0.3.11"
+appVersion: "0.3.12-dev-bcea3f1f"

internal/manager/manager.go

@@ -161,14 +161,15 @@ func (m *Manager) handleProjectEvent(event plugins.Event) error {
 	return err
 }
 
-func (m *Manager) CreateProject(organizationName string, projectName string, projectUUID string, project nexushook.NexusProjectInterface) {
-	log.Debugf("Creating project with organizationName=%s; projectName=%s; projectUUID=%s", organizationName, projectName, projectUUID)
+func (m *Manager) CreateProject(organizationName string, projectName string, projectUUID string, project nexushook.NexusProjectInterface, action string) {
+	log.Debugf("Creating project with organizationName=%s; projectName=%s; projectUUID=%s; action=%s", organizationName, projectName, projectUUID, action)
 	e := plugins.Event{
 		EventType:    "create",
 		Organization: organizationName,
 		Name:         projectName,
 		UUID:         projectUUID,
 		Project:      project,
+		Action:       action,
 	}
 	m.eventChan <- e
 }

internal/nexus/mock-nexus_test.go

@@ -5,6 +5,7 @@
 
 import (
 	"context"
+	"errors"
 	projectActiveWatcherv1 "github.com/open-edge-platform/orch-utils/tenancy-datamodel/build/apis/projectactivewatcher.edge-orchestrator.intel.com/v1"
 	nexus "github.com/open-edge-platform/orch-utils/tenancy-datamodel/build/nexus-client"
 )
@@ -27,7 +28,10 @@
 	return f.parent, nil
 }
 
-type MockNexusProjectActiveWatcher nexus.ProjectactivewatcherProjectActiveWatcher
+type MockNexusProjectActiveWatcher struct {
+	nexus.ProjectactivewatcherProjectActiveWatcher
+	annotations map[string]string
+}
 
 func (w *MockNexusProjectActiveWatcher) Update(ctx context.Context) error {
 	_ = ctx
@@ -38,8 +42,19 @@
 	return &w.Spec
 }
 
+func (w *MockNexusProjectActiveWatcher) GetAnnotations() map[string]string {
+	if w.annotations == nil {
+		w.annotations = make(map[string]string)
+	}
+	return w.annotations
+}
+
+func (w *MockNexusProjectActiveWatcher) SetAnnotations(annotations map[string]string) {
+	w.annotations = annotations
+}
+
 func (w *MockNexusProjectActiveWatcher) DisplayName() string {
-	return (*nexus.ProjectactivewatcherProjectActiveWatcher)(w).DisplayName()
+	return w.Name
 }
 
 type MockNexusProject struct {
@@ -57,9 +72,18 @@
 
 func (p *MockNexusProject) AddActiveWatchers(ctx context.Context, watcher *projectActiveWatcherv1.ProjectActiveWatcher) (NexusProjectActiveWatcherInterface, error) {
 	_ = ctx
-	p.activeWatchers[watcher.Name] = &MockNexusProjectActiveWatcher{ProjectActiveWatcher: watcher}
-
-	return p.activeWatchers[watcher.Name], nil
+
+	// Check if watcher already exists
+	if existingWatcher, exists := p.activeWatchers[watcher.Name]; exists {
+		return existingWatcher, errors.New("already exists")
+	}
+
+	mockWatcher := &MockNexusProjectActiveWatcher{
+		ProjectactivewatcherProjectActiveWatcher: nexus.ProjectactivewatcherProjectActiveWatcher{ProjectActiveWatcher: watcher},
+		annotations: make(map[string]string),
+	}
+	p.activeWatchers[watcher.Name] = mockWatcher
+	return mockWatcher, nil
 }
 
 func (p *MockNexusProject) DeleteActiveWatchers(ctx context.Context, name string) error {

internal/nexus/nexus-hook.go

@@ -5,6 +5,7 @@
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"github.com/open-edge-platform/orch-library/go/dazl"
 	projectActiveWatcherv1 "github.com/open-edge-platform/orch-utils/tenancy-datamodel/build/apis/projectactivewatcher.edge-orchestrator.intel.com/v1"
@@ -35,7 +36,7 @@
 )
 
 type ProjectManager interface {
-	CreateProject(orgName string, projectName string, projectUUID string, project NexusProjectInterface)
+	CreateProject(orgName string, projectName string, projectUUID string, project NexusProjectInterface, action string)
 	DeleteProject(orgName string, projectName string, projectUUID string, project NexusProjectInterface)
 	ManifestTag() string
 }
@@ -189,7 +190,11 @@
 	}
 	if watcherObj != nil {
 		log.Debug("Setting watcher annotations")
-		annotations := make(map[string]string)
+		// Preserve existing annotations and update only the manifest tag
+		annotations := watcherObj.GetAnnotations()
+		if annotations == nil {
+			annotations = make(map[string]string)
+		}
 		annotations[ManifestTagAnnotationKey] = h.dispatcher.ManifestTag()
 		watcherObj.SetAnnotations(annotations)
 		return watcherObj.Update(context.Background())
@@ -318,27 +323,73 @@
 	})
 
 	if err != nil {
-		log.Errorf("Failed to create ProjectActiveWatcher object with an error: %v", err)
-		return err
+		// Handle the case where watcher already exists (upgrade scenario)
+		if strings.Contains(err.Error(), "already exists") {
+			log.Infof("ActiveWatcher already exists for project %s, retrieving existing watcher", project.DisplayName())
+			watcherObj, err = project.GetActiveWatchers(ctx, appName)
+			if err != nil {
+				log.Errorf("Failed to get existing ProjectActiveWatcher object with an error: %v", err)
+				return err
+			}
+			if watcherObj == nil {
+				log.Errorf("Existing ProjectActiveWatcher object is nil for project %s", project.DisplayName())
+				return errors.New("existing watcher is nil")
+			}
+		} else {
+			log.Errorf("Failed to create ProjectActiveWatcher object with an error: %v", err)
+			return err
+		}
 	}
 
 	var action string
 
+	// Always check manifest tag, regardless of watcher status
+	// This ensures we handle upgrade scenarios where existing watchers lack manifest tags
+	annotations := watcherObj.GetAnnotations()
+	currentTag := ""
+	if annotations != nil {
+		currentTag = annotations[ManifestTagAnnotationKey]
+	}
+	expectedTag := h.dispatcher.ManifestTag()
+
+	// Check if watcher is already properly provisioned and has correct manifest tag
+	if watcherObj.GetSpec().StatusIndicator == projectActiveWatcherv1.StatusIndicationIdle && 
+		watcherObj.GetSpec().Message == "Created" && 
+		currentTag == expectedTag {
+		// Everything is correct, no need to reprocess
+		log.Infof("Watch %s for project %s already provisioned with correct manifest tag %s", 
+			watcherObj.DisplayName(), project.DisplayName(), currentTag)
+		return nil
+	}
+
+	// If we reach here, we need to process/update the project
 	if watcherObj.GetSpec().StatusIndicator == projectActiveWatcherv1.StatusIndicationIdle && watcherObj.GetSpec().Message == "Created" {
-		// This is a rerun of an event we already processed - check for update
-		log.Infof("Watch %s for project %s already provisioned", watcherObj.DisplayName(), project.DisplayName())
-		log.Debugf("existing watcher annotations are: %+v", watcherObj.GetAnnotations())
-		annotations := watcherObj.GetAnnotations()
-		if annotations[ManifestTagAnnotationKey] == h.dispatcher.ManifestTag() {
-			// Manifest tag is correct
-			log.Infof("Manifest tag is correct, no need to update")
-			return nil
-		}
-		log.Infof("Manifest tag is not correct, updating. Have %s, want %s", annotations[ManifestTagAnnotationKey], h.dispatcher.ManifestTag())
+		log.Infof("Watch %s for project %s already provisioned but manifest tag needs update. Have '%s', want '%s'", 
+			watcherObj.DisplayName(), project.DisplayName(), currentTag, expectedTag)
+		action = "update"
+	} else if currentTag != expectedTag {
+		log.Infof("Manifest tag missing or incorrect for project %s. Have '%s', want '%s'. Current status: %s", 
+			project.DisplayName(), currentTag, expectedTag, watcherObj.GetSpec().StatusIndicator)
 		action = "update"
 	} else {
 		action = "created"
 	}
+
+	// Always ensure manifest tag is set correctly before processing
+	if currentTag != expectedTag {
+		log.Infof("Setting manifest tag for project %s from '%s' to '%s'", project.DisplayName(), currentTag, expectedTag)
+		if annotations == nil {
+			annotations = make(map[string]string)
+		}
+		annotations[ManifestTagAnnotationKey] = expectedTag
+		watcherObj.SetAnnotations(annotations)
+		err = watcherObj.Update(context.Background())
+		if err != nil {
+			log.Errorf("Failed to update manifest tag annotation: %v", err)
+			return err
+		}
+		log.Infof("Successfully updated manifest tag for project %s to %s", project.DisplayName(), expectedTag)
+	}
 
 	if nexus.IsAlreadyExists(err) {
 		log.Infof("Watch %s already exists for project %s", watcherObj.DisplayName(), project.DisplayName())
@@ -354,7 +405,7 @@
 		// If there is an error, validateArgs() will also set the watcher status appropriately.
 		return err
 	}
-	h.dispatcher.CreateProject(organizationName, project.DisplayName(), project.GetUID(), project)
+	h.dispatcher.CreateProject(organizationName, project.DisplayName(), project.GetUID(), project, action)
 
 	log.Infof("Active watcher %s %s created for Project %s", watcherObj.DisplayName(), action, project.DisplayName())
 

internal/nexus/nexus-hook_test.go

@@ -16,10 +16,11 @@ type MockProjectManager struct {
 	created []string
 }
 
-func (m *MockProjectManager) CreateProject(orgName string, projectName string, projectUUID string, project NexusProjectInterface) {
+func (m *MockProjectManager) CreateProject(orgName string, projectName string, projectUUID string, project NexusProjectInterface, action string) {
 	_ = orgName
 	_ = projectUUID
 	_ = project
+	_ = action
 	m.created = append(m.created, projectName)
 }
 
@@ -31,7 +32,7 @@ func (m *MockProjectManager) DeleteProject(orgName string, projectName string, p
 }
 
 func (m *MockProjectManager) ManifestTag() string {
-	return ""
+	return "v1.3.5"
 }
 
 type NexusHookTestSuite struct {

internal/nexus/upgrade_scenario_test.go

@@ -0,0 +1,135 @@
+// SPDX-FileCopyrightText: (C) 2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+package nexus
+
+import (
+	"testing"
+
+	projectActiveWatcherv1 "github.com/open-edge-platform/orch-utils/tenancy-datamodel/build/apis/projectactivewatcher.edge-orchestrator.intel.com/v1"
+	"github.com/stretchr/testify/suite"
+)
+
+type UpgradeScenarioTestSuite struct {
+	suite.Suite
+}
+
+func (s *UpgradeScenarioTestSuite) TestUpgradeScenarioWithoutManifestTag() {
+	// Test simulates upgrade scenario where existing watcher lacks manifest-tag
+	m := &MockProjectManager{}
+	h := NewNexusHook(m)
+
+	project := NewMockNexusProject("test-project-upgrade", "test-uid-123")
+
+	// First, create the project normally (simulating pre-upgrade state)
+	err := h.projectCreated(project)
+	s.NoError(err, "Expected no error when creating project initially")
+
+	// Now simulate upgrade by removing manifest-tag from the watcher 
+	// (this simulates pre-upgrade watcher without manifest-tag)
+	watcher := project.activeWatchers["config-provisioner"]
+	s.NotNil(watcher, "Expected active watcher to exist")
+
+	// Remove the manifest-tag to simulate pre-upgrade state
+	annotations := watcher.GetAnnotations()
+	delete(annotations, ManifestTagAnnotationKey)
+
+	// Now call projectCreated again which should handle the upgrade scenario
+	err = h.projectCreated(project) 
+	s.NoError(err, "Expected no error when handling upgrade scenario")
+}
+
+func (s *UpgradeScenarioTestSuite) TestUpgradeScenarioWithIncorrectManifestTag() {
+	// Test simulates upgrade scenario where existing watcher has wrong manifest-tag
+	m := &MockProjectManager{}
+	h := NewNexusHook(m)
+
+	project := NewMockNexusProject("test-project-upgrade", "test-uid-456")
+
+	// First, create the project normally
+	err := h.projectCreated(project)
+	s.NoError(err, "Expected no error when creating project initially")
+
+	// Now simulate upgrade by changing manifest-tag to an old value
+	watcher := project.activeWatchers["config-provisioner"]
+	s.NotNil(watcher, "Expected active watcher to exist")
+
+	// Set an old manifest-tag to simulate upgrade from older version
+	annotations := watcher.GetAnnotations()
+	annotations[ManifestTagAnnotationKey] = "v1.2.0" // Old tag, different from current v1.3.5
+
+	// Now call projectCreated again which should handle the upgrade scenario
+	err = h.projectCreated(project) 
+	s.NoError(err, "Expected no error when handling upgrade scenario with incorrect manifest tag")
+}
+
+func (s *UpgradeScenarioTestSuite) TestCorrectManifestTagNoUpdate() {
+	// Test simulates scenario where manifest tag is already correct (no upgrade needed)
+	m := &MockProjectManager{}
+	h := NewNexusHook(m)
+
+	project := NewMockNexusProject("correct-tag-project", "correct-uid-789")
+
+	// First, create the project normally
+	err := h.projectCreated(project)
+	s.NoError(err, "Expected no error when creating project initially")
+
+	// Simulate the project creation completing successfully by updating watcher status
+	watcher := project.activeWatchers["config-provisioner"]
+	s.NotNil(watcher, "Expected active watcher to exist")
+
+	// Simulate successful completion - this would normally be done by the ProjectManager
+	annotations := watcher.GetAnnotations()
+	annotations[ManifestTagAnnotationKey] = "v1.3.5" // Set correct manifest tag
+	watcher.GetSpec().StatusIndicator = projectActiveWatcherv1.StatusIndicationIdle
+	watcher.GetSpec().Message = "Created"
+
+	initialCreatedCount := len(m.created)
+
+	// Now call projectCreated again - since manifest tag is already correct, no update should occur
+	err = h.projectCreated(project)
+	s.NoError(err, "Expected no error when manifest tag is already correct")
+
+	// Verify that the manifest tag remains correct
+	annotations = watcher.GetAnnotations()
+	s.Contains(annotations, ManifestTagAnnotationKey, "Expected manifest tag to be present")
+	s.Equal("v1.3.5", annotations[ManifestTagAnnotationKey], "Expected manifest tag to remain unchanged")
+
+	// Verify that CreateProject was NOT called again since no update was needed
+	s.Equal(initialCreatedCount, len(m.created), "Expected no additional project creation when manifest tag is correct")
+}
+
+func (s *UpgradeScenarioTestSuite) TestSameManifestTagNoProcessing() {
+	// Test simulates scenario where manifest tag hasn't changed during upgrade
+	m := &MockProjectManager{}
+	h := NewNexusHook(m)
+
+	project := NewMockNexusProject("same-tag-project", "same-tag-uid-999")
+
+	// First, create the project normally
+	err := h.projectCreated(project)
+	s.NoError(err, "Expected no error when creating project initially")
+
+	// Simulate successful completion with correct manifest tag
+	watcher := project.activeWatchers["config-provisioner"]
+	s.NotNil(watcher, "Expected active watcher to exist")
+
+	annotations := watcher.GetAnnotations()
+	annotations[ManifestTagAnnotationKey] = "v1.3.5" // Same as MockProjectManager returns
+	watcher.GetSpec().StatusIndicator = projectActiveWatcherv1.StatusIndicationIdle
+	watcher.GetSpec().Message = "Created"
+
+	initialCreatedCount := len(m.created)
+
+	// Simulate "upgrade" where manifest tag hasn't actually changed
+	// This should result in early exit with no processing
+	err = h.projectCreated(project)
+	s.NoError(err, "Expected no error when manifest tag is unchanged")
+
+	// Verify that NO additional processing occurred
+	s.Equal(initialCreatedCount, len(m.created), "Expected no project creation when manifest tag unchanged")
+}
+
+func TestUpgradeScenario(t *testing.T) {
+	suite.Run(t, &UpgradeScenarioTestSuite{})
+}