V16 QA Implement new release tag for E2E tests (#19697)

* Removed this as these tests are covered in other files

* Added release tags

* Make all tests with @smoke and @Release tags run in the pipeline

* Updated npx command

* Updated npx command

* Updated npx command

* Fixed failed tests related to document type folder

* Cleaned up

* Used grep in yaml file instead of package.json file

* Updated yml file

* Updated testCommand

* Fixed command

* Added releaseTest command

* Added another job to run regression test in the release build

* Fixed comments

* Updated name of test job

* Make all release tests run in the pipeline

* Updated warning message

* Reverted npm command

Author: nhudinh0309

build/azure-pipelines.yml

@@ -513,9 +513,9 @@ stages:
       UMBRACO__CMS__WEBROUTING__UMBRACOAPPLICATIONURL: https://localhost:44331/
       ASPNETCORE_URLS: https://localhost:44331
     jobs:
-      # E2E Tests
+      # E2E Smoke Tests
       - job:
-        displayName: E2E Tests (SQLite)
+        displayName: E2E Smoke Tests (SQLite)
         # currently disabled due to DB locks randomly occuring.
         condition: eq(${{parameters.sqliteAcceptanceTests}}, True)
         variables:
@@ -678,7 +678,7 @@ stages:
               testRunTitle: "$(Agent.JobName)"
 
       - job:
-        displayName: E2E Tests (SQL Server)
+        displayName: E2E Smoke Tests (SQL Server)
         variables:
           # Connection string
           CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True
@@ -862,6 +862,176 @@ stages:
               searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
               testRunTitle: "$(Agent.JobName)"
 
+      - job:
+        displayName: E2E Release Tests (SQL Server)
+        variables:
+          # Connection string
+          CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True
+          CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
+        condition: eq(dependencies.Build.outputs['A.build.NBGV_PublicRelease'], 'True')
+        strategy:
+          matrix:
+            WindowsPart1Of3:
+              vmImage: "windows-latest"
+              testCommand: "npm run releaseTest -- --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npm run releaseTest -- --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npm run releaseTest -- --shard=3/3"
+        pool:
+          vmImage: $(vmImage)
+        steps:
+          # Setup test environment
+          - task: DownloadPipelineArtifact@2
+            displayName: Download NuGet artifacts
+            inputs:
+              artifact: nupkg
+              path: $(Agent.BuildDirectory)/app/nupkg
+
+          - task: NodeTool@0
+            displayName: Use Node.js $(nodeVersion)
+            inputs:
+              versionSpec: $(nodeVersion)
+
+          - task: UseDotNet@2
+            displayName: Use .NET SDK from global.json
+            inputs:
+              useGlobalJson: true
+
+          - pwsh: |
+              "UMBRACO_USER_LOGIN=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL)
+              UMBRACO_USER_PASSWORD=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+              URL=$(ASPNETCORE_URLS)
+              STORAGE_STAGE_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/playwright/.auth/user.json
+              CONSOLE_ERRORS_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/console-errors.json" | Out-File .env
+            displayName: Generate .env
+            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
+
+          # Cache and restore NPM packages
+          - task: Cache@2
+            displayName: Cache NPM packages
+            inputs:
+              key: 'npm_e2e | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json'
+              restoreKeys: |
+                npm_e2e | "$(Agent.OS)"
+                npm_e2e
+              path: $(npm_config_cache)
+
+          - script: npm ci --no-fund --no-audit --prefer-offline
+            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
+            displayName: Restore NPM packages
+
+          # Build application
+          - pwsh: |
+              $cmsVersion = "$(Build.BuildNumber)" -replace "\+",".g"
+              dotnet new nugetconfig
+              dotnet nuget add source ./nupkg --name Local
+              dotnet new install Umbraco.Templates::$cmsVersion
+              dotnet new umbraco --name UmbracoProject --version $cmsVersion --exclude-gitignore --no-restore --no-update-check
+              dotnet restore UmbracoProject
+              cp $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest.UmbracoProject/*.cs UmbracoProject
+              dotnet build UmbracoProject --configuration $(buildConfiguration) --no-restore
+              dotnet dev-certs https
+            displayName: Build application
+            workingDirectory: $(Agent.BuildDirectory)/app
+
+          # Start SQL Server
+          - powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest
+            displayName: Start SQL Server Docker image (Linux)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
+
+          - pwsh: SqlLocalDB start MSSQLLocalDB
+            displayName: Start SQL Server LocalDB (Windows)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+
+          # Run application
+          - bash: |
+              nohup dotnet run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1 &
+              echo "##vso[task.setvariable variable=AcceptanceTestProcessId]$!"
+            displayName: Run application (Linux)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
+            workingDirectory: $(Agent.BuildDirectory)/app
+
+          - pwsh: |
+              $process = Start-Process dotnet "run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -NoNewWindow -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log
+              Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)"
+            displayName: Run application (Windows)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+            workingDirectory: $(Agent.BuildDirectory)/app
+
+          # Wait for application to start responding to requests
+          - pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS)
+            displayName: Wait for application
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+
+          # Install Playwright and dependencies
+          - pwsh: npx playwright install chromium
+            displayName: Install Playwright only with Chromium browser
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+
+          # Test
+          - pwsh: $(testCommand)
+            displayName: Run Playwright tests
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+            env:
+              CI: true
+              CommitId: $(Build.SourceVersion)
+              AgentOs: $(Agent.OS)
+
+          # Stop application
+          - bash: kill -15 $(AcceptanceTestProcessId)
+            displayName: Stop application (Linux)
+            condition: and(ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Linux'))
+
+          - pwsh: Stop-Process -Id $(AcceptanceTestProcessId)
+            displayName: Stop application (Windows)
+            condition: and(ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Windows_NT'))
+
+          # Stop SQL Server
+          - pwsh: docker stop mssql
+            displayName: Stop SQL Server Docker image (Linux)
+            condition: eq(variables['Agent.OS'], 'Linux')
+
+          - pwsh: SqlLocalDB stop MSSQLLocalDB
+            displayName: Stop SQL Server LocalDB (Windows)
+            condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+          # Copy artifacts
+          - pwsh: |
+              if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
+                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
+              }
+            displayName: Copy Playwright results
+            condition: succeededOrFailed()
+
+          # Copy console error log
+          - pwsh: |
+              if (Test-Path tests/Umbraco.Tests.AcceptanceTest/console-errors.json) {
+                  Copy-Item tests/Umbraco.Tests.AcceptanceTest/console-errors.json $(Build.ArtifactStagingDirectory)
+              }
+            displayName: Copy console error log
+            condition: succeededOrFailed()
+
+          # Publish test artifacts
+          - task: PublishPipelineArtifact@1
+            displayName: Publish test artifacts
+            condition: succeededOrFailed()
+            inputs:
+              targetPath: $(Build.ArtifactStagingDirectory)
+              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
+
+          # Publish test results
+          - task: PublishTestResults@2
+            displayName: "Publish test results"
+            condition: succeededOrFailed()
+            inputs:
+              testResultsFormat: 'JUnit'
+              testResultsFiles: '*.xml'
+              searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
+              testRunTitle: "$(Agent.JobName)"
+
   ###############################################
   ## Release
   ###############################################
@@ -1067,4 +1237,4 @@ stages:
               storage: umbracoapidocs
               ContainerName: "$web"
               BlobPrefix: v$(umbracoMajorVersion)/ui-api
-              CleanTargetBeforeCopy: true
+              CleanTargetBeforeCopy: true

tests/Umbraco.Tests.AcceptanceTest/package.json

@@ -10,7 +10,8 @@
     "all": "npx playwright test",
     "createTest": "node createTest.js",
     "smokeTest": "npx playwright test DefaultConfig --grep \"@smoke\"",
-    "smokeTestSqlite": "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\""
+    "smokeTestSqlite": "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\"",
+    "releaseTest": "npx playwright test DefaultConfig --grep \"@release\""
   },
   "devDependencies": {
     "@playwright/test": "^1.50",
@@ -26,4 +27,4 @@
     "dotenv": "^16.3.1",
     "node-fetch": "^2.6.7"
   }
-}
+}

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithApprovedColor.spec.ts

@@ -58,7 +58,7 @@ test('can publish content with the approved color data type', async ({umbracoApi
   expect(contentData.values).toEqual([]);
 });
 
-test('can create content with the custom approved color data type', async ({umbracoApi, umbracoUi}) => {
+test('can create content with the custom approved color data type', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const customDataTypeName = 'CustomApprovedColor';
   const colorValue = 'd73737';

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithCheckboxList.spec.ts

@@ -80,7 +80,7 @@ test('can create content with the custom checkbox list data type', async ({umbra
   expect(contentData.values[0].value).toEqual([optionValues[0]]);
 });
 
-test('can not publish a mandatory checkbox list with an empty value', async ({umbracoApi, umbracoUi}) => {
+test('can not publish a mandatory checkbox list with an empty value', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const optionValues = ['testOption1', 'testOption2'];
   const customDataTypeId = await umbracoApi.dataType.createCheckboxListDataType(customDataTypeName, optionValues);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithCustomDataType.spec.ts

@@ -265,7 +265,7 @@ test('can create content with the custom data type with slider property editor',
   expect(contentData.values).toEqual([]);
 });
 
-test('can change slider value in the content section', async ({umbracoApi, umbracoUi}) => {
+test('can change slider value in the content section', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   customDataTypeName = 'Slider';
   const sliderValue = 10;

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithDropdown.spec.ts

@@ -83,7 +83,7 @@ for (const dataTypeName of dataTypeNames) {
   });
 }
 
-test('can not publish a mandatory dropdown with an empty value', async ({umbracoApi, umbracoUi}) => {
+test('can not publish a mandatory dropdown with an empty value', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const optionValues = ['testOption1', 'testOption2', 'testOption3'];
   const customDataTypeId = await umbracoApi.dataType.createDropdownDataType(customDataTypeName, false, optionValues);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithImageMediaPicker.spec.ts

@@ -18,7 +18,7 @@ test.afterEach(async ({umbracoApi}) => {
   await umbracoApi.documentType.ensureNameNotExists(documentTypeName);
 });
 
-test('can save content with a image media picker', async ({umbracoApi, umbracoUi}) => {
+test('can create content with a image media picker', async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const expectedState = 'Draft';
   const dataType = await umbracoApi.dataType.getByName(dataTypeName);
@@ -60,7 +60,7 @@ test('can publish content with a image media picker', async ({umbracoApi, umbrac
   expect(contentData.variants[0].state).toBe(expectedState);
 });
 
-test('can add an image to the image media picker', async ({umbracoApi, umbracoUi}) => {
+test('can add an image to the image media picker', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const dataType = await umbracoApi.dataType.getByName(dataTypeName);
   await umbracoApi.media.ensureNameNotExists(mediaName);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithMultiURLPicker.spec.ts

@@ -170,7 +170,7 @@ test('can create content with the media link', async ({umbracoApi, umbracoUi}) =
   await umbracoApi.media.ensureNameNotExists(mediaFileName);
 });
 
-test('can add multiple links in the content', async ({umbracoApi, umbracoUi}) => {
+test('can add multiple links in the content', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   const documentTypeId = await umbracoApi.documentType.createDocumentTypeWithPropertyEditor(documentTypeName, dataTypeName, dataTypeData.id);
@@ -190,6 +190,7 @@ test('can add multiple links in the content', async ({umbracoApi, umbracoUi}) =>
   await umbracoUi.content.selectMediaWithName(mediaFileName);
   await umbracoUi.content.clickChooseModalButton();
   await umbracoUi.content.clickAddButton();
+  await umbracoUi.waitForTimeout(500); // Wait for the media link to be added
   // Add external link
   await umbracoUi.content.clickAddMultiURLPickerButton();
   await umbracoUi.content.clickManualLinkButton();
@@ -267,7 +268,7 @@ test('can edit the URL picker in the content', async ({umbracoApi, umbracoUi}) =
   expect(contentData.values[0].value[0].url).toEqual(link);
 });
 
-test('cannot submit an empty link', async ({umbracoApi, umbracoUi}) => {
+test('cannot submit an empty link', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   const documentTypeId = await umbracoApi.documentType.createDocumentTypeWithPropertyEditor(documentTypeName, dataTypeName, dataTypeData.id);
@@ -288,7 +289,7 @@ test('cannot submit an empty link', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.content.isTextWithMessageVisible(ConstantHelper.validationMessages.emptyLinkPicker);
 });
 
-test('cannot update the URL picker with an empty link', async ({umbracoApi, umbracoUi}) => {
+test('cannot update the URL picker with an empty link', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   const documentTypeId = await umbracoApi.documentType.createDocumentTypeWithPropertyEditor(documentTypeName, dataTypeName, dataTypeData.id);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithMultipleImageMediaPicker.spec.ts

@@ -69,7 +69,7 @@ test('can publish content with multiple image media picker data type', async ({u
   expect(contentData.values).toEqual([]);
 });
 
-test('can add multiple images to the multiple image media picker', async ({umbracoApi, umbracoUi}) => {
+test('can add multiple images to the multiple image media picker', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   const documentTypeId = await umbracoApi.documentType.createDocumentTypeWithPropertyEditor(documentTypeName, dataTypeName, dataTypeData.id);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithMultipleMediaPicker.spec.ts

@@ -97,7 +97,7 @@ test('can add multiple media files to the multiple media picker', async ({umbrac
   expect(contentData.values[0].value[1].mediaTypeAlias).toEqual(secondMediaTypeName);
 });
 
-test('can remove a media picker in the content', async ({umbracoApi, umbracoUi}) => {
+test('can remove a media picker in the content', {tag: '@release'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   const documentTypeId = await umbracoApi.documentType.createDocumentTypeWithPropertyEditor(documentTypeName, dataTypeName, dataTypeData.id);