Merge pull request #1693 from AzureAD/fadi/ci-repair

Work to get 2 MSAL Automation pipelines to green

Author: fadidurah

azure-pipelines/ui-automation/msal-broker-BrokerHost-nightly.yml

@@ -45,7 +45,7 @@ parameters:
   - name: testTargetPackages
     displayName: Packages as Test Targets
     type: string
-    default: package com.microsoft.identity.client.msal.automationapp.testpass.broker
+    default: package com.microsoft.identity.client.msal.automationapp.testpass.broker, notAnnotation org.junit.Ignore
   - name: oldBrokerHostVersion
     displayName: Old Broker host Version
     type: string

azure-pipelines/ui-automation/msal-broker-ProdBroker-nightly.yml

@@ -45,7 +45,7 @@ parameters:
   - name: testTargetPackages
     displayName: Packages as Test Targets
     type: string
-    default: package com.microsoft.identity.client.msal.automationapp.testpass.broker
+    default: package com.microsoft.identity.client.msal.automationapp.testpass.broker, notAnnotation org.junit.Ignore
   - name: oldBrokerHostVersion
     displayName: Old Broker host Version
     type: string

azure-pipelines/ui-automation/msal-local-nightly.yml

@@ -45,7 +45,7 @@ parameters:
   - name: testTargetPackages
     displayName: Packages as Test Targets
     type: string
-    default: package com.microsoft.identity.client.msal.automationapp.testpass.msalonly, package com.microsoft.identity.client.msal.automationapp.testpass.b2c
+    default: package com.microsoft.identity.client.msal.automationapp.testpass.msalonly, package com.microsoft.identity.client.msal.automationapp.testpass.b2c, notAnnotation org.junit.Ignore
 
 variables:
   engineeringProjectId: 'fac9d424-53d2-45c0-91b5-ef6ba7a6bf26'

azure-pipelines/ui-automation/templates/flank/flank.yml

@@ -1,19 +1,27 @@
 flank:
-  ## Max Test Shards
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test. default: 1
+   ## Max Test Shards
+   # test shards - the amount of groups to split the test suite into
+   # set to -1 to use one shard per test. default: 1
    max-test-shards: 5
 
-  ## Number of Test Runs
-  # test runs - the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
+   ## Number of Test Runs
+   # test runs - the amount of times to run the tests.
+   # 1 runs the tests once. 10 runs all the tests 10x
    num-test-runs: 1
 
-  ## Use Average Test Time for New Tests flag
-  # Enable using average time from previous tests duration when using SmartShard and tests did not run before.
-  # Default: false
+   ## Use Average Test Time for New Tests flag
+   # Enable using average time from previous tests duration when using SmartShard and tests did not run before.
+   # Default: false
    use-average-test-time-for-new-tests: true
 
-  ## Local Results Directory
-  # Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
-   local-result-dir: flank
+   ## Local Results Directory
+   # Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
+   local-result-dir: flank
+
+   ## Output Style flag
+   # Output style of execution status. May be one of [verbose, multi, single, compact].
+   # For runs with only one test execution the default value is 'verbose', in other cases
+   # 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles
+   # which don't support ansi codes, to avoid corrupted output use single or verbose.
+   # The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports.
+   output-style: single

common

@@ -29,6 +29,7 @@
 import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers;
 import com.microsoft.identity.client.ui.automation.broker.BrokerHost;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
@@ -44,6 +45,7 @@
 // Remove Broker Account
 // https://identitydivision.visualstudio.com/Engineering/_workitems/edit/1561137
 @SupportedBrokers(brokers = BrokerHost.class)
+@RetryOnFailure(retryCount = 2)
 public class TestCase1561137 extends AbstractMsalBrokerTest {
     @Test
     public void test_1561137() throws Throwable {

msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/brokerapi/TestCase1561137.java

@@ -33,6 +33,7 @@
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractGuestAccountMsalBrokerUiTest;
 import com.microsoft.identity.client.ui.automation.TestContext;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.constants.GlobalConstants;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
 import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters;
@@ -55,6 +56,7 @@
 
 // Acquire token for cross cloud guest account (with broker)
 // https://identitydivision.visualstudio.com/DefaultCollection/IDDP/_workitems/edit/1420494
+@RetryOnFailure(retryCount = 2)
 @RunWith(Parameterized.class)
 public class TestCase1420494 extends AbstractGuestAccountMsalBrokerUiTest {
 

msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1420494.java

@@ -32,6 +32,7 @@
 import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractGuestAccountMsalBrokerUiTest;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.constants.GlobalConstants;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
 import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters;
@@ -57,6 +58,7 @@
 
 // Acquire token from cross cloud when Device CA is required (via PKeyAuth)
 // https://identitydivision.visualstudio.com/DefaultCollection/IDDP/_workitems/edit/1592468
+@RetryOnFailure(retryCount = 2)
 @RunWith(Parameterized.class)
 public class TestCase1592468 extends AbstractGuestAccountMsalBrokerUiTest {
 

msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1592468.java

@@ -31,6 +31,7 @@
 import com.microsoft.identity.client.exception.MsalException;
 import com.microsoft.identity.client.msal.automationapp.R;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers;
 import com.microsoft.identity.client.ui.automation.broker.BrokerHost;
 import com.microsoft.identity.client.ui.automation.broker.BrokerMicrosoftAuthenticator;
@@ -50,10 +51,12 @@
 import org.junit.Test;
 
 import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
 
 // End My Shift - In Shared device mode, only account from the same tenant should be able to acquire token.
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/833513
 @SupportedBrokers(brokers = {BrokerMicrosoftAuthenticator.class, BrokerHost.class})
+@RetryOnFailure(retryCount = 2)
 public class TestCase833513 extends AbstractMsalBrokerTest {
 
     @Test
@@ -119,7 +122,7 @@ public void test_833513() throws MsalException, InterruptedException, LabApiExce
 
         // expecting error in WebView now
         final UiObject errMsg = UiAutomatorUtils.obtainUiObjectWithText("AADSTS50020");
-        Assert.assertTrue(errMsg.exists());
+        Assert.assertTrue(errMsg.waitForExists(TimeUnit.MINUTES.toMillis(1)));
     }
 
     @Override

msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833513.java

@@ -36,6 +36,7 @@
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TokenRequestLatch;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers;
 import com.microsoft.identity.client.ui.automation.app.AzureSampleApp;
 import com.microsoft.identity.client.ui.automation.broker.BrokerHost;
@@ -58,6 +59,7 @@
 // End My Shift - In Shared device mode, an account signed in through App A can be used by App B.
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/833514
 @SupportedBrokers(brokers = {BrokerMicrosoftAuthenticator.class, BrokerHost.class})
+@RetryOnFailure(retryCount = 2)
 public class TestCase833514 extends AbstractMsalBrokerTest {
 
     @Test