Add app testing agent tools and prompts

src/appdistribution/client.ts

@@ -9,6 +9,7 @@
 
 import {
   AabInfo,
+  AIInstruction,
   BatchRemoveTestersResponse,
   Group,
   ListGroupsResponse,
@@ -67,7 +68,7 @@
     });
   }
 
-  async updateReleaseNotes(releaseName: string, releaseNotes: string): Promise<void> {
+  async updateReleaseNotes(releaseName: string, releaseNotes?: string): Promise<void> {
     if (!releaseNotes) {
       utils.logWarning("no release notes specified, skipping");
       return;
@@ -114,9 +115,9 @@
     } catch (err: any) {
       let errorMessage = err.message;
       const errorStatus = err?.context?.body?.error?.status;
       if (errorStatus === "FAILED_PRECONDITION") {
         errorMessage = "invalid testers";
       } else if (errorStatus === "INVALID_ARGUMENT") {
         errorMessage = "invalid groups";
       }
       throw new FirebaseError(`failed to distribute to testers/groups: ${errorMessage}`, {
@@ -272,6 +273,7 @@
   async createReleaseTest(
     releaseName: string,
     devices: TestDevice[],
+    aiInstruction?: AIInstruction,
     loginCredential?: LoginCredential,
     testCaseName?: string,
   ): Promise<ReleaseTest> {
@@ -283,6 +285,7 @@
           deviceExecutions: devices.map(mapDeviceToExecution),
           loginCredential,
           testCase: testCaseName,
+          aiInstructions: aiInstruction,
         },
       });
       return response.body;

src/appdistribution/distribution.ts

@@ -1,14 +1,74 @@
 import * as fs from "fs-extra";
-import { FirebaseError, getErrMsg } from "../error";
 import { logger } from "../logger";
 import * as pathUtil from "path";
+import * as utils from "../utils";
+import { UploadReleaseResult, TestDevice, ReleaseTest } from "../appdistribution/types";
+import { AppDistributionClient } from "./client";
+import { FirebaseError, getErrMsg, getErrStatus } from "../error";
+
+const TEST_MAX_POLLING_RETRIES = 40;
+const TEST_POLLING_INTERVAL_MILLIS = 30_000;
 
 export enum DistributionFileType {
   IPA = "ipa",
   APK = "apk",
   AAB = "aab",
 }
 
+export async function upload(
+  requests: AppDistributionClient,
+  appName: string,
+  distribution: Distribution,
+): Promise<string> {
+  utils.logBullet("uploading binary...");
+  try {
+    const operationName = await requests.uploadRelease(appName, distribution);
+
+    // The upload process is asynchronous, so poll to figure out when the upload has finished successfully
+    const uploadResponse = await requests.pollUploadStatus(operationName);
+
+    const release = uploadResponse.release;
+    switch (uploadResponse.result) {
+      case UploadReleaseResult.RELEASE_CREATED:
+        utils.logSuccess(
+          `uploaded new release ${release.displayVersion} (${release.buildVersion}) successfully!`,
+        );
+        break;
+      case UploadReleaseResult.RELEASE_UPDATED:
+        utils.logSuccess(
+          `uploaded update to existing release ${release.displayVersion} (${release.buildVersion}) successfully!`,
+        );
+        break;
+      case UploadReleaseResult.RELEASE_UNMODIFIED:
+        utils.logSuccess(
+          `re-uploaded already existing release ${release.displayVersion} (${release.buildVersion}) successfully!`,
+        );
+        break;
+      default:
+        utils.logSuccess(
+          `uploaded release ${release.displayVersion} (${release.buildVersion}) successfully!`,
+        );
+    }
+    utils.logSuccess(`View this release in the Firebase console: ${release.firebaseConsoleUri}`);
+    utils.logSuccess(`Share this release with testers who have access: ${release.testingUri}`);
+    utils.logSuccess(
+      `Download the release binary (link expires in 1 hour): ${release.binaryDownloadUri}`,
+    );
+    return uploadResponse.release.name;
+  } catch (err: unknown) {
+    if (getErrStatus(err) === 404) {
+      throw new FirebaseError(
+        `App Distribution could not find your app ${appName}. ` +
+          `Make sure to onboard your app by pressing the "Get started" ` +
+          "button on the App Distribution page in the Firebase console: " +
+          "https://console.firebase.google.com/project/_/appdistribution",
+        { exit: 1 },
+      );
+    }
+    throw new FirebaseError(`Failed to upload release. ${getErrMsg(err)}`, { exit: 1 });
+  }
+}
+
 /**
  * Object representing an APK, AAB or IPA file. Used for uploading app distributions.
  */
@@ -58,3 +118,62 @@
     return this.fileName;
   }
 }
+
+export async function awaitTestResults(
+  releaseTests: ReleaseTest[],
+  requests: AppDistributionClient,
+): Promise<void> {
+  const releaseTestNames = new Set(
+    releaseTests.map((rt) => rt.name).filter((n): n is string => !!n),
+  );
+  for (let i = 0; i < TEST_MAX_POLLING_RETRIES; i++) {
+    utils.logBullet(`${releaseTestNames.size} automated test results are pending...`);
+    await delay(TEST_POLLING_INTERVAL_MILLIS);
+    for (const releaseTestName of releaseTestNames) {
+      const releaseTest = await requests.getReleaseTest(releaseTestName);
+      if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) {
+        releaseTestNames.delete(releaseTestName);
+        if (releaseTestNames.size === 0) {
+          utils.logSuccess("Automated test(s) passed!");
+          return;
+        } else {
+          continue;
+        }
+      }
+      for (const execution of releaseTest.deviceExecutions) {
+        const device = deviceToString(execution.device);
+        switch (execution.state) {
+          case "PASSED":
+          case "IN_PROGRESS":
+            continue;
+          case "FAILED":
+            throw new FirebaseError(
+              `Automated test failed for ${device}: ${execution.failedReason}`,
+              { exit: 1 },
+            );
+          case "INCONCLUSIVE":
+            throw new FirebaseError(
+              `Automated test inconclusive for ${device}: ${execution.inconclusiveReason}`,
+              { exit: 1 },
+            );
+          default:
+            throw new FirebaseError(
+              `Unsupported automated test state for ${device}: ${execution.state}`,
+              { exit: 1 },
+            );
+        }
+      }
+    }
+  }
+  throw new FirebaseError("It took longer than expected to run your test(s), please try again.", {
+    exit: 1,
+  });
+}
+
+function delay(ms: number): Promise<number> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function deviceToString(device: TestDevice): string {
+  return `${device.model} (${device.version}/${device.orientation}/${device.locale})`;
+}

src/appdistribution/options-parser-util.ts

@@ -8,7 +8,7 @@
  * file and converts the input into an string[].
  * Value takes precedent over file.
  */
-export function parseIntoStringArray(value: string, file: string): string[] {
+export function parseIntoStringArray(value: string, file = ""): string[] {
   // If there is no value then the file gets parsed into a string to be split
   if (!value && file) {
     ensureFileExists(file);
@@ -36,7 +36,7 @@
 }
 
 // Ensures a the file path that the user input is valid
 export function ensureFileExists(file: string, message = ""): void {
   if (!fs.existsSync(file)) {
     throw new FirebaseError(`File ${file} does not exist: ${message}`);
   }
@@ -51,7 +51,7 @@
 }
 
 // Gets project name from project number
 export async function getProjectName(options: any): Promise<string> {
   const projectNumber = await needProjectNumber(options);
   return `projects/${projectNumber}`;
 }
@@ -61,7 +61,10 @@
   if (!options.app) {
     throw new FirebaseError("set the --app option to a valid Firebase app id and try again");
   }
-  const appId = options.app;
+  return toAppName(options.app);
+}
+
+export function toAppName(appId: string) {
   return `projects/${appId.split(":")[1]}/apps/${appId}`;
 }
 
@@ -70,7 +73,7 @@
  * and converts the input into a string[] of test device strings.
  * Value takes precedent over file.
  */
-export function parseTestDevices(value: string, file: string): TestDevice[] {
+export function parseTestDevices(value: string, file = ""): TestDevice[] {
   // If there is no value then the file gets parsed into a string to be split
   if (!value && file) {
     ensureFileExists(file);

src/appdistribution/types.ts

@@ -127,4 +127,15 @@ export interface ReleaseTest {
   deviceExecutions: DeviceExecution[];
   loginCredential?: LoginCredential;
   testCase?: string;
+  aiInstructions?: AIInstruction;
+}
+
+export interface AIInstruction {
+  steps: AIStep[];
+}
+
+export interface AIStep {
+  goal: string;
+  hint?: string;
+  successCriteria?: string;
 }