add new e2e test for validating backwards compatibility of amplify outputs

.changeset/cool-bulldogs-accept.md

@@ -0,0 +1,2 @@
+---
+---

.github/actions/setup_baseline_version/action.yml

@@ -0,0 +1,52 @@
+name: setup_baseline_version
+description: Set up a baseline or "previous" version of the library for testing. Mostly useful for backwards compatibility
+outputs:
+  baseline_dir:
+    description: 'Path where baseline project directory is setup'
+    value: ${{ steps.move_baseline_version.outputs.baseline_dir }}
+runs:
+  using: composite
+  steps:
+    - name: Get baseline commit sha
+      id: get_baseline_commit_sha
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        if [[ ${{ github.event_name }} == 'push' ]]; then
+          # The SHA of the most recent commit on ref before the push.
+          baseline_commit_sha="${{ github.event.before }}"
+        elif [[ ${{ github.event_name }} == 'pull_request' ]]; then
+          # The SHA of the HEAD commit on base branch.
+          baseline_commit_sha="${{ github.event.pull_request.base.sha }}"
+        elif [[ ${{ github.event_name }} == 'schedule' ]] || [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
+          # The SHA of the parent of HEAD commit on main branch.
+          # This assumes linear history of main branch, i.e. one parent.
+          # These events have only information about HEAD commit, hence the need for lookup.
+          baseline_commit_sha=$(gh api /repos/${{ github.repository }}/commits/${{ github.sha }} | jq -r '.parents[0].sha')
+        else
+          echo Unable to determine baseline commit sha;
+          exit 1;
+        fi
+        echo baseline commit sha is $baseline_commit_sha;
+        echo "baseline_commit_sha=$baseline_commit_sha" >> "$GITHUB_OUTPUT";
+    - name: Checkout baseline version
+      uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
+      with:
+        ref: ${{ steps.get_baseline_commit_sha.outputs.baseline_commit_sha }}
+    - uses: ./.github/actions/setup_node
+    - name: Install and build baseline version
+      shell: bash
+      run: |
+        npm ci
+        npm run build
+    - name: Move baseline version
+      id: move_baseline_version
+      shell: bash
+      run: |
+        BASELINE_DIR=$(mktemp -d)
+        # Command below makes shell include .hidden files in file system commands (i.e. mv).
+        # This is to make sure that .git directory is moved with the repo content.
+        shopt -s dotglob
+        mv ./* $BASELINE_DIR
+        echo "baseline_dir=$BASELINE_DIR" >> "$GITHUB_OUTPUT";

.github/workflows/health_checks.yml

@@ -160,46 +160,13 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Get baseline commit sha
-        id: get_baseline_commit_sha
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          if [[ ${{ github.event_name }} == 'push' ]]; then
-            # The SHA of the most recent commit on ref before the push.
-            baseline_commit_sha="${{ github.event.before }}"
-          elif [[ ${{ github.event_name }} == 'pull_request' ]]; then
-            # The SHA of the HEAD commit on base branch.
-            baseline_commit_sha="${{ github.event.pull_request.base.sha }}"
-          elif [[ ${{ github.event_name }} == 'schedule' ]] || [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
-            # The SHA of the parent of HEAD commit on main branch.
-            # This assumes linear history of main branch, i.e. one parent.
-            # These events have only information about HEAD commit, hence the need for lookup.
-            baseline_commit_sha=$(gh api /repos/${{ github.repository }}/commits/${{ github.sha }} | jq -r '.parents[0].sha')
-          else
-            echo Unable to determine baseline commit sha;
-            exit 1;
-          fi
-          echo baseline commit sha is $baseline_commit_sha;
-          echo "baseline_commit_sha=$baseline_commit_sha" >> "$GITHUB_OUTPUT";
-      - name: Checkout baseline version
+      # This checkout is needed for the setup_baseline_version action to run `checkout` inside
+      # See https://github.com/actions/checkout/issues/692
+      - name: Checkout version for baseline
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
-        with:
-          ref: ${{ steps.get_baseline_commit_sha.outputs.baseline_commit_sha }}
-      - uses: ./.github/actions/setup_node
-      - name: Install and build baseline version
-        run: |
-          npm ci
-          npm run build
-      - name: Move baseline version
-        id: move_baseline_version
-        run: |
-          BASELINE_DIR=$(mktemp -d)
-          # Command below makes shell include .hidden files in file system commands (i.e. mv).
-          # This is to make sure that .git directory is moved with the repo content.
-          shopt -s dotglob
-          mv ./* $BASELINE_DIR
-          echo "baseline_dir=$BASELINE_DIR" >> "$GITHUB_OUTPUT";
+      - name: Setup baseline version
+        uses: ./.github/actions/setup_baseline_version
+        id: setup_baseline_version
       - name: Checkout current version
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
       - name: Run e2e iam access drift test
@@ -209,7 +176,36 @@ jobs:
           node_version: ${{ matrix.node-version }}
           run: npm run test:dir packages/integration-tests/lib/test-e2e/iam_access_drift.test.js
         env:
-          BASELINE_DIR: ${{ steps.move_baseline_version.outputs.baseline_dir }}
+          BASELINE_DIR: ${{ steps.setup_baseline_version.outputs.baseline_dir }}
+  e2e_amplify_outputs_backwards_compatibility:
+    if: needs.do_include_e2e.outputs.run_e2e == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    needs:
+      - do_include_e2e
+      - build
+    permissions:
+      # these permissions are required for the configure-aws-credentials action to get a JWT from GitHub
+      id-token: write
+      contents: read
+    steps:
+      # This checkout is needed for the setup_baseline_version action to run `checkout` inside
+      # See https://github.com/actions/checkout/issues/692
+      - name: Checkout version for baseline
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
+      - name: Setup baseline version
+        uses: ./.github/actions/setup_baseline_version
+        id: setup_baseline_version
+      - name: Checkout current version
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
+      - name: Run e2e amplify outputs backwards compatibility test
+        uses: ./.github/actions/run_with_e2e_account
+        with:
+          e2e_test_accounts: ${{ vars.E2E_TEST_ACCOUNTS }}
+          node_version: ${{ matrix.node-version }}
+          run: npm run test:dir packages/integration-tests/lib/test-e2e/amplify_outputs_backwards_compatibility.test.js
+        env:
+          BASELINE_DIR: ${{ steps.setup_baseline_version.outputs.baseline_dir }}
   e2e_deployment:
     if: needs.do_include_e2e.outputs.run_e2e == 'true'
     strategy:

packages/integration-tests/src/test-e2e/amplify_outputs_backwards_compatibility.test.ts

@@ -0,0 +1,212 @@
+import { after, before, describe, it } from 'node:test';
+import { execa } from 'execa';
+import path from 'path';
+import { TestBranch, amplifyAppPool } from '../amplify_app_pool.js';
+import { BackendIdentifier } from '@aws-amplify/plugin-types';
+import {
+  CloudFormationClient,
+  DeleteStackCommand,
+} from '@aws-sdk/client-cloudformation';
+import fsp from 'fs/promises';
+import { e2eToolingClientConfig } from '../e2e_tooling_client_config.js';
+import { NpmProxyController } from '../npm_proxy_controller.js';
+import assert from 'assert';
+import os from 'os';
+import { generateClientConfig } from '@aws-amplify/client-config';
+import { BackendIdentifierConversions } from '@aws-amplify/platform-core';
+import { amplifyAtTag } from '../constants.js';
+
+void describe('client config backwards compatibility', () => {
+  let branchBackendIdentifier: BackendIdentifier;
+  let testBranch: TestBranch;
+  let cfnClient: CloudFormationClient;
+  let tempDir: string;
+  let baselineDir: string;
+  let baselineNpmProxyController: NpmProxyController;
+  let currentNpmProxyController: NpmProxyController;
+
+  before(async () => {
+    assert.ok(
+      process.env.BASELINE_DIR,
+      'BASELINE_DIR environment variable must be set and point to amplify-backend repo at baseline version'
+    );
+    baselineDir = process.env.BASELINE_DIR;
+
+    tempDir = await fsp.mkdtemp(
+      path.join(os.tmpdir(), 'test-amplify-outputs-backwards-compatibility')
+    );
+
+    console.log(`Temp dir is ${tempDir}`);
+
+    cfnClient = new CloudFormationClient(e2eToolingClientConfig);
+    baselineNpmProxyController = new NpmProxyController(baselineDir);
+    currentNpmProxyController = new NpmProxyController();
+    testBranch = await amplifyAppPool.createTestBranch();
+    branchBackendIdentifier = {
+      namespace: testBranch.appId,
+      name: testBranch.branchName,
+      type: 'branch',
+    };
+  });
+
+  after(async () => {
+    await cfnClient.send(
+      new DeleteStackCommand({
+        StackName: BackendIdentifierConversions.toStackName(
+          branchBackendIdentifier
+        ),
+      })
+    );
+    await fsp.rm(tempDir, { recursive: true });
+
+    await baselineNpmProxyController.tearDown();
+    await currentNpmProxyController.tearDown();
+  });
+
+  const deploy = async (): Promise<void> => {
+    await execa(
+      'npx',
+      [
+        'ampx',
+        'pipeline-deploy',
+        '--branch',
+        branchBackendIdentifier.name,
+        '--appId',
+        branchBackendIdentifier.namespace,
+      ],
+      {
+        cwd: tempDir,
+        stdio: 'inherit',
+        env: {
+          CI: 'true',
+        },
+      }
+    );
+  };
+
+  const reinstallDependencies = async (): Promise<void> => {
+    await fsp.rm(path.join(tempDir, 'node_modules'), {
+      recursive: true,
+      force: true,
+    });
+    await fsp.unlink(path.join(tempDir, 'package-lock.json'));
+
+    await execa('npm', ['install'], {
+      cwd: tempDir,
+      stdio: 'inherit',
+    });
+  };
+
+  const assertGenerateClientConfigAPI = async (
+    type: 'baseline' | 'current'
+  ) => {
+    try {
+      assert.ok(
+        await generateClientConfig(branchBackendIdentifier, '1'),
+        `outputs v1 failed to be generated for an app created with ${type} library version`
+      );
+    } catch (e) {
+      throw new Error(
+        `outputs v1 failed to be generated for an app created with ${type} library version. Error: ${JSON.stringify(
+          e
+        )}`
+      );
+    }
+    try {
+      assert.ok(
+        await generateClientConfig(branchBackendIdentifier, '1.1'),
+        `outputs v1.1 failed to be generated for an app created with ${type} library version`
+      );
+    } catch (e) {
+      throw new Error(
+        `outputs v1.1 failed to be generated for an app created with ${type} library version. Error: ${JSON.stringify(
+          e
+        )}`
+      );
+    }
+  };
+
+  const assertGenerateClientConfigCommand = async (
+    type: 'baseline' | 'current'
+  ) => {
+    await execa(
+      'npx',
+      [
+        'ampx',
+        'generate',
+        'outputs',
+        '--stack',
+        BackendIdentifierConversions.toStackName(branchBackendIdentifier),
+      ],
+      {
+        cwd: tempDir,
+        stdio: 'inherit',
+      }
+    );
+
+    const fileSize = (
+      await fsp.stat(path.join(tempDir, 'amplify_outputs.json'))
+    ).size;
+    assert.ok(
+      fileSize > 100, // Validate that it's not just a shim
+      `outputs file should not be empty when generating for a ${
+        type === 'baseline' ? 'new' : 'old'
+      } new app with the ${type} version`
+    );
+  };
+
+  void it('outputs generation should be backwards and forward compatible', async () => {
+    // build an app using previous (baseline) version
+    await baselineNpmProxyController.setUp();
+    await execa('npm', ['create', amplifyAtTag, '--yes'], {
+      cwd: tempDir,
+      stdio: 'inherit',
+    });
+
+    // Replace backend.ts to add custom outputs without version as well.
+    await fsp.writeFile(
+      path.join(tempDir, 'amplify', 'backend.ts'),
+      `import { defineBackend } from '@aws-amplify/backend';
+import { auth } from './auth/resource';
+import { data } from './data/resource';
+
+const backend = defineBackend({
+  auth,
+  data,
+});
+
+backend.addOutput({
+  custom: {
+    someCustomOutput: 'someCustomOutputValue',
+  },
+});
+`
+    );
+    await deploy();
+    await baselineNpmProxyController.tearDown();
+
+    // Generate the outputs using the current version for apps built with baseline version
+
+    // 1. via CLI command
+    await currentNpmProxyController.setUp();
+    await reinstallDependencies();
+    await assertGenerateClientConfigCommand('current');
+
+    // 2. via API.
+    await assertGenerateClientConfigAPI('current');
+
+    // Re-deploy the app using the current version now
+    await deploy();
+
+    // Generate the outputs using the baseline version for apps built with current version
+
+    // 1. via CLI command
+    await currentNpmProxyController.tearDown();
+    await baselineNpmProxyController.setUp();
+    await reinstallDependencies();
+    await assertGenerateClientConfigCommand('baseline');
+
+    // 2. via API.
+    await assertGenerateClientConfigAPI('baseline');
+  });
+});