Skip to content

Commit 5af7781

Browse files
authored
add new e2e test for validating backwards compatibility of amplify outputs (#2009)
* add new e2e test for validating backwards compatibility of amplify outputs * add changeset * try this * try this * try this * try this * try this * PR Updates
1 parent ffc3b42 commit 5af7781

File tree

4 files changed

+302
-40
lines changed

4 files changed

+302
-40
lines changed

.changeset/cool-bulldogs-accept.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
---
2+
---
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: setup_baseline_version
2+
description: Set up a baseline or "previous" version of the library for testing. Mostly useful for backwards compatibility
3+
outputs:
4+
baseline_dir:
5+
description: 'Path where baseline project directory is setup'
6+
value: ${{ steps.move_baseline_version.outputs.baseline_dir }}
7+
runs:
8+
using: composite
9+
steps:
10+
- name: Get baseline commit sha
11+
id: get_baseline_commit_sha
12+
shell: bash
13+
env:
14+
GH_TOKEN: ${{ github.token }}
15+
run: |
16+
if [[ ${{ github.event_name }} == 'push' ]]; then
17+
# The SHA of the most recent commit on ref before the push.
18+
baseline_commit_sha="${{ github.event.before }}"
19+
elif [[ ${{ github.event_name }} == 'pull_request' ]]; then
20+
# The SHA of the HEAD commit on base branch.
21+
baseline_commit_sha="${{ github.event.pull_request.base.sha }}"
22+
elif [[ ${{ github.event_name }} == 'schedule' ]] || [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
23+
# The SHA of the parent of HEAD commit on main branch.
24+
# This assumes linear history of main branch, i.e. one parent.
25+
# These events have only information about HEAD commit, hence the need for lookup.
26+
baseline_commit_sha=$(gh api /repos/${{ github.repository }}/commits/${{ github.sha }} | jq -r '.parents[0].sha')
27+
else
28+
echo Unable to determine baseline commit sha;
29+
exit 1;
30+
fi
31+
echo baseline commit sha is $baseline_commit_sha;
32+
echo "baseline_commit_sha=$baseline_commit_sha" >> "$GITHUB_OUTPUT";
33+
- name: Checkout baseline version
34+
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
35+
with:
36+
ref: ${{ steps.get_baseline_commit_sha.outputs.baseline_commit_sha }}
37+
- uses: ./.github/actions/setup_node
38+
- name: Install and build baseline version
39+
shell: bash
40+
run: |
41+
npm ci
42+
npm run build
43+
- name: Move baseline version
44+
id: move_baseline_version
45+
shell: bash
46+
run: |
47+
BASELINE_DIR=$(mktemp -d)
48+
# Command below makes shell include .hidden files in file system commands (i.e. mv).
49+
# This is to make sure that .git directory is moved with the repo content.
50+
shopt -s dotglob
51+
mv ./* $BASELINE_DIR
52+
echo "baseline_dir=$BASELINE_DIR" >> "$GITHUB_OUTPUT";

.github/workflows/health_checks.yml

Lines changed: 36 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -160,46 +160,13 @@ jobs:
160160
id-token: write
161161
contents: read
162162
steps:
163-
- name: Get baseline commit sha
164-
id: get_baseline_commit_sha
165-
env:
166-
GH_TOKEN: ${{ github.token }}
167-
run: |
168-
if [[ ${{ github.event_name }} == 'push' ]]; then
169-
# The SHA of the most recent commit on ref before the push.
170-
baseline_commit_sha="${{ github.event.before }}"
171-
elif [[ ${{ github.event_name }} == 'pull_request' ]]; then
172-
# The SHA of the HEAD commit on base branch.
173-
baseline_commit_sha="${{ github.event.pull_request.base.sha }}"
174-
elif [[ ${{ github.event_name }} == 'schedule' ]] || [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
175-
# The SHA of the parent of HEAD commit on main branch.
176-
# This assumes linear history of main branch, i.e. one parent.
177-
# These events have only information about HEAD commit, hence the need for lookup.
178-
baseline_commit_sha=$(gh api /repos/${{ github.repository }}/commits/${{ github.sha }} | jq -r '.parents[0].sha')
179-
else
180-
echo Unable to determine baseline commit sha;
181-
exit 1;
182-
fi
183-
echo baseline commit sha is $baseline_commit_sha;
184-
echo "baseline_commit_sha=$baseline_commit_sha" >> "$GITHUB_OUTPUT";
185-
- name: Checkout baseline version
163+
# This checkout is needed for the setup_baseline_version action to run `checkout` inside
164+
# See https://github.com/actions/checkout/issues/692
165+
- name: Checkout version for baseline
186166
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
187-
with:
188-
ref: ${{ steps.get_baseline_commit_sha.outputs.baseline_commit_sha }}
189-
- uses: ./.github/actions/setup_node
190-
- name: Install and build baseline version
191-
run: |
192-
npm ci
193-
npm run build
194-
- name: Move baseline version
195-
id: move_baseline_version
196-
run: |
197-
BASELINE_DIR=$(mktemp -d)
198-
# Command below makes shell include .hidden files in file system commands (i.e. mv).
199-
# This is to make sure that .git directory is moved with the repo content.
200-
shopt -s dotglob
201-
mv ./* $BASELINE_DIR
202-
echo "baseline_dir=$BASELINE_DIR" >> "$GITHUB_OUTPUT";
167+
- name: Setup baseline version
168+
uses: ./.github/actions/setup_baseline_version
169+
id: setup_baseline_version
203170
- name: Checkout current version
204171
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
205172
- name: Run e2e iam access drift test
@@ -209,7 +176,36 @@ jobs:
209176
node_version: ${{ matrix.node-version }}
210177
run: npm run test:dir packages/integration-tests/lib/test-e2e/iam_access_drift.test.js
211178
env:
212-
BASELINE_DIR: ${{ steps.move_baseline_version.outputs.baseline_dir }}
179+
BASELINE_DIR: ${{ steps.setup_baseline_version.outputs.baseline_dir }}
180+
e2e_amplify_outputs_backwards_compatibility:
181+
if: needs.do_include_e2e.outputs.run_e2e == 'true'
182+
runs-on: ubuntu-latest
183+
timeout-minutes: 25
184+
needs:
185+
- do_include_e2e
186+
- build
187+
permissions:
188+
# these permissions are required for the configure-aws-credentials action to get a JWT from GitHub
189+
id-token: write
190+
contents: read
191+
steps:
192+
# This checkout is needed for the setup_baseline_version action to run `checkout` inside
193+
# See https://github.com/actions/checkout/issues/692
194+
- name: Checkout version for baseline
195+
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
196+
- name: Setup baseline version
197+
uses: ./.github/actions/setup_baseline_version
198+
id: setup_baseline_version
199+
- name: Checkout current version
200+
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
201+
- name: Run e2e amplify outputs backwards compatibility test
202+
uses: ./.github/actions/run_with_e2e_account
203+
with:
204+
e2e_test_accounts: ${{ vars.E2E_TEST_ACCOUNTS }}
205+
node_version: ${{ matrix.node-version }}
206+
run: npm run test:dir packages/integration-tests/lib/test-e2e/amplify_outputs_backwards_compatibility.test.js
207+
env:
208+
BASELINE_DIR: ${{ steps.setup_baseline_version.outputs.baseline_dir }}
213209
e2e_deployment:
214210
if: needs.do_include_e2e.outputs.run_e2e == 'true'
215211
strategy:
Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
import { after, before, describe, it } from 'node:test';
2+
import { execa } from 'execa';
3+
import path from 'path';
4+
import { TestBranch, amplifyAppPool } from '../amplify_app_pool.js';
5+
import { BackendIdentifier } from '@aws-amplify/plugin-types';
6+
import {
7+
CloudFormationClient,
8+
DeleteStackCommand,
9+
} from '@aws-sdk/client-cloudformation';
10+
import fsp from 'fs/promises';
11+
import { e2eToolingClientConfig } from '../e2e_tooling_client_config.js';
12+
import { NpmProxyController } from '../npm_proxy_controller.js';
13+
import assert from 'assert';
14+
import os from 'os';
15+
import { generateClientConfig } from '@aws-amplify/client-config';
16+
import { BackendIdentifierConversions } from '@aws-amplify/platform-core';
17+
import { amplifyAtTag } from '../constants.js';
18+
19+
void describe('client config backwards compatibility', () => {
20+
let branchBackendIdentifier: BackendIdentifier;
21+
let testBranch: TestBranch;
22+
let cfnClient: CloudFormationClient;
23+
let tempDir: string;
24+
let baselineDir: string;
25+
let baselineNpmProxyController: NpmProxyController;
26+
let currentNpmProxyController: NpmProxyController;
27+
28+
before(async () => {
29+
assert.ok(
30+
process.env.BASELINE_DIR,
31+
'BASELINE_DIR environment variable must be set and point to amplify-backend repo at baseline version'
32+
);
33+
baselineDir = process.env.BASELINE_DIR;
34+
35+
tempDir = await fsp.mkdtemp(
36+
path.join(os.tmpdir(), 'test-amplify-outputs-backwards-compatibility')
37+
);
38+
39+
console.log(`Temp dir is ${tempDir}`);
40+
41+
cfnClient = new CloudFormationClient(e2eToolingClientConfig);
42+
baselineNpmProxyController = new NpmProxyController(baselineDir);
43+
currentNpmProxyController = new NpmProxyController();
44+
testBranch = await amplifyAppPool.createTestBranch();
45+
branchBackendIdentifier = {
46+
namespace: testBranch.appId,
47+
name: testBranch.branchName,
48+
type: 'branch',
49+
};
50+
});
51+
52+
after(async () => {
53+
await cfnClient.send(
54+
new DeleteStackCommand({
55+
StackName: BackendIdentifierConversions.toStackName(
56+
branchBackendIdentifier
57+
),
58+
})
59+
);
60+
await fsp.rm(tempDir, { recursive: true });
61+
62+
await baselineNpmProxyController.tearDown();
63+
await currentNpmProxyController.tearDown();
64+
});
65+
66+
const deploy = async (): Promise<void> => {
67+
await execa(
68+
'npx',
69+
[
70+
'ampx',
71+
'pipeline-deploy',
72+
'--branch',
73+
branchBackendIdentifier.name,
74+
'--appId',
75+
branchBackendIdentifier.namespace,
76+
],
77+
{
78+
cwd: tempDir,
79+
stdio: 'inherit',
80+
env: {
81+
CI: 'true',
82+
},
83+
}
84+
);
85+
};
86+
87+
const reinstallDependencies = async (): Promise<void> => {
88+
await fsp.rm(path.join(tempDir, 'node_modules'), {
89+
recursive: true,
90+
force: true,
91+
});
92+
await fsp.unlink(path.join(tempDir, 'package-lock.json'));
93+
94+
await execa('npm', ['install'], {
95+
cwd: tempDir,
96+
stdio: 'inherit',
97+
});
98+
};
99+
100+
const assertGenerateClientConfigAPI = async (
101+
type: 'baseline' | 'current'
102+
) => {
103+
try {
104+
assert.ok(
105+
await generateClientConfig(branchBackendIdentifier, '1'),
106+
`outputs v1 failed to be generated for an app created with ${type} library version`
107+
);
108+
} catch (e) {
109+
throw new Error(
110+
`outputs v1 failed to be generated for an app created with ${type} library version. Error: ${JSON.stringify(
111+
e
112+
)}`
113+
);
114+
}
115+
try {
116+
assert.ok(
117+
await generateClientConfig(branchBackendIdentifier, '1.1'),
118+
`outputs v1.1 failed to be generated for an app created with ${type} library version`
119+
);
120+
} catch (e) {
121+
throw new Error(
122+
`outputs v1.1 failed to be generated for an app created with ${type} library version. Error: ${JSON.stringify(
123+
e
124+
)}`
125+
);
126+
}
127+
};
128+
129+
const assertGenerateClientConfigCommand = async (
130+
type: 'baseline' | 'current'
131+
) => {
132+
await execa(
133+
'npx',
134+
[
135+
'ampx',
136+
'generate',
137+
'outputs',
138+
'--stack',
139+
BackendIdentifierConversions.toStackName(branchBackendIdentifier),
140+
],
141+
{
142+
cwd: tempDir,
143+
stdio: 'inherit',
144+
}
145+
);
146+
147+
const fileSize = (
148+
await fsp.stat(path.join(tempDir, 'amplify_outputs.json'))
149+
).size;
150+
assert.ok(
151+
fileSize > 100, // Validate that it's not just a shim
152+
`outputs file should not be empty when generating for a ${
153+
type === 'baseline' ? 'new' : 'old'
154+
} new app with the ${type} version`
155+
);
156+
};
157+
158+
void it('outputs generation should be backwards and forward compatible', async () => {
159+
// build an app using previous (baseline) version
160+
await baselineNpmProxyController.setUp();
161+
await execa('npm', ['create', amplifyAtTag, '--yes'], {
162+
cwd: tempDir,
163+
stdio: 'inherit',
164+
});
165+
166+
// Replace backend.ts to add custom outputs without version as well.
167+
await fsp.writeFile(
168+
path.join(tempDir, 'amplify', 'backend.ts'),
169+
`import { defineBackend } from '@aws-amplify/backend';
170+
import { auth } from './auth/resource';
171+
import { data } from './data/resource';
172+
173+
const backend = defineBackend({
174+
auth,
175+
data,
176+
});
177+
178+
backend.addOutput({
179+
custom: {
180+
someCustomOutput: 'someCustomOutputValue',
181+
},
182+
});
183+
`
184+
);
185+
await deploy();
186+
await baselineNpmProxyController.tearDown();
187+
188+
// Generate the outputs using the current version for apps built with baseline version
189+
190+
// 1. via CLI command
191+
await currentNpmProxyController.setUp();
192+
await reinstallDependencies();
193+
await assertGenerateClientConfigCommand('current');
194+
195+
// 2. via API.
196+
await assertGenerateClientConfigAPI('current');
197+
198+
// Re-deploy the app using the current version now
199+
await deploy();
200+
201+
// Generate the outputs using the baseline version for apps built with current version
202+
203+
// 1. via CLI command
204+
await currentNpmProxyController.tearDown();
205+
await baselineNpmProxyController.setUp();
206+
await reinstallDependencies();
207+
await assertGenerateClientConfigCommand('baseline');
208+
209+
// 2. via API.
210+
await assertGenerateClientConfigAPI('baseline');
211+
});
212+
});

0 commit comments

Comments
 (0)