Merge pull request #222 from woocommerce/24-09/orchestration

Improve test orchestration for more powerful Compatibility Tests between plugins

Author: Luc45

.github/workflows/qit-orchestration-test.yml

@@ -0,0 +1,53 @@
+name: QIT Orchestration Tests
+
+on:
+  # Every day at 11pm UTC (6pm ET)
+  schedule:
+    - cron: '0 23 * * *'
+  # Manually
+  workflow_dispatch:
+
+jobs:
+  orchestration-tests:
+    runs-on: ubuntu-latest
+    env:
+      NO_COLOR: 1
+      QIT_DISABLE_ONBOARDING: yes
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.3'
+
+      - name: Enable dev mode
+        run: ./qit dev
+
+      - name: Connect to Staging QIT
+        run: ./qit backend:add --environment="staging" --qit_secret="${{ secrets.QIT_STAGING_SECRET }}" --manager_url="https://stagingcompatibilitydashboard.wpcomstaging.com"
+
+      - name: Composer install
+        working-directory: src
+        run: composer install
+
+      - name: Run Orchestration Tests
+        working-directory: src
+        run: |
+          cd _tests/orchestration
+          php ../../qit-cli.php run:e2e woocommerce-amazon-s3-storage -v
+
+      - name: Get Report Path
+        if: ${{ !cancelled() }}
+        working-directory: src
+        run: |
+          echo "REPORT_PATH=$(php qit-cli.php e2e-report --dir_only)" >> $GITHUB_ENV
+
+      - name: Upload Test Report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: orchestration-test-report
+          path: ${{ env.REPORT_PATH }}
+          retention-days: 14

_tests/orchestration/README.md

@@ -0,0 +1,14 @@
+# Orchestration Tests
+
+This directory contains compatibility test suites to test orchestration. To run it, do `qit run:e2e woocommerce-amazon-s3-storage` in this directory.
+
+## Overview
+
+Each test suite verifies that plugins can coexist and handle their resources properly. The test lifecycle works as follows:
+
+1. **Shared Setup**: Each plugin establishes its shared state
+2. **DB Export**: A DB snapshot is created
+3. **Plugin A - Full Cycle**: Setup → Test → Teardown
+4. **DB Import**: The DB snapshot is restored
+5. **Plugin B - Full Cycle**: Setup → Test → Teardown
+6. **Shared Teardown**: Cleanup in reverse order

_tests/orchestration/qit.yml

@@ -0,0 +1,10 @@
+plugins:
+  woocommerce-amazon-s3-storage:
+    source: ./woocommerce-amazon-s3-storage
+    test_tags:
+      - ./woocommerce-amazon-s3-storage-tests
+  woocommerce-progressive-discounts:
+    action: test
+    source: ./woocommerce-progressive-discounts
+    test_tags:
+      - ./woocommerce-progressive-discounts-tests

_tests/orchestration/woocommerce-amazon-s3-storage-tests/bootstrap/dependencies.json

@@ -0,0 +1 @@
+[]

src/src/Commands/CustomTests/scaffolding/mu-plugin.txt renamed to _tests/orchestration/woocommerce-amazon-s3-storage-tests/bootstrap/mu-plugin.php

@@ -0,0 +1,22 @@
+import { test, expect } from '@playwright/test';
+import qit from '/qitHelpers';
+
+test('Isolated Setup', async ({ page }) => {
+    await page.goto('/wp-admin');
+    console.log('[Isolated Setup JS] Amazon S3 Storage');
+    const shared = await qit.wp('option get woocommerce-amazon-s3-storage_shared_order', true);
+    expect(shared.output.trim()).toBe("first");
+    const setup = await qit.wp('option get woocommerce-amazon-s3-storage_isolated_order', true);
+    expect(setup.output.trim()).toBe("second");
+
+    // Check that Plugin B's isolated option does not exist
+    try {
+        await qit.wp('option get woocommerce-progressive-discounts_isolated_order', true);
+        throw new Error('Expected Plugin B isolated option not to exist, but it does.');
+    } catch (e) {
+        expect(e.message).toContain("Does it exist?");
+    }
+
+    const pluginBShared = await qit.wp('option get woocommerce-progressive-discounts_shared_order', true);
+    expect(pluginBShared.output.trim()).toBe("first");
+});

_tests/orchestration/woocommerce-amazon-s3-storage-tests/bootstrap/setup.js

@@ -0,0 +1,7 @@
+echo "[Isolated Setup Shell] Amazon S3 Storage"
+shared_order=$(wp option get woocommerce-amazon-s3-storage_shared_order)
+if [ "$shared_order" != "first" ]; then
+   echo "Error: Shared setup did not run first"
+   exit 1
+fi
+wp option update woocommerce-amazon-s3-storage_isolated_order "second"

_tests/orchestration/woocommerce-amazon-s3-storage-tests/bootstrap/setup.sh

@@ -0,0 +1,10 @@
+import { test, expect } from '@playwright/test';
+import qit from '/qitHelpers';
+
+test('Shared Setup', async ({ page }) => {
+    await page.goto('/wp-admin');
+    console.log('[Shared Setup JS] Amazon S3 Storage');
+    const result = await qit.wp('option get woocommerce-amazon-s3-storage_shared_order', true);
+    console.log('Shared Setup Order:', result.output);
+    expect(result.output.trim()).toBe("first");
+});

_tests/orchestration/woocommerce-amazon-s3-storage-tests/bootstrap/shared-setup.js

@@ -0,0 +1,5 @@
+#!/bin/bash
+# shared-setup.sh
+echo "[Shared Setup Shell] Amazon S3 Storage"
+wp option update "woocommerce-amazon-s3-storage_shared_order" "first"
+wp option get "woocommerce-amazon-s3-storage_shared_order"

_tests/orchestration/woocommerce-amazon-s3-storage-tests/bootstrap/shared-setup.sh

@@ -0,0 +1,39 @@
+// /tmp/foo/local/bootstrap/shared-teardown.js
+import {test, expect} from '@playwright/test';
+import qit from '/qitHelpers';
+
+// shared-teardown.js for Amazon S3
+test('Shared Teardown', async ({page}) => {
+    await page.goto('/wp-admin');
+    console.log('[Shared Teardown JS] Amazon S3 Storage');
+
+    // Verify shared setup exists
+    const shared = await qit.wp('option get woocommerce-amazon-s3-storage_shared_order', true);
+    expect(shared.output.trim()).toBe("first");
+
+    // Verify the other plugin's shared option does NOT exist (since we run last)
+    try {
+        await qit.wp('option get woocommerce-progressive-discounts_shared_order', true);
+        throw new Error("Expected other plugin's shared option not to exist, but it does.");
+    } catch (e) {
+        expect(e.message).toContain("Does it exist?");
+    }
+
+    // Verify isolated options don't exist
+    try {
+        await qit.wp('option get woocommerce-amazon-s3-storage_isolated_order', true);
+        throw new Error("Expected isolated option not to exist, but it does.");
+    } catch (e) {
+        expect(e.message).toContain("Does it exist?");
+    }
+
+    try {
+        await qit.wp('option get woocommerce-amazon-s3-storage_teardown_order', true);
+        throw new Error("Expected teardown option not to exist, but it does.");
+    } catch (e) {
+        expect(e.message).toContain("Does it exist?");
+    }
+
+    // Finally delete shared option
+    await qit.wp('option delete woocommerce-amazon-s3-storage_shared_order');
+});