Merge pull request #170 from woocommerce/24-05/send-custom-tests-results

Send custom tests results to Manager

Author: Luc45

_tests/custom_tests/tests/CompatibilityTest.php

@@ -103,7 +103,8 @@ public function test_multiple_tags_and_multiple_plugins_with_multiple_tags() {
 
 		$output = qit( [
 			'run:e2e',
-			'automatewoo:test:self-test-multiple-test-tags,self-test-multiple-test-tags-another',
+			'automatewoo',
+			'self-test-multiple-test-tags,self-test-multiple-test-tags-another',
 			'--plugin',
 			'woocommerce:test:self-test-multiple-test-tags,self-test-multiple-test-tags-another',
 		] );

_tests/custom_tests/tests/EnvTest.php

@@ -220,7 +220,7 @@ public function test_env_up_woocommerce_nightly_version() {
 			'wp plugin get woocommerce',
 		] );
 
-		$this->assertMatchesNormalizedSnapshot( $output );
+		$this->assertStringContainsString( '-dev', $output );
 	}
 
 	public function test_env_up_woocommerce_rc_version() {

_tests/custom_tests/tests/RunE2ETest.php

@@ -46,7 +46,8 @@ public function test_tag_and_run_test() {
 
 		$output = qit( [
 			'run:e2e',
-			'automatewoo:test:self-test-tag-and-run',
+			'automatewoo',
+			'self-test-tag-and-run',
 			'--plugin',
 			'woocommerce:activate',
 		] );
@@ -73,7 +74,8 @@ public function test_multiple_tags_and_run_tests() {
 
 		$output = qit( [
 			'run:e2e',
-			'automatewoo:test:self-test-multiple-test-tags,self-test-multiple-test-tags-another',
+			'automatewoo',
+			'self-test-multiple-test-tags,self-test-multiple-test-tags-another',
 			'--plugin',
 			'woocommerce:activate',
 		] );

_tests/custom_tests/tests/__snapshots__/EnvTest__test_env_up_woocommerce_nightly_version__1.txt

@@ -16,8 +16,8 @@
 use QIT_CLI\Environment\Environments\E2E\E2EEnvironment;
 use QIT_CLI\Environment\Environments\EnvInfo;
 use QIT_CLI\Environment\Environments\Environment;
-use QIT_CLI\Environment\Extension;
 use QIT_CLI\LocalTests\E2E\E2ETestManager;
+use QIT_CLI\LocalTests\LocalTestRunNotifier;
 use QIT_CLI\WooExtensionsList;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\ArrayInput;
@@ -27,6 +27,7 @@
 use Symfony\Component\Console\Output\NullOutput;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Output\StreamOutput;
+use Symfony\Component\Console\Style\SymfonyStyle;
 use function QIT_CLI\is_windows;
 
 class RunE2ECommand extends DynamicCommand {
@@ -45,20 +46,25 @@ class RunE2ECommand extends DynamicCommand {
 	/** @var WooExtensionsList */
 	protected $woo_extensions_list;
 
+	/** @var LocalTestRunNotifier */
+	protected $test_run_notifier;
+
 	protected static $defaultName = 'run:e2e'; // phpcs:ignore WordPress.NamingConventions.ValidVariableName.PropertyNotSnakeCase
 
 	public function __construct(
 		E2EEnvironment $e2e_environment,
 		Cache $cache,
 		OutputInterface $output,
 		E2ETestManager $e2e_test_manager,
-		WooExtensionsList $woo_extensions_list
+		WooExtensionsList $woo_extensions_list,
+		LocalTestRunNotifier $test_run_notifier
 	) {
 		$this->e2e_environment     = $e2e_environment;
 		$this->cache               = $cache;
 		$this->output              = $output;
 		$this->e2e_test_manager    = $e2e_test_manager;
 		$this->woo_extensions_list = $woo_extensions_list;
+		$this->test_run_notifier   = $test_run_notifier;
 		parent::__construct( static::$defaultName ); // phpcs:ignore WordPress.NamingConventions.ValidVariableName.UsedPropertyNotSnakeCase
 	}
 
@@ -74,8 +80,9 @@ protected function configure() {
 		] );
 
 		$this
-			->addArgument( 'woo_extension', InputArgument::OPTIONAL, 'A QIT plugin-syntax as defined in the documentation: source:action:test-tags:slug. Only "source" is required, and it can be a slug, a file, a URL. Action can be "activate", "bootstrap", and "test", and test-tags are a comme-separated list of tests. Slug is usually not required. Read the docs.' )
+			->addArgument( 'woo_extension', InputArgument::OPTIONAL, 'The slug or WooCommerce ID of the main extension under test.' )
 			->addArgument( 'test', InputArgument::OPTIONAL, '(Optional) The tests for the main extension under test. Accepts test tags, or a test directory. If not set, will use the "default" test tag of this extension.' )
+			->addOption( 'source', null, InputOption::VALUE_OPTIONAL, 'The source of the main extension under test. Accepts a slug, a file, a URL. If not provided, the source will be the slug.' )
 			->addOption( 'wp', null, InputOption::VALUE_OPTIONAL, 'The WordPress version. Accepts "stable", "nightly", or a version number.', 'stable' )
 			->addOption( 'woo', null, InputOption::VALUE_OPTIONAL, 'The WooCommerce Version. Accepts "stable", "nightly", or a GitHub Tag (eg: 8.6.1).' )
 			->addOption( 'plugin', 'p', InputOption::VALUE_OPTIONAL | InputOption::VALUE_IS_ARRAY, 'Plugin to activate in the environment. Accepts paths, Woo.com slugs/product IDs, WordPress.org slugs or GitHub URLs.', [] )
@@ -85,8 +92,10 @@ protected function configure() {
 			->addOption( 'require', 'r', InputOption::VALUE_OPTIONAL | InputOption::VALUE_IS_ARRAY, 'Load PHP file before running the command (may be used more than once).' )
 			->addOption( 'config', null, InputOption::VALUE_OPTIONAL, '(Optional) QIT config file to use.' )
 			->addOption( 'object_cache', 'o', InputOption::VALUE_NONE, 'Whether to enable Object Cache (Redis) in the environment.' )
+			->addOption( 'sut_action', null, InputOption::VALUE_OPTIONAL, 'What action to use for the main extension under test. Accepts "activate", "install" and "test". Default to "test"', 'test' )
 			->addOption( 'no_activate', 's', InputOption::VALUE_NONE, 'Skip activating plugins in the environment.' )
 			->addOption( 'shard', null, InputOption::VALUE_OPTIONAL, 'Playwright Sharding argument.' )
+			->addOption( 'no_upload_report', null, InputOption::VALUE_NONE, 'Do not upload the report to QIT Manager.' )
 			->addOption( 'update_snapshots', null, InputOption::VALUE_NONE, 'Update snapshots where applicable (eg: Playwright Snapshots).' )
 			->addOption( 'pw_options', null, InputOption::VALUE_OPTIONAL, 'Additional options and parameters to pass to Playwright.' )
 			->addOption( 'ui', null, InputOption::VALUE_NONE, 'Runs tests in UI mode. In this mode, you can start and view the tests running.' )
@@ -127,12 +136,15 @@ protected function execute( InputInterface $input, OutputInterface $output ): in
 			$test_mode = E2ETestManager::$test_modes['headless'];
 		}
 
-		$wait             = $input->getOption( 'up_only' ) || $test_mode === E2ETestManager::$test_modes['codegen'];
-		$woo_extension    = $input->getArgument( 'woo_extension' );
-		$test             = $input->getArgument( 'test' );
-		$shard            = $input->getOption( 'shard' );
-		$update_snapshots = $input->getOption( 'update_snapshots' );
-		$pw_options       = $input->getOption( 'pw_options' ) ?? '';
+		$wait                = $input->getOption( 'up_only' ) || $test_mode === E2ETestManager::$test_modes['codegen'];
+		$woo_extension       = $input->getArgument( 'woo_extension' );
+		$test                = $input->getArgument( 'test' );
+		$woocommerce_version = $input->getOption( 'woo' );
+		$shard               = $input->getOption( 'shard' );
+		$update_snapshots    = $input->getOption( 'update_snapshots' );
+		$pw_options          = $input->getOption( 'pw_options' ) ?? '';
+		$source              = $input->getOption( 'source' ) ?? $woo_extension;
+		$sut_action          = $input->getOption( 'sut_action' );
 
 		if ( ! empty( $pw_options ) ) {
 			// Remove wrapping double quotes if they exist.
@@ -148,39 +160,49 @@ protected function execute( InputInterface $input, OutputInterface $output ): in
 		App::setVar( 'pw_options', $pw_options );
 
 		// Validate the extension is set if needed.
-		if ( empty( $woo_extension ) && ! $wait ) {
-			$output->writeln( '<error>The extension parameter is only optional in --up_only or --codegen modes.</error>' );
+		if ( empty( $woo_extension ) ) {
+			if ( ! empty( $source ) ) {
+				$output->writeln( '<error>The extension parameter is required when the source parameter is set.</error>' );
 
-			return Command::INVALID;
+				return Command::INVALID;
+			}
+			if ( ! empty( $sut_action ) ) {
+				$output->writeln( '<error>The extension parameter is required when the sut_action parameter is set.</error>' );
+
+				return Command::INVALID;
+			}
+			if ( ! $wait ) {
+				$output->writeln( '<error>The extension parameter is only optional in --up_only or --codegen modes.</error>' );
+
+				return Command::INVALID;
+			}
 		}
 
 		if ( ! empty( $woo_extension ) ) {
-			if ( ! empty( $test ) ) {
-				if ( ! file_exists( $test ) ) {
-					$output->writeln( "<error>Test file '$test' does not exist.</error>" );
-
-					return Command::INVALID;
+			// Validate WooExtension.
+			try {
+				if ( is_numeric( $woo_extension ) ) {
+					$woo_extension    = $this->woo_extensions_list->get_woo_extension_slug_by_id( $woo_extension );
+					$woo_extension_id = $woo_extension;
+				} else {
+					$woo_extension_id = $this->woo_extensions_list->get_woo_extension_id_by_slug( $woo_extension );
 				}
-				$woo_extension = sprintf( '%s:test:%s', $woo_extension, realpath( $test ) );
-			} else {
-				$has_action = false;
+			} catch ( \Exception $e ) {
+				$output->writeln( sprintf( '<error>%s</error>', $e->getMessage() ) );
 
-				foreach ( Extension::ACTIONS as $action ) {
-					if ( strpos( $woo_extension, ":$action" ) !== false ) {
-						$has_action = true;
-						break;
-					}
-				}
+				return Command::INVALID;
+			}
 
-				if ( ! $has_action ) {
-					$woo_extension = "$woo_extension:test";
-				}
+			if ( ! empty( $test ) ) {
+				$woo_extension_extension_syntax = sprintf( '%s:%s:base64%s', $woo_extension, $sut_action, base64_encode( $test ) );
+			} else {
+				$woo_extension_extension_syntax = sprintf( '%s:%s', $woo_extension, $sut_action );
 			}
 
 			if ( $input->getOption( 'testing_theme' ) ) {
-				$env_up_options['--theme'][] = $woo_extension;
+				$env_up_options['--theme'][] = $woo_extension_extension_syntax;
 			} else {
-				$env_up_options['--plugin'][] = $woo_extension;
+				$env_up_options['--plugin'][] = $woo_extension_extension_syntax;
 			}
 		}
 
@@ -240,6 +262,11 @@ protected function execute( InputInterface $input, OutputInterface $output ): in
 
 		putenv( 'QIT_UP_AND_TEST=1' ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.runtime_configuration_putenv
 		putenv( sprintf( 'QIT_SUT=%s', $woo_extension ) ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.runtime_configuration_putenv
+		if ( ! empty( $woo_extension_id ) ) {
+			App::setVar( 'QIT_SUT', (int) $woo_extension_id );
+		}
+
+		App::setVar( 'should_upload_report', ! $input->getOption( 'no_upload_report' ) );
 
 		$up_exit_status_code = $env_up_command->run(
 			new ArrayInput( $env_up_options ),
@@ -259,6 +286,14 @@ protected function execute( InputInterface $input, OutputInterface $output ): in
 		/** @var E2EEnvInfo $env_info */
 		$env_info = E2EEnvInfo::from_array( $env_json );
 
+		App::singleton( E2EEnvInfo::class, $env_info );
+
+		if ( ! empty( $woo_extension_id ) ) {
+			$env_info->sut_slug = $woo_extension;
+			$env_info->sut_id   = $woo_extension_id;
+			$this->test_run_notifier->notify_test_started( $woo_extension_id, $woocommerce_version ?? 'none', $env_info );
+		}
+
 		// Store in $GLOBALS so that's available in the shutdown function.
 		$GLOBALS['env_to_shutdown'] = $env_info;
 
@@ -271,11 +306,15 @@ protected function execute( InputInterface $input, OutputInterface $output ): in
 
 		$exit_status_code = $this->e2e_test_manager->run_tests( $env_info, $test_mode, $wait, $shard );
 
+		$io = new SymfonyStyle( $input, $output );
+
 		if ( $exit_status_code === Command::SUCCESS ) {
+			$io->success( "Tests passed. Run 'qit e2e-report' to view the report." );
+
 			return Command::SUCCESS;
 		} else {
 			if ( $test_mode === E2ETestManager::$test_modes['headless'] ) {
-				$this->output->writeln( sprintf( '<error>Tests failed. Exit status code: %s</error>', $exit_status_code ) );
+				$io->error( "Tests failed. Run 'qit e2e-report' to view the report." );
 			}
 
 			return Command::FAILURE;

qit

@@ -28,36 +28,53 @@ public function __construct( Cache $cache ) {
 	protected function configure() {
 		$this
 			->addArgument( 'report_dir', InputArgument::OPTIONAL, '(Optional) The report directory. If not set, will show the last report.' )
+			->addOption( 'local', null, null, 'Force showing the local report instead of the remote one.' )
 			->addOption( 'dir_only', null, null, 'Only show the report directory.' )
 			->setDescription( 'Shows a test report.' );
 	}
 
 	protected function execute( InputInterface $input, OutputInterface $output ): int {
 		if ( ! is_null( $input->getArgument( 'report_dir' ) ) ) {
-			$report_dir = $input->getArgument( 'report_dir' );
+			$local_report = $input->getArgument( 'report_dir' );
 		} else {
-			$report_dir = $this->cache->get( 'last_e2e_report' );
+			$report_dir = json_decode( $this->cache->get( 'last_e2e_report' ) ?: '', true );
+
+			if ( empty( $report_dir ) ) {
+				$output->writeln( 'No report found.' );
+
+				return Command::FAILURE;
+			}
+
+			$local_report  = $report_dir['local_playwright'];
+			$remote_report = $report_dir['remote_qit'];
+		}
+
+		// If it has both local and remote, show the remote, unless "force-local" is true.
+		if ( ! $input->getOption( 'local' ) && ! empty( $remote_report ) ) {
+			open_in_browser( $remote_report );
+
+			return Command::SUCCESS;
 		}
 
-		if ( ! file_exists( $report_dir ) ) {
-			throw new \RuntimeException( sprintf( 'Could not find the report directory: %s', $report_dir ) );
+		if ( ! file_exists( $local_report ) ) {
+			throw new \RuntimeException( sprintf( 'Could not find the report directory: %s', $local_report ) );
 		}
 
-		if ( ! file_exists( $report_dir . '/index.html' ) ) {
-			throw new \RuntimeException( sprintf( 'Could not find the report file: %s', $report_dir . '/index.html' ) );
+		if ( ! file_exists( $local_report . '/index.html' ) ) {
+			throw new \RuntimeException( sprintf( 'Could not find the report file: %s', $local_report . '/index.html' ) );
 		}
 
 		if ( $input->getOption( 'dir_only' ) ) {
 			// We usually want the "HTML" report, but here print the general result directory.
-			$report_dir = dirname( $report_dir );
+			$local_report = dirname( $local_report );
 
-			$output->writeln( $report_dir );
+			$output->writeln( $local_report );
 
 			return Command::SUCCESS;
 		}
 
 		try {
-			$port = $this->start_server( $report_dir );
+			$port = $this->start_server( $local_report );
 			echo "Server started on port: $port\n"; // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped
 		} catch ( \RuntimeException $e ) {
 			echo 'Error: ' . $e->getMessage() . "\n"; // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped

src/src/Commands/CustomTests/RunE2ECommand.php

@@ -198,6 +198,74 @@ public function run_inside_docker( EnvInfo $env_info, array $command, array $env
 		}
 	}
 
+	/**
+	 * Copy a file from inside a Docker container to a local path in the filesystem.
+	 *
+	 * @param EnvInfo $env_info
+	 * @param string  $container_path The file path inside the Docker container.
+	 * @param string  $local_path The destination path on the local filesystem.
+	 * @param string  $image The Docker image to use.
+	 *
+	 * @return void
+	 * @throws \RuntimeException If the file does not exist inside the Docker container.
+	 */
+	public function copy_from_docker( EnvInfo $env_info, string $container_path, string $local_path, string $image = 'php' ): void {
+		$docker_container = $env_info->get_docker_container( $image );
+		$docker_command   = [ $this->find_docker(), 'exec', $docker_container, 'test', '-f', $container_path ];
+
+		$process = new Process( $docker_command );
+		$process->setTimeout( 30 );
+		$process->setIdleTimeout( 30 );
+
+		if ( $this->output->isVeryVerbose() ) {
+			$this->output->writeln( $process->getCommandLine() );
+		}
+
+		$process->run();
+
+		if ( ! $process->isSuccessful() ) {
+			throw new \RuntimeException( "File $container_path does not exist in container $docker_container." );
+		}
+
+		$docker_command = [ $this->find_docker(), 'cp', "$docker_container:$container_path", $local_path ];
+
+		$process = new Process( $docker_command );
+		$process->setTimeout( 30 );
+		$process->setIdleTimeout( 30 );
+
+		if ( $this->output->isVeryVerbose() ) {
+			$this->output->writeln( $process->getCommandLine() );
+		}
+
+		$process->run( function ( $type, $buffer ) {
+			if ( $this->output->isVerbose() ) {
+				$this->output->write( $buffer );
+			}
+		} );
+
+		if ( ! $process->isSuccessful() ) {
+			$exit_code    = $process->getExitCode();
+			$output       = $process->getOutput();
+			$error_output = $process->getErrorOutput();
+
+			$message = "Failed to copy file from Docker container (Container $docker_container exited with $exit_code).";
+
+			if ( ! empty( $output ) ) {
+				$message .= "\n" . $output;
+			}
+
+			if ( ! empty( $error_output ) ) {
+				$message .= "\n" . $error_output;
+			}
+
+			if ( $this->output->isVerbose() ) {
+				$message .= "\n" . 'Command that was executed: ' . $process->getCommandLine();
+			}
+
+			throw new \RuntimeException( $message );
+		}
+	}
+
 	/**
 	 * @return bool Whether the user should be set in the docker command.
 	 */