Bug 2019457 - Refactor CI/CD test cases to make it easier to incorporate automated tests from extensions into the normal test framework

.github/workflows/ci.yml

@@ -1,7 +1,6 @@
 name: BMO Test Suite
 
-on:
-  pull_request
+on: pull_request
 
 jobs:
   test_sanity:
@@ -13,7 +12,7 @@ jobs:
       - name: Build Docker test images
         run: docker-compose -f docker-compose.test.yml build bmo.test
       - name: Run sanity tests
-        run: docker-compose -f docker-compose.test.yml run --no-deps bmo.test test_sanity t/*.t extensions/*/t/*.t
+        run: docker-compose -f docker-compose.test.yml run --no-deps bmo.test test_sanity -qf t/*.t extensions/*/t/*.t
 
   test_webservices:
     runs-on: ubuntu-latest
@@ -24,7 +23,10 @@ jobs:
       - name: Build Docker test images
         run: docker-compose -f docker-compose.test.yml build
       - name: Run webservice tests
-        run: docker-compose -f docker-compose.test.yml run bmo.test test_webservices
+        run: |
+          docker-compose -f docker-compose.test.yml run bmo.test test_webservices -qf \
+            qa/t/{webservice,rest}_*.t \
+            $(find extensions \( -path "*/qa/t/webservice_*.t" -o -path "*/qa/t/rest_*.t" \))
 
   test_bmo:
     runs-on: ubuntu-latest
@@ -35,7 +37,7 @@ jobs:
       - name: Build Docker test images
         run: docker-compose -f docker-compose.test.yml build
       - name: Run bmo specific tests
-        run: docker-compose -f docker-compose.test.yml run -e CI=1 bmo.test test_bmo -q -f t/bmo/*.t extensions/*/t/bmo/*.t
+        run: docker-compose -f docker-compose.test.yml run -e CI=1 bmo.test test_bmo -qf t/bmo/*.t extensions/*/t/bmo/*.t
 
   test_selenium_1:
     runs-on: ubuntu-latest
@@ -46,7 +48,10 @@ jobs:
       - name: Build Docker test images
         run: docker-compose -f docker-compose.test.yml build
       - name: Run Selenium tests (1)
-        run: docker-compose -f docker-compose.test.yml run -e SELENIUM_GROUP=1 bmo.test test_selenium
+        run: |
+          docker-compose -f docker-compose.test.yml run bmo.test test_selenium -qf \
+            qa/t/1_test_*.t \
+            $(find extensions -path "*/qa/t/1_test_*.t")
 
   test_selenium_2:
     runs-on: ubuntu-latest
@@ -57,7 +62,10 @@ jobs:
       - name: Build Docker test images
         run: docker-compose -f docker-compose.test.yml build
       - name: Run Selenium tests (2)
-        run: docker-compose -f docker-compose.test.yml run -e SELENIUM_GROUP=2 bmo.test test_selenium
+        run: |
+          docker-compose -f docker-compose.test.yml run bmo.test test_selenium -qf \
+            qa/t/2_test_*.t \
+            $(find extensions -path "*/qa/t/2_test_*.t")
 
   test_selenium_3:
     runs-on: ubuntu-latest
@@ -68,7 +76,10 @@ jobs:
       - name: Build Docker test images
         run: docker-compose -f docker-compose.test.yml build
       - name: Run Selenium tests (3)
-        run: docker-compose -f docker-compose.test.yml run -e SELENIUM_GROUP=3 bmo.test test_selenium
+        run: |
+          docker-compose -f docker-compose.test.yml run bmo.test test_selenium -qf \
+            qa/t/3_test_*.t \
+            $(find extensions -path "*/qa/t/3_test_*.t")
 
   test_selenium_4:
     runs-on: ubuntu-latest
@@ -79,4 +90,7 @@ jobs:
       - name: Build Docker test images
         run: docker-compose -f docker-compose.test.yml build
       - name: Run Selenium tests (4)
-        run: docker-compose -f docker-compose.test.yml run -e SELENIUM_GROUP=4 bmo.test test_selenium
+        run: |
+          docker-compose -f docker-compose.test.yml run bmo.test test_selenium -qf \
+            qa/t/4_test_*.t \
+            $(find extensions -path "*/qa/t/4_test_*.t")

qa/t/lib/QA/REST.pm → Bugzilla/QA/REST.pm

@@ -5,7 +5,7 @@
 # This Source Code Form is "Incompatible With Secondary Licenses", as
 # defined by the Mozilla Public License, v. 2.0.
 
-package QA::REST;
+package Bugzillas::QA::REST;
 
 use 5.10.1;
 use strict;
@@ -14,11 +14,11 @@ use autodie;
 
 use LWP::UserAgent;
 use JSON;
-use QA::Util;
+use Bugzilla::QA::Util;
 
 use parent qw(LWP::UserAgent Exporter);
 
-@QA::REST::EXPORT = qw(
+@Bugzilla::QA::REST::EXPORT = qw(
   MUST_FAIL
   get_rest_client
 );
@@ -27,7 +27,7 @@ use constant MUST_FAIL => 1;
 
 sub get_rest_client {
   my $rest_client = LWP::UserAgent->new(ssl_opts => {verify_hostname => 0});
-  bless($rest_client, 'QA::REST');
+  bless($rest_client, 'Bugzilla::QA::REST');
   my $config = $rest_client->{bz_config} = get_config();
   $rest_client->{bz_url}
     = $config->{browser_url} . '/rest/';

qa/t/lib/QA/RPC.pm → Bugzilla/QA/RPC.pm

@@ -7,11 +7,15 @@
 
 # -*- Mode: perl; indent-tabs-mode: nil -*-
 
-package QA::RPC;
+package Bugzilla::QA::RPC;
+
+use 5.10.1;
 use strict;
+use warnings;
+
 use Data::Dumper;
-use QA::Util;
-use QA::Tests qw(PRIVATE_BUG_USER create_bug_fields);
+use Bugzilla::QA::Util;
+use Bugzilla::QA::Tests qw(PRIVATE_BUG_USER create_bug_fields);
 use Storable qw(dclone);
 use Test::More;
 
@@ -278,7 +282,7 @@ sub bz_test_bug {
       my $creation_day;
 
       # XML-RPC and JSON-RPC have different date formats.
-      if ($self->isa('QA::RPC::XMLRPC')) {
+      if ($self->isa('Bugzilla::QA::RPC::XMLRPC')) {
         $creation_day = $creation_time->ymd('');
       }
       else {

qa/t/lib/QA/RPC/JSONRPC.pm → Bugzilla/QA/RPC/JSONRPC.pm

@@ -7,13 +7,16 @@
 
 # -*- Mode: perl; indent-tabs-mode: nil -*-
 
-package QA::RPC::JSONRPC;
+package Bugzilla::QA::RPC::JSONRPC;
+
+use 5.10.1;
 use strict;
+use warnings;
 
-use QA::RPC;
+use Bugzilla::QA::RPC;
 
 BEGIN {
-  our @ISA = qw(QA::RPC);
+  our @ISA = qw(Bugzilla::QA::RPC);
 
   if (eval { require JSON::RPC::Client }) {
     push(@ISA, 'JSON::RPC::Client');
@@ -39,8 +42,8 @@ sub TYPE {
 
 sub ua {
   my $self = shift;
-  if ($self->{ua} and not $self->{ua}->isa('QA::RPC::UserAgent')) {
-    bless $self->{ua}, 'QA::RPC::UserAgent';
+  if ($self->{ua} and not $self->{ua}->isa('Bugzilla::QA::RPC::UserAgent')) {
+    bless $self->{ua}, 'Bugzilla::QA::RPC::UserAgent';
   }
   return $self->SUPER::ua(@_);
 }
@@ -97,7 +100,7 @@ sub call {
   }
 
   if ($result) {
-    bless $result, 'QA::RPC::JSONRPC::ReturnObject';
+    bless $result, 'Bugzilla::QA::RPC::JSONRPC::ReturnObject';
   }
   return $result;
 }
@@ -123,7 +126,7 @@ sub _get {
 
 1;
 
-package QA::RPC::JSONRPC::ReturnObject;
+package Bugzilla::QA::RPC::JSONRPC::ReturnObject;
 use strict;
 
 BEGIN {
@@ -146,7 +149,7 @@ sub fault       { $_[0]->is_error }
 
 1;
 
-package QA::RPC::UserAgent;
+package Bugzilla::QA::RPC::UserAgent;
 use strict;
 use base qw(LWP::UserAgent);
 

qa/t/lib/QA/RPC/XMLRPC.pm → Bugzilla/QA/RPC/XMLRPC.pm

@@ -7,9 +7,13 @@
 
 # -*- Mode: perl; indent-tabs-mode: nil -*-
 
-package QA::RPC::XMLRPC;
+package Bugzilla::QA::RPC::XMLRPC;
+
+use 5.10.1;
 use strict;
-use base qw(QA::RPC XMLRPC::Lite);
+use warnings;
+
+use base qw(Bugzilla::QA::RPC XMLRPC::Lite);
 
 use constant TYPE           => 'XML-RPC';
 use constant DATETIME_REGEX => qr/^\d{8}T\d\d:\d\d:\d\d$/;

qa/t/lib/QA/Tests.pm → Bugzilla/QA/Tests.pm

@@ -7,8 +7,12 @@
 
 # -*- Mode: perl; indent-tabs-mode: nil -*-
 
-package QA::Tests;
+package Bugzilla::QA::Tests;
+
+use 5.10.1;
 use strict;
+use warnings;
+
 use base qw(Exporter);
 our @EXPORT_OK = qw(
   PRIVATE_BUG_USER

qa/t/lib/QA/Util.pm → Bugzilla/QA/Util.pm

@@ -7,9 +7,12 @@
 
 # -*- Mode: perl; indent-tabs-mode: nil -*-
 
-package QA::Util;
+package Bugzilla::QA::Util;
 
+use 5.10.1;
 use strict;
+use warnings;
+
 use Data::Dumper;
 use Test::More;
 use Test::WWW::Selenium;
@@ -34,7 +37,7 @@ BEGIN {
 }
 
 use base qw(Exporter);
-@QA::Util::EXPORT = qw(
+@Bugzilla::QA::Util::EXPORT = qw(
   trim
   url_quote
   random_string
@@ -65,9 +68,8 @@ use base qw(Exporter);
 );
 
 # How long we wait for pages to load.
-use constant WAIT_TIME => 60000;
-use constant CONF_FILE => $ENV{BZ_QA_CONF_FILE}
-  // "../config/selenium_test.conf";
+use constant WAIT_TIME   => 60000;
+use constant CONF_FILE   => $ENV{BZ_QA_CONF_FILE} // "conf/selenium_test.conf";
 use constant CHROME_MODE => 1;
 
 #####################
@@ -154,15 +156,15 @@ sub get_xmlrpc_client {
     = $config->{browser_url}
     . "/xmlrpc.cgi";
 
-  require QA::RPC::XMLRPC;
-  my $rpc = new QA::RPC::XMLRPC(proxy => $xmlrpc_url);
+  require Bugzilla::QA::RPC::XMLRPC;
+  my $rpc = new Bugzilla::QA::RPC::XMLRPC(proxy => $xmlrpc_url);
   return ($rpc, $config);
 }
 
 sub get_jsonrpc_client {
   my ($get_mode) = @_;
-  require QA::RPC::JSONRPC;
-  my $rpc = new QA::RPC::JSONRPC();
+  require Bugzilla::QA::RPC::JSONRPC;
+  my $rpc = new Bugzilla::QA::RPC::JSONRPC();
 
   # If we don't set a long timeout, then the Bug.add_comment test
   # where we add a too-large comment fails.
@@ -325,7 +327,6 @@ sub go_to_bug {
   my $bug_title = $sel->get_title();
   utf8::encode($bug_title) if utf8::is_utf8($bug_title);
   $sel->title_like(qr/^$bug_id /, $bug_title);
-  sleep(1); # FIXME: Sometimes we try to click edit bug before it is ready so wait a second
   $sel->click_ok('mode-btn-readonly', 'Click Edit Bug') if !$no_edit;
   $sel->click_ok('action-menu-btn', 'Expand action menu');
   $sel->click_ok('action-expand-all', 'Expand all modal panels');

t/Support/Files.pm → Bugzilla/Test/Files.pm

@@ -6,7 +6,7 @@
 # defined by the Mozilla Public License, v. 2.0.
 
 
-package Support::Files;
+package Bugzilla::Test::Files;
 
 use 5.10.1;
 use strict;

t/Support/Systemexec.pm → Bugzilla/Test/Systemexec.pm

@@ -5,14 +5,14 @@
 # This Source Code Form is "Incompatible With Secondary Licenses", as
 # defined by the Mozilla Public License, v. 2.0.
 
-package Support::Systemexec;
+package Bugzilla::Test::Systemexec;
 
 use 5.10.1;
 use strict;
 use warnings;
 
 use parent qw(Exporter);
-@Support::Systemexec::EXPORT = qw(system exec);
+@Bugzilla::Test::Systemexec::EXPORT = qw(system exec);
 
 sub system($$@) {
   1;

t/Support/Templates.pm → Bugzilla/Test/Templates.pm

@@ -5,22 +5,22 @@
 # This Source Code Form is "Incompatible With Secondary Licenses", as
 # defined by the Mozilla Public License, v. 2.0.
 
-package Support::Templates;
+package Bugzilla::Test::Templates;
 
 use 5.10.1;
 use strict;
 use warnings;
 
 use lib 't';
 use base qw(Exporter);
-@Support::Templates::EXPORT
+@Bugzilla::Test::Templates::EXPORT
   = qw(@languages @include_paths @english_default_include_paths
   %include_path @referenced_files %actual_files $num_actual_files);
 
 use Bugzilla;
 use Bugzilla::Constants;
 use Bugzilla::Install::Util qw(template_include_path);
-use Support::Files;
+use Bugzilla::Test::Files;
 
 use File::Find;
 use File::Spec;
@@ -30,7 +30,7 @@ our @english_default_include_paths
   = (File::Spec->catdir(bz_locations()->{'templatedir'}, 'en', 'default'));
 
 # And the extensions too
-foreach my $extension (@Support::Files::extensions) {
+foreach my $extension (@Bugzilla::Test::Files::extensions) {
   my $dir = File::Spec->catdir($extension, 'template', 'en', 'default');
   if (-e $dir) {
     push @english_default_include_paths, $dir;
@@ -100,7 +100,7 @@ foreach my $include_path (@include_paths) {
 # in the @referenced_files array to be used by the 004template.t test.
 my %seen;
 
-foreach my $file (@Support::Files::testitems) {
+foreach my $file (@Bugzilla::Test::Files::testitems) {
   open(FILE, $file);
   my @lines = <FILE>;
   close(FILE);

qa/config/selenium_test.conf → conf/selenium_test.conf

@@ -14,7 +14,7 @@
 {   'browser'                           => '*firefox',
     'experimental_browser_launcher'     => '*chrome',
     'browser_url'                       => 'http://bmo.test',
-    'attachment_file'                   => '/app/qa/config/patch.diff',
+    'attachment_file'                   => '/app/conf/patch.diff',
     'bugzilla_path'                     => '/app',
     'test_bug_1'                        => 1,
     'test_bug_2'                        => 2,

docker-compose.test.yml

@@ -25,7 +25,7 @@ services:
       - BMO_urlbase=AUTOMATIC
       - BUGZILLA_ALLOW_INSECURE_HTTP=1
       - BZ_ANSWERS_FILE=/app/conf/checksetup_answers.txt
-      - BZ_QA_CONF_FILE=/app/qa/config/selenium_test.conf
+      - BZ_QA_CONF_FILE=/app/conf/selenium_test.conf
       - BZ_QA_CONFIG=1
       - CI=1
       - LOCALCONFIG_ENV=1