Test Orchestration Integration For Nightwatch SDK (#41)

* feat: Implement Nightwatch Test Orchestration Module

- Added helpers.js for utility functions related to test orchestration.
- Created index.js to serve as the main entry point for the orchestration module.
- Introduced orchestrationUtils.js for managing orchestration settings and operations.
- Developed requestUtils.js for handling API requests to the BrowserStack orchestration API.
- Implemented testOrchestrationHandler.js to manage test orchestration operations.
- Added testOrchestrationIntegration.js for integrating orchestration with Nightwatch.
- Created testOrderingServer.js to handle communication with the BrowserStack server for test ordering.
- Updated requestHelper.js to improve error handling for API requests.

* feat: Enhance test file collection and orchestration logic with improved glob pattern matching

* feat: Refactor test orchestration logic to apply specific feature path for cucumber tests

* feat: Update cucumber feature path for API tests in Nightwatch globals

* feat: Update cucumber feature path handling to use ordered files

* feat: Add framework name to build data payload in OrchestrationUtils

* feat: Remove unused helper functions and debug logs from orchestration files

* feat: Update framework name comment in build data payload and enhance error logging in RequestUtils

* feat: Remove unused orchestration files and refactor helper functions for improved Git metadata handling

* style: eslint update

* feat: Remove framework name from build data payload in OrchestrationUtils

Author: harshit-bstack

nightwatch/globals.js

@@ -8,6 +8,8 @@ const {v4: uuidv4} = require('uuid');
 const path = require('path');
 const AccessibilityAutomation = require('../src/accessibilityAutomation');
 const eventHelper = require('../src/utils/eventHelper');
+const OrchestrationUtils = require('../src/testorchestration/orchestrationUtils');
+const {type} = require('os');
 const localTunnel = new LocalTunnel();
 const testObservability = new TestObservability();
 const accessibilityAutomation = new AccessibilityAutomation();
@@ -270,7 +272,7 @@ module.exports = {
     }
   },
 
-  async before(settings) {
+  async before(settings, testEnvSettings) {
     if (!settings.desiredCapabilities['bstack:options']) {
       settings.desiredCapabilities['bstack:options'] = {};
     }
@@ -322,6 +324,91 @@ module.exports = {
     } catch (error) {
       Logger.error(`Could not configure or launch test reporting and analytics - ${error}`);
     }
+    
+    // Initialize and configure test orchestration
+    try {
+      const orchestrationUtils = OrchestrationUtils.getInstance(settings);
+      if (orchestrationUtils && orchestrationUtils.testOrderingEnabled()) {        
+        // Apply test orchestration to reorder test files before execution
+        const TestOrchestrationIntegration = require('../src/testorchestration/testOrchestrationIntegration');
+        const orchestrationIntegration = TestOrchestrationIntegration.getInstance();
+        orchestrationIntegration.configure(settings);
+                
+        // Check if we have test files to reorder from various sources
+        let allTestFiles = [];
+        
+        // Checking either for Feature Path or src_folders, feature path take priority
+        if (helper.isCucumberTestSuite(settings) && settings?.test_runner?.options?.feature_path){
+          Logger.debug('Getting test files from feature_path configuration...');
+          if (Array.isArray(settings.test_runner.options.feature_path)){
+            settings.test_runner.options.feature_path.forEach(featurePath => {
+              const files = helper.collectTestFiles(featurePath, 'feature_path config');
+              allTestFiles = allTestFiles.concat(files);
+            });
+          } else if (typeof settings.test_runner.options.feature_path === 'string'){
+            const files = helper.collectTestFiles(settings.test_runner.options.feature_path, 'feature_path config');
+            allTestFiles = allTestFiles.concat(files);
+          }
+        } else if (settings.src_folders && Array.isArray(settings.src_folders) && settings.src_folders.length > 0) {
+          Logger.debug('Getting test files from src_folders configuration...');
+          settings.src_folders.forEach(folder => {
+            const files = helper.collectTestFiles(folder, 'src_folders config');
+            allTestFiles = allTestFiles.concat(files);
+          });
+        }
+
+        // Remove duplicates and ensure all paths are relative to cwd
+        allTestFiles = [...new Set(allTestFiles)].map(file => {
+          return path.isAbsolute(file) ? path.relative(process.cwd(), file) : file;
+        });
+
+
+        if (allTestFiles.length > 0) {
+          Logger.info(`Applying test orchestration to reorder test files... Found ${allTestFiles.length} test files`);
+          Logger.debug(`Test files: ${JSON.stringify(allTestFiles)}`);
+          
+          // Apply orchestration to get ordered test files (synchronously)
+          try {
+            const orderedFiles = await orchestrationIntegration.applyOrchestration(allTestFiles, settings);
+            if (orderedFiles && orderedFiles.length > 0) {
+              Logger.info(`Test files reordered by orchestration: ${orderedFiles.length} files`);
+
+              Logger.info('Test orchestration recommended order change:');
+              Logger.info(`Original: ${allTestFiles.join(', ')}`);
+              Logger.info(`Optimized: ${orderedFiles.join(', ')}`);
+                                
+              try {
+                if (helper.isCucumberTestSuite(settings) && settings?.test_runner?.options?.feature_path){
+                  // For cucumber, we override the feature_path option with ordered files
+                  settings.test_runner.options['feature_path'] = orderedFiles;
+                } else {
+                  settings.src_folders = orderedFiles;
+                  for (const envName in testEnvSettings) {
+                    testEnvSettings[envName].src_folders = orderedFiles;
+                    testEnvSettings[envName].test_runner.src_folders = orderedFiles;
+                  }
+                  if (settings.test_runner && typeof settings.test_runner === 'object' && !Array.isArray(settings.test_runner)) {
+                    settings.test_runner.src_folders = orderedFiles;
+                  }
+                }
+                  
+              } catch (reorderError) {
+                Logger.error(`Runtime reordering failed: ${reorderError.message}`);
+                Logger.info('Falling back to original order for current execution.');
+              } 
+            } else {
+              Logger.info('Split test API called - no reordering available');
+            }
+          } catch (error) {
+            Logger.error(`Error applying test orchestration: ${error}`);
+          }
+        } else {
+          Logger.debug('No test files found for orchestration - skipping split test API call');
+        }
+      }
+    } catch (error) {
+      Logger.error(`Could not configure test orchestration - ${error}`);
+    }
 
     try {
       accessibilityAutomation.configure(settings);
@@ -344,6 +431,18 @@ module.exports = {
 
   async after() {
     localTunnel.stop();
+    
+    // Collect build data for test orchestration if enabled
+    try {
+      const orchestrationUtils = OrchestrationUtils.getInstance();
+      if (orchestrationUtils && orchestrationUtils.testOrderingEnabled()) {
+        Logger.info('Collecting build data for test orchestration...');
+        await orchestrationUtils.collectBuildData(this.settings || {});
+      }
+    } catch (error) {
+      Logger.error(`Error collecting build data for test orchestration: ${error}`);
+    }
+    
     if (helper.isTestObservabilitySession()) {
       process.env.NIGHTWATCH_RERUN_FAILED = nightwatchRerun;
       process.env.NIGHTWATCH_RERUN_REPORT_FILE = nightwatchRerunFile;

src/testorchestration/applyOrchestration.js

@@ -0,0 +1,70 @@
+const {performance} = require('perf_hooks');
+const Logger = require('../utils/logger');
+const TestOrchestrationHandler = require('./testOrchestrationHandler');
+
+/**
+ * Applies test orchestration to the Nightwatch test run
+ * This function is the main entry point for the orchestration integration
+ */
+async function applyOrchestrationIfEnabled(specs, config) {
+  // Initialize orchestration handler
+  const orchestrationHandler = TestOrchestrationHandler.getInstance(config);
+  Logger.info('Orchestration handler is initialized');
+  
+  if (!orchestrationHandler) {
+    Logger.warn('Orchestration handler is not initialized. Skipping orchestration.');
+
+    return specs;
+  }
+
+  // Check if runSmartSelection is enabled in config
+  const testOrchOptions = config.testOrchestrationOptions || config['@nightwatch/browserstack']?.testOrchestrationOptions || {};
+  const runSmartSelectionEnabled = Boolean(testOrchOptions?.runSmartSelection?.enabled);
+  
+  if (!runSmartSelectionEnabled) {
+    Logger.info('runSmartSelection is not enabled in config. Skipping orchestration.');
+
+    return specs;
+  }
+
+  // Check if orchestration is enabled
+  let testOrderingApplied = false;
+  orchestrationHandler.addToOrderingInstrumentationData('enabled', orchestrationHandler.testOrderingEnabled());
+  
+  const startTime = performance.now();
+  
+  Logger.info('Test orchestration is enabled. Attempting to reorder test files.');
+  
+  // Get the test files from the specs
+  const testFiles = specs;
+  testOrderingApplied = true;
+  Logger.info(`Test files to be reordered: ${testFiles.join(', ')}`);
+  
+  // Reorder the test files
+  const orderedFiles = await orchestrationHandler.reorderTestFiles(testFiles);
+  
+  if (orderedFiles && orderedFiles.length > 0) {
+    orchestrationHandler.setTestOrderingApplied(testOrderingApplied);
+    Logger.info(`Tests reordered using orchestration: ${orderedFiles.join(', ')}`);
+    
+    // Return the ordered files as the new specs
+    orchestrationHandler.addToOrderingInstrumentationData(
+      'timeTakenToApply', 
+      Math.floor(performance.now() - startTime) // Time in milliseconds
+    );
+    
+    return orderedFiles;
+  } 
+  Logger.info('No test files were reordered by orchestration.');
+  orchestrationHandler.addToOrderingInstrumentationData(
+    'timeTakenToApply', 
+    Math.floor(performance.now() - startTime) // Time in milliseconds
+  );
+  
+  
+  return specs;
+}
+
+module.exports = {
+  applyOrchestrationIfEnabled
+};