Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions .github/workflows/browserstack.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: BrowserStack Tests

on:
push:
branches: [main]
pull_request:
workflow_dispatch:

jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
include:
- platform: android
device: 'Samsung Galaxy S23'
os_version: '13.0'
app: ./path/to/android-app.apk
- platform: ios
device: 'iPhone 15'
os_version: '17'
app: ./path/to/ios-app.ipa
env:
BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
TEST_PATH: tests/sample_login.json
AI_SERVICE: gemini
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 18
cache: 'npm'
cache-dependency-path: backend/package-lock.json
- name: Install backend dependencies
working-directory: backend
run: npm ci
- name: Run BrowserStack tests
id: run-tests
working-directory: backend
env:
APP_PATH: ${{ matrix.app }}
PLATFORM: ${{ matrix.platform }}
DEVICE_NAME: ${{ matrix.device }}
OS_VERSION: ${{ matrix.os_version }}
run: npm run test:browserstack
- name: Collect BrowserStack artifacts
if: ${{ steps.run-tests.outputs.session_id }}
working-directory: backend
env:
SESSION_ID: ${{ steps.run-tests.outputs.session_id }}
BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
run: |
mkdir -p artifacts
curl -u "$BROWSERSTACK_USERNAME:$BROWSERSTACK_ACCESS_KEY" \
"https://api.browserstack.com/app-automate/sessions/$SESSION_ID.json" -o session.json
VIDEO_URL=$(jq -r '.automation_session.video_url' session.json)
LOG_URL=$(jq -r '.automation_session.appium_logs_url' session.json)
curl -u "$BROWSERSTACK_USERNAME:$BROWSERSTACK_ACCESS_KEY" "$VIDEO_URL" -o artifacts/${SESSION_ID}-video.mp4
curl -u "$BROWSERSTACK_USERNAME:$BROWSERSTACK_ACCESS_KEY" "$LOG_URL" -o artifacts/${SESSION_ID}-appium.log
echo "### BrowserStack Session $SESSION_ID" >> $GITHUB_STEP_SUMMARY
echo "- [Video]($VIDEO_URL)" >> $GITHUB_STEP_SUMMARY
echo "- [Appium Log]($LOG_URL)" >> $GITHUB_STEP_SUMMARY
- uses: actions/upload-artifact@v4
if: ${{ steps.run-tests.outputs.session_id }}
with:
name: browserstack-${{ matrix.platform }}-${{ matrix.os_version }}
path: backend/artifacts
3 changes: 2 additions & 1 deletion backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
"main": "src/app.js",
"scripts": {
"start": "node src/app.js",
"dev": "nodemon src/app.js"
"dev": "nodemon src/app.js",
"test:browserstack": "node scripts/run-browserstack-tests.js"
},
"keywords": [
"appium",
Expand Down
43 changes: 43 additions & 0 deletions backend/scripts/run-browserstack-tests.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const { executeTest } = require('../src/test-runner/test_executor');

async function main() {
const appPath = process.env.APP_PATH;
const testsPath = process.env.TEST_PATH;
const platform = process.env.PLATFORM || 'android';
const deviceName = process.env.DEVICE_NAME || '';
const osVersion = process.env.OS_VERSION || '';
const aiService = process.env.AI_SERVICE || 'gemini';

if (!appPath || !testsPath) {
console.error('APP_PATH and TEST_PATH environment variables are required');
process.exit(1);
}

const rawStepsText = fs.readFileSync(path.resolve(testsPath), 'utf8');

const sessionId = await executeTest(
path.resolve(appPath),
rawStepsText,
{ to: () => ({ emit: () => {} }) },
'cli',
aiService,
'browserstack',
platform,
deviceName,
osVersion,
);

console.log(`BrowserStack session ID: ${sessionId}`);

if (process.env.GITHUB_OUTPUT) {
fs.appendFileSync(process.env.GITHUB_OUTPUT, `session_id=${sessionId}\n`);
Comment on lines +21 to +36

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Fail the CLI when test execution fails

The new BrowserStack runner awaits executeTest and then exits successfully regardless of whether the test actually passes. executeTest catches all exceptions and resolves (lines 581‑585 in test_executor.js), so when a test fails the function returns null or an unfinished session ID but never rejects. As a result, the Run BrowserStack tests job will report success and still emit a session ID even when the underlying test suite failed, letting CI pipelines go green erroneously. The script should treat a missing/failed run as an error—either by propagating the exception from executeTest or by checking its result and exiting with a non‑zero code—so that workflow failures surface in CI.

Useful? React with 👍 / 👎.

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@codex fix comments

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Summary

Testing

  • node --check backend/scripts/run-browserstack-tests.js
  • node --check backend/src/test-runner/test_executor.js

View task →

}
}

main().catch((err) => {
console.error(err);
process.exit(1);
});
7 changes: 4 additions & 3 deletions backend/src/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ config.testsDir = resolveRelativePath(process.env.TESTS_DIR, defaultTestsDir);
// Maximum upload size (in megabytes). Values from process.env are
// coerced to integers; invalid values fall back to 50MB.
config.maxUploadMb = (() => {
const raw = process.env.MAX_UPLOAD_MB;
//const raw = process.env.MAX_UPLOAD_MB;
const raw = 100; // default to 100MB
const parsed = parseInt(raw, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : 50;
})();
Expand All @@ -53,8 +54,8 @@ config.maxUploadMb = (() => {
config.allowedOrigin = process.env.ALLOWED_ORIGIN || '*';

// Should uploaded APKs be cleaned up when a test completes? Accept
// 'true' or 'false' strings; any other value is treated as false.
config.cleanUploadsAfterTest = (process.env.CLEAN_UPLOADS_AFTER_TEST || 'false').toLowerCase() === 'true';
// 'true' or 'false' strings; any other value is treated as true.
config.cleanUploadsAfterTest = (process.env.CLEAN_UPLOADS_AFTER_TEST || 'true').toLowerCase() === 'false';

// AI service keys. These are required for the NLP service to run.
config.geminiApiKey = process.env.GEMINI_API_KEY;
Expand Down
4 changes: 4 additions & 0 deletions backend/src/test-runner/test_executor.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ async function executeTest(
platformVersion = '',
) {
let browser;
let sessionId = null;
// Normalise the platform once so that helper functions can use it. This
// variable persists through the lifetime of the test execution.
const targetPlatform = (platform || 'android').toLowerCase();
Expand Down Expand Up @@ -131,7 +132,9 @@ async function executeTest(

console.log('Attempting to start remote session...');
browser = await remote({ ...appiumOptions, capabilities });
sessionId = browser.sessionId;
console.log('Remote session started successfully.');
console.log(`BrowserStack session ID: ${sessionId}`);

// --- NEW: Identify the app and prepare its specific cache ---
// Use appPackage for Android or bundleId for iOS when caching selectors.
Expand Down Expand Up @@ -408,6 +411,7 @@ async function executeTest(
console.error('Failed to delete uploaded app file:', fileCleanupError);
}
}
return sessionId;
}


Expand Down
2 changes: 1 addition & 1 deletion tests/sample_login.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@
"Verify that Toggle with option *Pickup* is available",
"Verify that *search* textbox is visible with placeholder text as ‘What can we get you?’ Is available on home page"
]
}
}
Loading