diff --git a/browser_tests/customTestRunner.ts b/browser_tests/customTestRunner.ts index 85b4ac37f..e81833679 100644 --- a/browser_tests/customTestRunner.ts +++ b/browser_tests/customTestRunner.ts @@ -3,9 +3,9 @@ * Custom test runner for optimized sharding * This script determines which tests to run based on shard configuration */ - import { spawn } from 'child_process' -import { getShardTests, NO_SHARD_PROJECTS } from './shardConfig' + +import { NO_SHARD_PROJECTS, getShardTests } from './shardConfig' const projectName = process.env.PLAYWRIGHT_PROJECT || 'chromium' const shardInfo = process.env.PLAYWRIGHT_SHARD @@ -24,12 +24,16 @@ if (shardInfo) { if (NO_SHARD_PROJECTS.includes(projectName)) { // For projects that don't need sharding, only run on shard 1 if (shardIndex > 1) { - console.log(`Skipping shard ${shardIndex}/${totalShards} for project ${projectName} (no sharding needed)`) + console.log( + `Skipping shard ${shardIndex}/${totalShards} for project ${projectName} (no sharding needed)` + ) process.exit(0) } console.log(`Running all tests for project ${projectName} (no sharding)`) } else { - console.log(`Running shard ${shardIndex}/${totalShards} for project ${projectName}`) + console.log( + `Running shard ${shardIndex}/${totalShards} for project ${projectName}` + ) } // Get the test files for this shard @@ -40,7 +44,7 @@ const args = ['playwright', 'test', `--project=${projectName}`] if (shardTests && shardTests.length > 0) { // Add specific test files for this shard - shardTests.forEach(testFile => { + shardTests.forEach((testFile) => { args.push(`browser_tests/tests/${testFile}`) }) } else if (shardTests === null) { @@ -66,4 +70,4 @@ const child = spawn('npx', args, { child.on('exit', (code) => { process.exit(code || 0) -}) \ No newline at end of file +}) diff --git a/browser_tests/scripts/optimizeSharding.js b/browser_tests/scripts/optimizeSharding.js index d6dd325d4..6c056829f 100644 --- a/browser_tests/scripts/optimizeSharding.js +++ b/browser_tests/scripts/optimizeSharding.js @@ -3,10 +3,9 @@ * Script to analyze test distribution and create optimized shard configurations * Run with: node browser_tests/scripts/optimizeSharding.js */ - +import { execSync } from 'child_process' import fs from 'fs' import path from 'path' -import { execSync } from 'child_process' import { fileURLToPath } from 'url' const __filename = fileURLToPath(import.meta.url) @@ -60,7 +59,7 @@ const TEST_WEIGHTS = { 'userSelectView.spec.ts': 10, 'versionMismatchWarnings.spec.ts': 10, 'workflowTabThumbnail.spec.ts': 10, - 'actionbar.spec.ts': 10, + 'actionbar.spec.ts': 10 } /** @@ -69,13 +68,13 @@ const TEST_WEIGHTS = { function getTestFiles() { const testsDir = path.join(__dirname, '..', 'tests') const files = [] - + function scanDir(dir, prefix = '') { const items = fs.readdirSync(dir) for (const item of items) { const fullPath = path.join(dir, item) const relativePath = prefix ? `${prefix}/${item}` : item - + if (fs.statSync(fullPath).isDirectory()) { scanDir(fullPath, relativePath) } else if (item.endsWith('.spec.ts')) { @@ -83,7 +82,7 @@ function getTestFiles() { } } } - + scanDir(testsDir) return files } @@ -93,20 +92,20 @@ function getTestFiles() { */ function createBalancedShards(testFiles, numShards) { // Create test entries with weights - const tests = testFiles.map(file => ({ + const tests = testFiles.map((file) => ({ file, weight: TEST_WEIGHTS[file] || 15 // Default weight for unknown tests })) - + // Sort tests by weight (heaviest first) tests.sort((a, b) => b.weight - a.weight) - + // Initialize shards const shards = Array.from({ length: numShards }, () => ({ tests: [], totalWeight: 0 })) - + // Distribute tests using a greedy algorithm (assign to shard with least weight) for (const test of tests) { // Find shard with minimum weight @@ -116,12 +115,12 @@ function createBalancedShards(testFiles, numShards) { minShard = shard } } - + // Add test to the lightest shard minShard.tests.push(test.file) minShard.totalWeight += test.weight } - + return shards } @@ -130,28 +129,30 @@ function createBalancedShards(testFiles, numShards) { */ function printShardConfig(shards) { console.log('\n=== Optimized Shard Configuration ===\n') - + shards.forEach((shard, index) => { console.log(`Shard ${index + 1} (weight: ${shard.totalWeight})`) console.log(' Tests:') - shard.tests.forEach(test => { + shard.tests.forEach((test) => { const weight = TEST_WEIGHTS[test] || 15 console.log(` - ${test} (weight: ${weight})`) }) console.log() }) - + // Print weight balance analysis - const weights = shards.map(s => s.totalWeight) + const weights = shards.map((s) => s.totalWeight) const maxWeight = Math.max(...weights) const minWeight = Math.min(...weights) const avgWeight = weights.reduce((a, b) => a + b, 0) / weights.length - + console.log('=== Balance Analysis ===') console.log(`Max weight: ${maxWeight}`) console.log(`Min weight: ${minWeight}`) console.log(`Avg weight: ${avgWeight.toFixed(1)}`) - console.log(`Imbalance: ${((maxWeight - minWeight) / avgWeight * 100).toFixed(1)}%`) + console.log( + `Imbalance: ${(((maxWeight - minWeight) / avgWeight) * 100).toFixed(1)}%` + ) } /** @@ -163,7 +164,11 @@ function generateConfigFile(shards) { * Generated on: ${new Date().toISOString()} */ -export const OPTIMIZED_SHARDS = ${JSON.stringify(shards.map(s => s.tests), null, 2)} +export const OPTIMIZED_SHARDS = ${JSON.stringify( + shards.map((s) => s.tests), + null, + 2 + )} export function getShardTests(shardIndex: number): string[] { return OPTIMIZED_SHARDS[shardIndex - 1] || [] @@ -173,7 +178,7 @@ export function getShardPattern(shardIndex: number): string[] { return getShardTests(shardIndex).map(test => \`**/\${test}\`) } ` - + const configPath = path.join(__dirname, '..', 'shardConfig.generated.ts') fs.writeFileSync(configPath, config) console.log(`\n✅ Generated configuration file: ${configPath}`) @@ -182,15 +187,15 @@ export function getShardPattern(shardIndex: number): string[] { // Main execution function main() { const numShards = parseInt(process.argv[2]) || 5 - + console.log(`Analyzing test distribution for ${numShards} shards...`) - + const testFiles = getTestFiles() console.log(`Found ${testFiles.length} test files`) - + const shards = createBalancedShards(testFiles, numShards) printShardConfig(shards) generateConfigFile(shards) } -main() \ No newline at end of file +main() diff --git a/browser_tests/shardConfig.generated.ts b/browser_tests/shardConfig.generated.ts index 9e3ec60b5..0654c52c6 100644 --- a/browser_tests/shardConfig.generated.ts +++ b/browser_tests/shardConfig.generated.ts @@ -5,60 +5,60 @@ export const OPTIMIZED_SHARDS = [ [ - "interaction.spec.ts", - "selectionToolbox.spec.ts", - "chatHistory.spec.ts", - "litegraphEvent.spec.ts", - "versionMismatchWarnings.spec.ts" + 'interaction.spec.ts', + 'selectionToolbox.spec.ts', + 'chatHistory.spec.ts', + 'litegraphEvent.spec.ts', + 'versionMismatchWarnings.spec.ts' ], [ - "subgraph.spec.ts", - "sidebar/workflows.spec.ts", - "primitiveNode.spec.ts", - "bottomPanelShortcuts.spec.ts", - "nodeBadge.spec.ts", - "execution.spec.ts", - "rerouteNode.spec.ts", - "changeTracker.spec.ts", - "keybindings.spec.ts", - "userSelectView.spec.ts" + 'subgraph.spec.ts', + 'sidebar/workflows.spec.ts', + 'primitiveNode.spec.ts', + 'bottomPanelShortcuts.spec.ts', + 'nodeBadge.spec.ts', + 'execution.spec.ts', + 'rerouteNode.spec.ts', + 'changeTracker.spec.ts', + 'keybindings.spec.ts', + 'userSelectView.spec.ts' ], [ - "widget.spec.ts", - "sidebar/nodeLibrary.spec.ts", - "nodeHelp.spec.ts", - "templates.spec.ts", - "featureFlags.spec.ts", - "copyPaste.spec.ts", - "loadWorkflowInMedia.spec.ts", - "actionbar.spec.ts", - "commands.spec.ts", - "minimap.spec.ts", - "workflowTabThumbnail.spec.ts" + 'widget.spec.ts', + 'sidebar/nodeLibrary.spec.ts', + 'nodeHelp.spec.ts', + 'templates.spec.ts', + 'featureFlags.spec.ts', + 'copyPaste.spec.ts', + 'loadWorkflowInMedia.spec.ts', + 'actionbar.spec.ts', + 'commands.spec.ts', + 'minimap.spec.ts', + 'workflowTabThumbnail.spec.ts' ], [ - "nodeSearchBox.spec.ts", - "rightClickMenu.spec.ts", - "colorPalette.spec.ts", - "useSettingSearch.spec.ts", - "graphCanvasMenu.spec.ts", - "domWidget.spec.ts", - "menu.spec.ts", - "backgroundImageUpload.spec.ts", - "customIcons.spec.ts", - "releaseNotifications.spec.ts" + 'nodeSearchBox.spec.ts', + 'rightClickMenu.spec.ts', + 'colorPalette.spec.ts', + 'useSettingSearch.spec.ts', + 'graphCanvasMenu.spec.ts', + 'domWidget.spec.ts', + 'menu.spec.ts', + 'backgroundImageUpload.spec.ts', + 'customIcons.spec.ts', + 'releaseNotifications.spec.ts' ], [ - "dialog.spec.ts", - "groupNode.spec.ts", - "nodeDisplay.spec.ts", - "remoteWidgets.spec.ts", - "extensionAPI.spec.ts", - "sidebar/queue.spec.ts", - "noteNode.spec.ts", - "browserTabTitle.spec.ts", - "graph.spec.ts", - "subgraph-rename-dialog.spec.ts" + 'dialog.spec.ts', + 'groupNode.spec.ts', + 'nodeDisplay.spec.ts', + 'remoteWidgets.spec.ts', + 'extensionAPI.spec.ts', + 'sidebar/queue.spec.ts', + 'noteNode.spec.ts', + 'browserTabTitle.spec.ts', + 'graph.spec.ts', + 'subgraph-rename-dialog.spec.ts' ] ] @@ -67,5 +67,5 @@ export function getShardTests(shardIndex: number): string[] { } export function getShardPattern(shardIndex: number): string[] { - return getShardTests(shardIndex).map(test => `**/${test}`) + return getShardTests(shardIndex).map((test) => `**/${test}`) } diff --git a/browser_tests/shardConfig.ts b/browser_tests/shardConfig.ts index a74583bb9..43fd02116 100644 --- a/browser_tests/shardConfig.ts +++ b/browser_tests/shardConfig.ts @@ -10,14 +10,14 @@ export interface ShardConfig { // Group tests by execution characteristics export const HEAVY_SCREENSHOT_TESTS = [ - 'interaction.spec.ts', // 61 tests, 81 screenshots - heaviest test file + 'interaction.spec.ts' // 61 tests, 81 screenshots - heaviest test file ] export const MEDIUM_SCREENSHOT_TESTS = [ 'widget.spec.ts', // 17 tests with screenshots 'rightClickMenu.spec.ts', // 11 tests with screenshots 'nodeSearchBox.spec.ts', // 23 tests with screenshots - 'groupNode.spec.ts', // 17 tests with screenshots + 'groupNode.spec.ts' // 17 tests with screenshots ] export const LIGHT_SCREENSHOT_TESTS = [ @@ -33,14 +33,14 @@ export const LIGHT_SCREENSHOT_TESTS = [ 'execution.spec.ts', 'rerouteNode.spec.ts', 'copyPaste.spec.ts', - 'loadWorkflowInMedia.spec.ts', + 'loadWorkflowInMedia.spec.ts' ] export const HEAVY_LOGIC_TESTS = [ 'subgraph.spec.ts', // 23 tests, complex logic 'dialog.spec.ts', // 21 tests 'sidebar/workflows.spec.ts', // 18 tests - 'sidebar/nodeLibrary.spec.ts', // 18 tests + 'sidebar/nodeLibrary.spec.ts' // 18 tests ] export const MEDIUM_LOGIC_TESTS = [ @@ -51,7 +51,7 @@ export const MEDIUM_LOGIC_TESTS = [ 'extensionAPI.spec.ts', // 11 tests 'bottomPanelShortcuts.spec.ts', // 11 tests 'featureFlags.spec.ts', // 9 tests - 'menu.spec.ts', // 9 tests + 'menu.spec.ts' // 9 tests ] export const LIGHT_LOGIC_TESTS = [ @@ -70,7 +70,7 @@ export const LIGHT_LOGIC_TESTS = [ 'userSelectView.spec.ts', 'versionMismatchWarnings.spec.ts', 'workflowTabThumbnail.spec.ts', - 'actionbar.spec.ts', + 'actionbar.spec.ts' ] // Optimized shard distribution for chromium tests @@ -115,7 +115,11 @@ export const NO_SHARD_PROJECTS = [ * @param totalShards Total number of shards * @param projectName Name of the Playwright project */ -export function getShardTests(shardIndex: number, totalShards: number, projectName: string): string[] | null { +export function getShardTests( + shardIndex: number, + totalShards: number, + projectName: string +): string[] | null { // For projects that don't need sharding, return null to run all tests if (NO_SHARD_PROJECTS.includes(projectName)) { return null @@ -137,14 +141,18 @@ export function getShardTests(shardIndex: number, totalShards: number, projectNa * @param totalShards Total number of shards * @param projectName Name of the Playwright project */ -export function getShardGrep(shardIndex: number, totalShards: number, projectName: string): RegExp | null { +export function getShardGrep( + shardIndex: number, + totalShards: number, + projectName: string +): RegExp | null { const tests = getShardTests(shardIndex, totalShards, projectName) - + if (!tests || tests.length === 0) { return null } // Create a regex pattern that matches any of the test files - const pattern = tests.map(file => file.replace(/\./g, '\\.')).join('|') + const pattern = tests.map((file) => file.replace(/\./g, '\\.')).join('|') return new RegExp(pattern) -} \ No newline at end of file +} diff --git a/playwright-sharded.config.ts b/playwright-sharded.config.ts index ea9267238..455303890 100644 --- a/playwright-sharded.config.ts +++ b/playwright-sharded.config.ts @@ -1,6 +1,7 @@ import { defineConfig } from '@playwright/test' -import baseConfig from './playwright.config' + import { getShardPattern } from './browser_tests/shardConfig.generated' +import baseConfig from './playwright.config' /** * Optimized Playwright configuration for CI with balanced sharding @@ -8,8 +9,12 @@ import { getShardPattern } from './browser_tests/shardConfig.generated' */ // Parse shard information from Playwright CLI -const shardInfo = process.env.SHARD || process.argv.find(arg => arg.includes('--shard='))?.split('=')[1] -const [currentShard, totalShards] = shardInfo ? shardInfo.split('/').map(Number) : [1, 1] +const shardInfo = + process.env.SHARD || + process.argv.find((arg) => arg.includes('--shard='))?.split('=')[1] +const [currentShard, totalShards] = shardInfo + ? shardInfo.split('/').map(Number) + : [1, 1] // Get test patterns for current shard const testMatch = totalShards === 5 ? getShardPattern(currentShard) : undefined @@ -21,17 +26,17 @@ if (testMatch) { export default defineConfig({ ...baseConfig, - + // Use optimized test distribution for 5-shard setup ...(testMatch && { testMatch }), - + // Optimize parallel execution based on shard content fullyParallel: true, workers: process.env.CI ? (currentShard === 1 ? 2 : 4) : baseConfig.workers, - + // Adjust timeouts for heavy tests timeout: currentShard === 1 ? 20000 : 15000, - + // Optimize retries - retries: process.env.CI ? (currentShard === 1 ? 2 : 3) : 0, -}) \ No newline at end of file + retries: process.env.CI ? (currentShard === 1 ? 2 : 3) : 0 +}) diff --git a/playwright.shard.config.ts b/playwright.shard.config.ts index 3bd1913a5..4c58233c1 100644 --- a/playwright.shard.config.ts +++ b/playwright.shard.config.ts @@ -1,4 +1,5 @@ -import { defineConfig, devices } from '@playwright/test' +import { defineConfig } from '@playwright/test' + import baseConfig from './playwright.config' /** @@ -7,7 +8,9 @@ import baseConfig from './playwright.config' */ // Helper to determine if we should apply custom test filtering -const shardInfo = process.env.SHARD ? process.env.SHARD.split('/').map(Number) : null +const shardInfo = process.env.SHARD + ? process.env.SHARD.split('/').map(Number) + : null const currentShard = shardInfo?.[0] || 1 const totalShards = shardInfo?.[1] || 1 const projectName = process.env.TEST_PROJECT || 'chromium' @@ -16,13 +19,13 @@ const projectName = process.env.TEST_PROJECT || 'chromium' const testGroups = { // Heavy tests (run in separate shards) heavy: [ - '**/interaction.spec.ts', // 61 tests with 81 screenshots + '**/interaction.spec.ts' // 61 tests with 81 screenshots ], // Medium-heavy tests mediumHeavy: [ '**/subgraph.spec.ts', // 23 complex tests '**/widget.spec.ts', // 17 tests with screenshots - '**/nodeSearchBox.spec.ts', // 23 tests with screenshots + '**/nodeSearchBox.spec.ts' // 23 tests with screenshots ], // Medium tests medium: [ @@ -30,7 +33,7 @@ const testGroups = { '**/groupNode.spec.ts', '**/rightClickMenu.spec.ts', '**/sidebar/workflows.spec.ts', - '**/sidebar/nodeLibrary.spec.ts', + '**/sidebar/nodeLibrary.spec.ts' ], // Light tests light: [ @@ -46,7 +49,7 @@ const testGroups = { '**/execution.spec.ts', '**/rerouteNode.spec.ts', '**/copyPaste.spec.ts', - '**/loadWorkflowInMedia.spec.ts', + '**/loadWorkflowInMedia.spec.ts' ], // Very light tests veryLight: [ @@ -73,7 +76,7 @@ const testGroups = { '**/userSelectView.spec.ts', '**/versionMismatchWarnings.spec.ts', '**/workflowTabThumbnail.spec.ts', - '**/actionbar.spec.ts', + '**/actionbar.spec.ts' ] } @@ -83,12 +86,16 @@ const shardPatterns: Record = { 2: testGroups.mediumHeavy, // Shard 2: Medium-heavy tests 3: testGroups.medium, // Shard 3: Medium tests 4: testGroups.light, // Shard 4: Light tests - 5: testGroups.veryLight, // Shard 5: Very light tests + 5: testGroups.veryLight // Shard 5: Very light tests } // Determine which tests to run based on shard let testMatch: string[] | undefined -if (projectName === 'chromium' && totalShards === 5 && shardPatterns[currentShard]) { +if ( + projectName === 'chromium' && + totalShards === 5 && + shardPatterns[currentShard] +) { testMatch = shardPatterns[currentShard] } @@ -96,30 +103,36 @@ export default defineConfig({ ...baseConfig, // Override testMatch if we have custom shard patterns ...(testMatch && { testMatch }), - + // Increase workers for lighter test shards use: { ...baseConfig.use, // More parallel workers for shards with lighter tests - ...(currentShard >= 4 && projectName === 'chromium' && { - workers: process.env.CI ? 4 : 2 - }) + ...(currentShard >= 4 && + projectName === 'chromium' && { + workers: process.env.CI ? 4 : 2 + }) }, // Optimize retries based on shard content retries: process.env.CI ? (currentShard === 1 ? 2 : 3) : 0, // Project-specific optimizations - projects: baseConfig.projects?.map(project => { - // For non-chromium projects that don't need sharding - if (['mobile-chrome', 'chromium-0.5x', 'chromium-2x'].includes(project.name || '')) { - return { - ...project, - // These projects should only run when not sharding or on first shard - ...(totalShards > 1 && currentShard > 1 && { testMatch: [] }) + projects: + baseConfig.projects?.map((project) => { + // For non-chromium projects that don't need sharding + if ( + ['mobile-chrome', 'chromium-0.5x', 'chromium-2x'].includes( + project.name || '' + ) + ) { + return { + ...project, + // These projects should only run when not sharding or on first shard + ...(totalShards > 1 && currentShard > 1 && { testMatch: [] }) + } } - } - - return project - }) || [] -}) \ No newline at end of file + + return project + }) || [] +})