Skip to content

Commit

Permalink
[test] Improve regression test suite debugging (mui#20194)
Browse files Browse the repository at this point in the history
  • Loading branch information
@eps1lon @EsoterikStare
eps1lon authored and EsoterikStare committed Mar 30, 2020
1 parent 27fa8ae commit f286d18
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 41 deletions.
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@
"test:coverage": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc mocha 'packages/**/*.test.js' 'docs/**/*.test.js' --exclude '**/node_modules/**' && nyc report -r lcovonly",
"test:coverage:html": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc mocha 'packages/**/**/*.test.js' --exclude '**/node_modules/**' && nyc report --reporter=html",
"test:karma": "cross-env NODE_ENV=test karma start test/karma.conf.js",
"test:regressions": "webpack --config test/regressions/webpack.config.js && rimraf test/regressions/screenshots/chrome/* && vrtest run --config test/vrtest.config.js --record",
"test:regressions": "yarn test:regressions:build && rimraf test/regressions/screenshots/chrome/* && vrtest run --config test/vrtest.config.js --record",
"test:regressions:build": "webpack --config test/regressions/webpack.config.js",
"test:umd": "node packages/material-ui/test/umd/run.js",
"test:unit": "cross-env NODE_ENV=test mocha 'packages/**/*.test.js' 'docs/**/*.test.js' 'scripts/**/*.test.js' --exclude '**/node_modules/**'",
"test:watch": "yarn test:unit --watch",
Expand Down
119 changes: 79 additions & 40 deletions test/regressions/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,21 +42,20 @@ const blacklist = [
'docs-components-badges/BadgeAlignment.png', // Redux isolation
'docs-components-badges/BadgeVisibility.png', // Needs interaction
'docs-components-breadcrumbs/ActiveLastBreadcrumb.png', // Redundant
'docs-components-buttons/ButtonBases.png', // Useless
'docs-components-buttons/FloatingActionButtonZoom.png', // Needs interaction
'docs-components-buttons/ButtonBases.png', // Flaky image loading
'docs-components-chips/ChipsPlayground.png', // Redux isolation
'docs-components-click-away-listener', // Needs interaction
'docs-components-container', // Not needed
'docs-components-container', // Can't see the impact
'docs-components-dialogs', // Needs interaction
'docs-components-drawers/SwipeableTemporaryDrawer.png', // Needs interaction
'docs-components-drawers/TemporaryDrawer.png', // Needs interaction
'docs-components-grid-list', // Not needed
'docs-components-grid-list/tileData.png', // No component
'docs-components-floating-action-button/FloatingActionButtonZoom.png', // Needs interaction
'docs-components-grid-list', // Image don't load
'docs-components-grid/InteractiveGrid.png', // Redux isolation
'docs-components-grid/SpacingGrid.png', // Needs interaction
'docs-components-hidden', // Not needed
'docs-components-hidden', // Need to dynamically resize to test
'docs-components-material-icons/synonyms.png', // No component
'docs-components-menus', // Not needed
'docs-components-menus', // Need interaction
'docs-components-modal/SimpleModal.png', // Needs interaction
'docs-components-modal/SpringModal.png', // Needs interaction
'docs-components-modal/TransitionsModal.png', // Needs interaction
Expand All @@ -74,7 +73,7 @@ const blacklist = [
'docs-components-popper/SpringPopper.png', // Needs interaction
'docs-components-popper/TransitionsPopper.png', // Needs interaction
'docs-components-portal/SimplePortal.png', // Needs interaction
'docs-components-progress', // Not needed
'docs-components-progress', // Flaky
'docs-components-selects/ControlledOpenSelect.png', // Needs interaction
'docs-components-selects/DialogSelect.png', // Needs interaction
'docs-components-selects/GroupedSelect.png', // Needs interaction
Expand All @@ -84,7 +83,7 @@ const blacklist = [
'docs-components-snackbars/ConsecutiveSnackbars.png', // Needs interaction
'docs-components-snackbars/CustomizedSnackbars.png', // Redundant
'docs-components-snackbars/DirectionSnackbar.png', // Needs interaction
'docs-components-snackbars/FabIntegrationSnackbar.png', // Not needed
'docs-components-snackbars/FabIntegrationSnackbar.png', // Needs interaction
'docs-components-snackbars/IntegrationNotistack.png', // Needs interaction
'docs-components-snackbars/PositionedSnackbar.png', // Needs interaction
'docs-components-snackbars/SimpleSnackbar.png', // Needs interaction
Expand All @@ -99,18 +98,18 @@ const blacklist = [
'docs-components-transitions', // Needs interaction
'docs-components-tree-view/ControlledTreeView.png', // Redundant
'docs-components-tree-view/CustomizedTreeView.png', // Flaky
'docs-components-use-media-query', // Not needed
'docs-customization-breakpoints', // Not needed
'docs-customization-color', // Not needed
'docs-components-use-media-query', // Need to dynamically resize to test
'docs-customization-breakpoints', // Need to dynamically resize to test
'docs-customization-color', // Escape viewport
'docs-customization-default-theme', // Redux isolation
'docs-customization-density/DensityTool.png', // Redux isolation
'docs-customization-typography/ResponsiveFontSizesChart.png', // Not needed
'docs-discover-more-languages', // Not needed
'docs-discover-more-showcase', // Not needed
'docs-discover-more-team', // Not needed
'docs-getting-started-templates', // Not needed
'docs-customization-typography/ResponsiveFontSizesChart.png',
'docs-discover-more-languages', // No public components
'docs-discover-more-showcase', // No public components
'docs-discover-more-team', // No public components
'docs-getting-started-templates', // No public components
'docs-getting-started-templates-album/Album.png', // Flaky image loading
'docs-getting-started-templates-blog', // Not needed
'docs-getting-started-templates-blog', // Flaky random images
'docs-getting-started-templates-checkout/AddressForm.png', // Already tested once assembled
'docs-getting-started-templates-checkout/PaymentForm.png', // Already tested once assembled
'docs-getting-started-templates-checkout/Review.png', // Already tested once assembled
Expand All @@ -119,21 +118,59 @@ const blacklist = [
'docs-getting-started-templates-dashboard/Orders.png', // Already tested once assembled
'docs-getting-started-templates-dashboard/Title.png', // Already tested once assembled
'docs-getting-started-templates-sign-in-side/SignInSide.png', // Flaky
'docs-getting-started-usage/Usage.png', // Not needed
'docs-guides', // Not needed
'docs-styles-advanced', // Not needed
'docs-system-borders', // Not needed
'docs-system-display', // Not needed
'docs-system-flexbox', // Not needed
'docs-system-palette', // Not needed
'docs-system-positions', // Not needed
'docs-system-shadows', // Not needed
'docs-system-sizing', // Not needed
'docs-system-spacing', // Not needed
'docs-system-typography', // Not needed
'docs-versions', // Not needed
'docs-getting-started-usage/Usage.png', // No public components
/^docs-guides-.*/, // No public components
'docs-styles-advanced', // Redudant
'docs-system-borders', // Unit tests are enough
'docs-system-display', // Unit tests are enough
'docs-system-flexbox', // Unit tests are enough
'docs-system-palette', // Unit tests are enough
'docs-system-positions', // Unit tests are enough
'docs-system-shadows', // Unit tests are enough
'docs-system-sizing', // Unit tests are enough
'docs-system-spacing', // Unit tests are enough
'docs-system-typography', // Unit tests are enough
'docs-versions', // No public components
];

const unusedBlacklistPatterns = new Set(blacklist);

function excludeTest(suite, name) {
if (/^docs-premium-themes(.*)/.test(suite)) {
// eslint-disable-next-line no-console
console.log('ignoring premium themes pages');
return true;
}

return blacklist.some(pattern => {
if (typeof pattern === 'string') {
if (pattern === suite) {
unusedBlacklistPatterns.delete(pattern);
// eslint-disable-next-line no-console
console.log(`suite exact match: ignoring '${suite}/${name}'`);
return true;
}
if (pattern === `${suite}/${name}.png`) {
unusedBlacklistPatterns.delete(pattern);
// eslint-disable-next-line no-console
console.log(`suite+name exact match: ignoring '${suite}/${name}'`);
return true;
}

return false;
}

// assume regex
if (pattern.test(suite)) {
unusedBlacklistPatterns.delete(pattern);
// eslint-disable-next-line no-console
console.log(`suite matches pattern '${pattern}': ignoring '${suite}/${name}'`);
return true;
}
return false;
});
}

// Also use some of the demos to avoid code duplication.
const requireDemos = require.context('docs/src/pages', true, /js$/);
const demos = requireDemos.keys().reduce((res, path) => {
Expand All @@ -144,17 +181,11 @@ const demos = requireDemos.keys().reduce((res, path) => {
.reverse();
const suite = `docs-${suiteArray.reverse().join('-')}`;

if (blacklist.includes(suite)) {
return res;
}

if (blacklist.includes(`${suite}/${name}.png`)) {
return res;
}

if (/^docs-premium-themes(.*)/.test(suite)) {
if (excludeTest(suite, name)) {
return res;
}
// eslint-disable-next-line no-console
console.log(`testing ${suite}/${name}`);

res.push({
path,
Expand Down Expand Up @@ -217,3 +248,11 @@ tests.forEach(test => {
);
});
});

if (unusedBlacklistPatterns.size > 0) {
console.warn(
`The following patterns are unused:\n\n${Array.from(unusedBlacklistPatterns)
.map(pattern => `- ${pattern}`)
.join('\n')}`,
);
}

0 comments on commit f286d18

Please sign in to comment.