diff --git a/package.json b/package.json index 300809d9cfd775..66dd33f13c8d57 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,8 @@ "test:coverage": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc mocha 'packages/**/*.test.js' 'docs/**/*.test.js' --exclude '**/node_modules/**' && nyc report -r lcovonly", "test:coverage:html": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc mocha 'packages/**/**/*.test.js' --exclude '**/node_modules/**' && nyc report --reporter=html", "test:karma": "cross-env NODE_ENV=test karma start test/karma.conf.js", - "test:regressions": "webpack --config test/regressions/webpack.config.js && rimraf test/regressions/screenshots/chrome/* && vrtest run --config test/vrtest.config.js --record", + "test:regressions": "yarn test:regressions:build && rimraf test/regressions/screenshots/chrome/* && vrtest run --config test/vrtest.config.js --record", + "test:regressions:build": "webpack --config test/regressions/webpack.config.js", "test:umd": "node packages/material-ui/test/umd/run.js", "test:unit": "cross-env NODE_ENV=test mocha 'packages/**/*.test.js' 'docs/**/*.test.js' 'scripts/**/*.test.js' --exclude '**/node_modules/**'", "test:watch": "yarn test:unit --watch", diff --git a/test/regressions/index.js b/test/regressions/index.js index f6fc785dea60c8..a75bf59914fa91 100644 --- a/test/regressions/index.js +++ b/test/regressions/index.js @@ -42,21 +42,20 @@ const blacklist = [ 'docs-components-badges/BadgeAlignment.png', // Redux isolation 'docs-components-badges/BadgeVisibility.png', // Needs interaction 'docs-components-breadcrumbs/ActiveLastBreadcrumb.png', // Redundant - 'docs-components-buttons/ButtonBases.png', // Useless - 'docs-components-buttons/FloatingActionButtonZoom.png', // Needs interaction + 'docs-components-buttons/ButtonBases.png', // Flaky image loading 'docs-components-chips/ChipsPlayground.png', // Redux isolation 'docs-components-click-away-listener', // Needs interaction - 'docs-components-container', // Not needed + 'docs-components-container', // Can't see the impact 'docs-components-dialogs', // Needs interaction 'docs-components-drawers/SwipeableTemporaryDrawer.png', // Needs interaction 'docs-components-drawers/TemporaryDrawer.png', // Needs interaction - 'docs-components-grid-list', // Not needed - 'docs-components-grid-list/tileData.png', // No component + 'docs-components-floating-action-button/FloatingActionButtonZoom.png', // Needs interaction + 'docs-components-grid-list', // Image don't load 'docs-components-grid/InteractiveGrid.png', // Redux isolation 'docs-components-grid/SpacingGrid.png', // Needs interaction - 'docs-components-hidden', // Not needed + 'docs-components-hidden', // Need to dynamically resize to test 'docs-components-material-icons/synonyms.png', // No component - 'docs-components-menus', // Not needed + 'docs-components-menus', // Need interaction 'docs-components-modal/SimpleModal.png', // Needs interaction 'docs-components-modal/SpringModal.png', // Needs interaction 'docs-components-modal/TransitionsModal.png', // Needs interaction @@ -74,7 +73,7 @@ const blacklist = [ 'docs-components-popper/SpringPopper.png', // Needs interaction 'docs-components-popper/TransitionsPopper.png', // Needs interaction 'docs-components-portal/SimplePortal.png', // Needs interaction - 'docs-components-progress', // Not needed + 'docs-components-progress', // Flaky 'docs-components-selects/ControlledOpenSelect.png', // Needs interaction 'docs-components-selects/DialogSelect.png', // Needs interaction 'docs-components-selects/GroupedSelect.png', // Needs interaction @@ -84,7 +83,7 @@ const blacklist = [ 'docs-components-snackbars/ConsecutiveSnackbars.png', // Needs interaction 'docs-components-snackbars/CustomizedSnackbars.png', // Redundant 'docs-components-snackbars/DirectionSnackbar.png', // Needs interaction - 'docs-components-snackbars/FabIntegrationSnackbar.png', // Not needed + 'docs-components-snackbars/FabIntegrationSnackbar.png', // Needs interaction 'docs-components-snackbars/IntegrationNotistack.png', // Needs interaction 'docs-components-snackbars/PositionedSnackbar.png', // Needs interaction 'docs-components-snackbars/SimpleSnackbar.png', // Needs interaction @@ -99,18 +98,18 @@ const blacklist = [ 'docs-components-transitions', // Needs interaction 'docs-components-tree-view/ControlledTreeView.png', // Redundant 'docs-components-tree-view/CustomizedTreeView.png', // Flaky - 'docs-components-use-media-query', // Not needed - 'docs-customization-breakpoints', // Not needed - 'docs-customization-color', // Not needed + 'docs-components-use-media-query', // Need to dynamically resize to test + 'docs-customization-breakpoints', // Need to dynamically resize to test + 'docs-customization-color', // Escape viewport 'docs-customization-default-theme', // Redux isolation 'docs-customization-density/DensityTool.png', // Redux isolation - 'docs-customization-typography/ResponsiveFontSizesChart.png', // Not needed - 'docs-discover-more-languages', // Not needed - 'docs-discover-more-showcase', // Not needed - 'docs-discover-more-team', // Not needed - 'docs-getting-started-templates', // Not needed + 'docs-customization-typography/ResponsiveFontSizesChart.png', + 'docs-discover-more-languages', // No public components + 'docs-discover-more-showcase', // No public components + 'docs-discover-more-team', // No public components + 'docs-getting-started-templates', // No public components 'docs-getting-started-templates-album/Album.png', // Flaky image loading - 'docs-getting-started-templates-blog', // Not needed + 'docs-getting-started-templates-blog', // Flaky random images 'docs-getting-started-templates-checkout/AddressForm.png', // Already tested once assembled 'docs-getting-started-templates-checkout/PaymentForm.png', // Already tested once assembled 'docs-getting-started-templates-checkout/Review.png', // Already tested once assembled @@ -119,21 +118,59 @@ const blacklist = [ 'docs-getting-started-templates-dashboard/Orders.png', // Already tested once assembled 'docs-getting-started-templates-dashboard/Title.png', // Already tested once assembled 'docs-getting-started-templates-sign-in-side/SignInSide.png', // Flaky - 'docs-getting-started-usage/Usage.png', // Not needed - 'docs-guides', // Not needed - 'docs-styles-advanced', // Not needed - 'docs-system-borders', // Not needed - 'docs-system-display', // Not needed - 'docs-system-flexbox', // Not needed - 'docs-system-palette', // Not needed - 'docs-system-positions', // Not needed - 'docs-system-shadows', // Not needed - 'docs-system-sizing', // Not needed - 'docs-system-spacing', // Not needed - 'docs-system-typography', // Not needed - 'docs-versions', // Not needed + 'docs-getting-started-usage/Usage.png', // No public components + /^docs-guides-.*/, // No public components + 'docs-styles-advanced', // Redudant + 'docs-system-borders', // Unit tests are enough + 'docs-system-display', // Unit tests are enough + 'docs-system-flexbox', // Unit tests are enough + 'docs-system-palette', // Unit tests are enough + 'docs-system-positions', // Unit tests are enough + 'docs-system-shadows', // Unit tests are enough + 'docs-system-sizing', // Unit tests are enough + 'docs-system-spacing', // Unit tests are enough + 'docs-system-typography', // Unit tests are enough + 'docs-versions', // No public components ]; +const unusedBlacklistPatterns = new Set(blacklist); + +function excludeTest(suite, name) { + if (/^docs-premium-themes(.*)/.test(suite)) { + // eslint-disable-next-line no-console + console.log('ignoring premium themes pages'); + return true; + } + + return blacklist.some(pattern => { + if (typeof pattern === 'string') { + if (pattern === suite) { + unusedBlacklistPatterns.delete(pattern); + // eslint-disable-next-line no-console + console.log(`suite exact match: ignoring '${suite}/${name}'`); + return true; + } + if (pattern === `${suite}/${name}.png`) { + unusedBlacklistPatterns.delete(pattern); + // eslint-disable-next-line no-console + console.log(`suite+name exact match: ignoring '${suite}/${name}'`); + return true; + } + + return false; + } + + // assume regex + if (pattern.test(suite)) { + unusedBlacklistPatterns.delete(pattern); + // eslint-disable-next-line no-console + console.log(`suite matches pattern '${pattern}': ignoring '${suite}/${name}'`); + return true; + } + return false; + }); +} + // Also use some of the demos to avoid code duplication. const requireDemos = require.context('docs/src/pages', true, /js$/); const demos = requireDemos.keys().reduce((res, path) => { @@ -144,17 +181,11 @@ const demos = requireDemos.keys().reduce((res, path) => { .reverse(); const suite = `docs-${suiteArray.reverse().join('-')}`; - if (blacklist.includes(suite)) { - return res; - } - - if (blacklist.includes(`${suite}/${name}.png`)) { - return res; - } - - if (/^docs-premium-themes(.*)/.test(suite)) { + if (excludeTest(suite, name)) { return res; } + // eslint-disable-next-line no-console + console.log(`testing ${suite}/${name}`); res.push({ path, @@ -217,3 +248,11 @@ tests.forEach(test => { ); }); }); + +if (unusedBlacklistPatterns.size > 0) { + console.warn( + `The following patterns are unused:\n\n${Array.from(unusedBlacklistPatterns) + .map(pattern => `- ${pattern}`) + .join('\n')}`, + ); +}