diff --git a/lib/internal/fs/glob.js b/lib/internal/fs/glob.js new file mode 100644 index 000000000000000..271e7c68c35dc56 --- /dev/null +++ b/lib/internal/fs/glob.js @@ -0,0 +1,202 @@ +'use strict'; +const { lstatSync, readdirSync } = require('fs'); +const { join, resolve } = require('path'); + +const { + kEmptyObject, +} = require('internal/util'); +const { isRegExp } = require('internal/util/types'); +const { + validateFunction, + validateObject, +} = require('internal/validators'); + +const { + ArrayPrototypeForEach, + ArrayPrototypeMap, + ArrayPrototypeFlatMap, + ArrayPrototypePop, + ArrayPrototypePush, + SafeMap, + SafeSet, +} = primordials; + +let minimatch; +function lazyMinimatch() { + minimatch ??= require('internal/deps/minimatch/index'); + return minimatch; +} + +function testPattern(pattern, path) { + if (pattern === lazyMinimatch().GLOBSTAR) { + return true; + } + if (typeof pattern === 'string') { + return true; + } + if (typeof pattern.test === 'function') { + return pattern.test(path); + } +} + +class Cache { + #caches = new SafeMap(); + #statsCache = new SafeMap(); + #readdirCache = new SafeMap(); + + stats(path) { + if (this.#statsCache.has(path)) { + return this.#statsCache.get(path); + } + let val; + try { + val = lstatSync(path); + } catch { + val = null; + } + this.#statsCache.set(path, val); + return val; + } + readdir(path) { + if (this.#readdirCache.has(path)) { + return this.#readdirCache.get(path); + } + let val; + try { + val = readdirSync(path, { withFileTypes: true }); + ArrayPrototypeForEach(val, (dirent) => this.#statsCache.set(join(path, dirent.name), dirent)); + } catch { + val = []; + } + this.#readdirCache.set(path, val); + return val; + } + + seen(pattern, index, path) { + return this.#caches.get(path)?.get(pattern)?.has(index); + } + add(pattern, index, path) { + if (!this.#caches.has(path)) { + this.#caches.set(path, new SafeMap([[pattern, new SafeSet([index])]])); + } else if (!this.#caches.get(path)?.has(pattern)) { + this.#caches.get(path)?.set(pattern, new SafeSet([index])); + } else { + this.#caches.get(path)?.get(pattern)?.add(index); + } + } + +} + +function glob(patterns, options = kEmptyObject) { + validateObject(options, 'options'); + const root = options.cwd ?? '.'; + const { exclude } = options; + if (exclude != null) { + validateFunction(exclude, 'options.exclude'); + } + + const { Minimatch, GLOBSTAR } = lazyMinimatch(); + const results = new SafeSet(); + const matchers = ArrayPrototypeMap(patterns, (pattern) => new Minimatch(pattern)); + const queue = ArrayPrototypeFlatMap(matchers, (matcher) => { + return ArrayPrototypeMap(matcher.set, + (pattern) => ({ __proto__: null, pattern, index: 0, path: '.', followSymlinks: true })); + }); + const cache = new Cache(matchers); + + while (queue.length > 0) { + const { pattern, index: currentIndex, path, followSymlinks } = ArrayPrototypePop(queue); + if (cache.seen(pattern, currentIndex, path)) { + continue; + } + cache.add(pattern, currentIndex, path); + + const currentPattern = pattern[currentIndex]; + const index = currentIndex + 1; + const isLast = pattern.length === index || (pattern.length === index + 1 && pattern[index] === ''); + + if (currentPattern === '') { + // Absolute path + ArrayPrototypePush(queue, { __proto__: null, pattern, index, path: '/', followSymlinks }); + continue; + } + + if (typeof currentPattern === 'string') { + const entryPath = join(path, currentPattern); + if (isLast && cache.stats(resolve(root, entryPath))) { + // last path + results.add(entryPath); + } else if (!isLast) { + // Keep traversing, we only check file existence for the last path + ArrayPrototypePush(queue, { __proto__: null, pattern, index, path: entryPath, followSymlinks }); + } + continue; + } + + const fullpath = resolve(root, path); + const stat = cache.stats(fullpath); + const isDirectory = stat?.isDirectory() || (followSymlinks !== false && stat?.isSymbolicLink()); + + if (isDirectory && isRegExp(currentPattern)) { + const entries = cache.readdir(fullpath); + for (const entry of entries) { + const entryPath = join(path, entry.name); + if (cache.seen(pattern, index, entryPath)) { + continue; + } + const matches = testPattern(currentPattern, entry.name); + if (matches && isLast) { + results.add(entryPath); + } else if (matches) { + ArrayPrototypePush(queue, { __proto__: null, pattern, index, path: entryPath, followSymlinks }); + } + } + } + + if (currentPattern === GLOBSTAR && isDirectory) { + const entries = cache.readdir(fullpath); + for (const entry of entries) { + if (entry.name[0] === '.' || (exclude && exclude(entry.name))) { + continue; + } + const entryPath = join(path, entry.name); + if (cache.seen(pattern, index, entryPath)) { + continue; + } + const isSymbolicLink = entry.isSymbolicLink(); + const isDirectory = entry.isDirectory(); + if (isDirectory) { + // Push child directory to queue at same pattern index + ArrayPrototypePush(queue, { + __proto__: null, pattern, index: currentIndex, path: entryPath, followSymlinks: !isSymbolicLink, + }); + } + + if (pattern.length === index || (isSymbolicLink && pattern.length === index + 1 && pattern[index] === '')) { + results.add(entryPath); + } else if (pattern[index] === '..') { + continue; + } else if (!isLast && + (isDirectory || (isSymbolicLink && (typeof pattern[index] !== 'string' || pattern[0] !== GLOBSTAR)))) { + ArrayPrototypePush(queue, { __proto__: null, pattern, index, path: entryPath, followSymlinks }); + } + } + if (isLast) { + results.add(path); + } else { + ArrayPrototypePush(queue, { __proto__: null, pattern, index, path, followSymlinks }); + } + } + } + + return { + __proto__: null, + results, + matchers, + }; +} + +module.exports = { + glob, + lazyMinimatch, +}; diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index ab3ce698512db1a..a30c52bc8d8053c 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -1,10 +1,12 @@ 'use strict'; const { ArrayFrom, + ArrayPrototypeEvery, ArrayPrototypeFilter, ArrayPrototypeForEach, ArrayPrototypeIncludes, ArrayPrototypeIndexOf, + ArrayPrototypeJoin, ArrayPrototypePush, ArrayPrototypeSlice, ArrayPrototypeSome, @@ -25,7 +27,6 @@ const { } = primordials; const { spawn } = require('child_process'); -const { readdirSync, statSync } = require('fs'); const { finished } = require('internal/streams/end-of-stream'); // TODO(aduh95): switch to internal/readline/interface when backporting to Node.js 16.x is no longer a concern. const { createInterface } = require('readline'); @@ -54,10 +55,9 @@ const { TokenKind } = require('internal/test_runner/tap_lexer'); const { countCompletedTest, - doesPathMatchFilter, - isSupportedFileType, + kDefaultPattern, } = require('internal/test_runner/utils'); -const { basename, join, resolve } = require('path'); +const { glob } = require('internal/fs/glob'); const { once } = require('events'); const { triggerUncaughtException, @@ -71,66 +71,18 @@ const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', const kCanceledTests = new SafeSet() .add(kCancelledByParent).add(kAborted).add(kTestTimeoutFailure); -// TODO(cjihrig): Replace this with recursive readdir once it lands. -function processPath(path, testFiles, options) { - const stats = statSync(path); - - if (stats.isFile()) { - if (options.userSupplied || - (options.underTestDir && isSupportedFileType(path)) || - doesPathMatchFilter(path)) { - testFiles.add(path); - } - } else if (stats.isDirectory()) { - const name = basename(path); - - if (!options.userSupplied && name === 'node_modules') { - return; - } - - // 'test' directories get special treatment. Recursively add all .js, - // .cjs, and .mjs files in the 'test' directory. - const isTestDir = name === 'test'; - const { underTestDir } = options; - const entries = readdirSync(path); - - if (isTestDir) { - options.underTestDir = true; - } - - options.userSupplied = false; - - for (let i = 0; i < entries.length; i++) { - processPath(join(path, entries[i]), testFiles, options); - } - - options.underTestDir = underTestDir; - } -} - function createTestFileList() { const cwd = process.cwd(); - const hasUserSuppliedPaths = process.argv.length > 1; - const testPaths = hasUserSuppliedPaths ? - ArrayPrototypeSlice(process.argv, 1) : [cwd]; - const testFiles = new SafeSet(); - - try { - for (let i = 0; i < testPaths.length; i++) { - const absolutePath = resolve(testPaths[i]); - - processPath(absolutePath, testFiles, { userSupplied: true }); - } - } catch (err) { - if (err?.code === 'ENOENT') { - console.error(`Could not find '${err.path}'`); - process.exit(kGenericUserError); - } + const hasUserSuppliedPattern = process.argv.length > 1; + const patterns = hasUserSuppliedPattern ? ArrayPrototypeSlice(process.argv, 1) : [kDefaultPattern]; + const { results, matchers } = glob(patterns, { __proto__: null, cwd, exclude: (name) => name === 'node_modules' }); - throw err; + if (hasUserSuppliedPattern && results.size === 0 && ArrayPrototypeEvery(matchers, (m) => !m.hasMagic())) { + console.error(`Could not find '${ArrayPrototypeJoin(patterns, ', ')}'`); + process.exit(kGenericUserError); } - return ArrayPrototypeSort(ArrayFrom(testFiles)); + return ArrayPrototypeSort(ArrayFrom(results)); } function filterExecArgv(arg, i, arr) { diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index e97f92484b8b8d5..21dbd44137ffd69 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -11,7 +11,7 @@ const { SafeMap, } = primordials; -const { basename, relative } = require('path'); +const { relative } = require('path'); const { createWriteStream } = require('fs'); const { pathToFileURL } = require('internal/url'); const { createDeferredPromise } = require('internal/util'); @@ -29,16 +29,10 @@ const { compose } = require('stream'); const kMultipleCallbackInvocations = 'multipleCallbackInvocations'; const kRegExpPattern = /^\/(.*)\/([a-z]*)$/; -const kSupportedFileExtensions = /\.[cm]?js$/; -const kTestFilePattern = /((^test(-.+)?)|(.+[.\-_]test))\.[cm]?js$/; -function doesPathMatchFilter(p) { - return RegExpPrototypeExec(kTestFilePattern, basename(p)) !== null; -} +const kPatterns = ['test', 'test/**/*', 'test-*', '*[.\\-_]test']; +const kDefaultPattern = `**/{${ArrayPrototypeJoin(kPatterns, ',')}}.?(c|m)js`; -function isSupportedFileType(p) { - return RegExpPrototypeExec(kSupportedFileExtensions, p) !== null; -} function createDeferredCallback() { let calledCount = 0; @@ -299,9 +293,8 @@ module.exports = { convertStringToRegExp, countCompletedTest, createDeferredCallback, - doesPathMatchFilter, - isSupportedFileType, isTestFailureError, + kDefaultPattern, parseCommandLine, setupTestReporters, getCoverageReport, diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js index 5e913eb6de9e5d4..baf742031298fad 100644 --- a/test/parallel/test-runner-cli.js +++ b/test/parallel/test-runner-cli.js @@ -21,8 +21,8 @@ const testFixtures = fixtures.path('test-runner'); { // Default behavior. node_modules is ignored. Files that don't match the // pattern are ignored except in test/ directories. - const args = ['--test', testFixtures]; - const child = spawnSync(process.execPath, args); + const args = ['--test']; + const child = spawnSync(process.execPath, args, { cwd: testFixtures }); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); @@ -30,19 +30,19 @@ const testFixtures = fixtures.path('test-runner'); const stdout = child.stdout.toString(); assert.match(stdout, /ok 1 - this should pass/); assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); assert.match(stdout, /ok 4 - this should pass/); } { // Same but with a prototype mutation in require scripts. - const args = ['--require', join(testFixtures, 'protoMutation.js'), '--test', testFixtures]; - const child = spawnSync(process.execPath, args); + const args = ['--require', join(testFixtures, 'protoMutation.js'), '--test']; + const child = spawnSync(process.execPath, args, { cwd: testFixtures }); const stdout = child.stdout.toString(); assert.match(stdout, /ok 1 - this should pass/); assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); assert.match(stdout, /ok 4 - this should pass/); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); @@ -51,23 +51,19 @@ const testFixtures = fixtures.path('test-runner'); { // User specified files that don't match the pattern are still run. - const args = ['--test', testFixtures, join(testFixtures, 'index.js')]; - const child = spawnSync(process.execPath, args); + const args = ['--test', join(testFixtures, 'index.js')]; + const child = spawnSync(process.execPath, args, { cwd: testFixtures }); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); assert.strictEqual(child.stderr.toString(), ''); const stdout = child.stdout.toString(); assert.match(stdout, /not ok 1 - .+index\.js/); - assert.match(stdout, /ok 2 - this should pass/); - assert.match(stdout, /not ok 3 - this should fail/); - assert.match(stdout, /ok 4 - .+subdir.+subdir_test\.js/); - assert.match(stdout, /ok 5 - this should pass/); } { // Searches node_modules if specified. - const args = ['--test', join(testFixtures, 'node_modules')]; + const args = ['--test', join(testFixtures, 'node_modules/*.js')]; const child = spawnSync(process.execPath, args); assert.strictEqual(child.status, 1); @@ -89,7 +85,7 @@ const testFixtures = fixtures.path('test-runner'); const stdout = child.stdout.toString(); assert.match(stdout, /ok 1 - this should pass/); assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); assert.match(stdout, /ok 4 - this should pass/); } diff --git a/test/parallel/test-runner-test-filter.js b/test/parallel/test-runner-test-filter.js deleted file mode 100644 index b6afba22a2e814e..000000000000000 --- a/test/parallel/test-runner-test-filter.js +++ /dev/null @@ -1,42 +0,0 @@ -// Flags: --expose-internals -'use strict'; -require('../common'); -const assert = require('assert'); -const { doesPathMatchFilter } = require('internal/test_runner/utils'); - -// Paths expected to match -[ - 'test.js', - 'test.cjs', - 'test.mjs', - 'test-foo.js', - 'test-foo.cjs', - 'test-foo.mjs', - 'foo.test.js', - 'foo.test.cjs', - 'foo.test.mjs', - 'foo-test.js', - 'foo-test.cjs', - 'foo-test.mjs', - 'foo_test.js', - 'foo_test.cjs', - 'foo_test.mjs', -].forEach((p) => { - assert.strictEqual(doesPathMatchFilter(p), true); -}); - -// Paths expected not to match -[ - 'test', - 'test.djs', - 'test.cs', - 'test.mj', - 'foo.js', - 'test-foo.sj', - 'test.foo.js', - 'test_foo.js', - 'testfoo.js', - 'foo-test1.mjs', -].forEach((p) => { - assert.strictEqual(doesPathMatchFilter(p), false); -});