Skip to content

Commit

Permalink
bootstrap: make CJS loader snapshotable
Browse files Browse the repository at this point in the history 
This patch makes the top-level access to runtime states in the
CJS loader lazy, and move the side-effects into a
initializeCJS() function that gets called during pre-execution.
As a result the CJS loader can be included into the built-in
snapshot.

PR-URL: #45849
Reviewed-By: Geoffrey Booth <[email protected]>
Reviewed-By: Chengzhong Wu <[email protected]>
  • Loading branch information
@joyeecheung @panva
joyeecheung authored and panva committed Jan 31, 2023
1 parent e83051c commit b1a1798
Show file tree
Hide file tree
Showing 6 changed files with 140 additions and 120 deletions.
191 changes: 105 additions & 86 deletions lib/internal/modules/cjs/loader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ const cjsParseCache = new SafeWeakMap();
// Set first due to cycle with ESM loader functions.
module.exports = {
wrapSafe, Module, toRealPath, readPackageScope, cjsParseCache,
get hasLoadedAnyUserCJSModule() { return hasLoadedAnyUserCJSModule; }
get hasLoadedAnyUserCJSModule() { return hasLoadedAnyUserCJSModule; },
initializeCJS,
};

const { BuiltinModule } = require('internal/bootstrap/loaders');
Expand All @@ -86,8 +87,8 @@ const {
kEmptyObject,
filterOwnProperties,
setOwnProperty,
getLazy,
} = require('internal/util');
const { Script } = require('vm');
const { internalCompileFunction } = require('internal/vm');
const assert = require('internal/assert');
const fs = require('fs');
Expand All @@ -97,21 +98,24 @@ const { sep } = path;
const { internalModuleStat } = internalBinding('fs');
const { safeGetenv } = internalBinding('credentials');
const {
cjsConditions,
getCjsConditions,
initializeCjsConditions,
hasEsmSyntax,
loadBuiltinModule,
makeRequireFunction,
normalizeReferrerURL,
stripBOM,
} = require('internal/modules/helpers');
const { getOptionValue } = require('internal/options');
const preserveSymlinks = getOptionValue('--preserve-symlinks');
const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main');
const shouldReportRequiredModules = process.env.WATCH_REPORT_DEPENDENCIES;
// Do not eagerly grab .manifest, it may be in TDZ
const policy = getOptionValue('--experimental-policy') ?
require('internal/process/policy') :
null;
const packageJsonReader = require('internal/modules/package_json_reader');
const { getOptionValue, getEmbedderOptions } = require('internal/options');
const policy = getLazy(
() => (getOptionValue('--experimental-policy') ? require('internal/process/policy') : null)
);
const shouldReportRequiredModules = getLazy(() => process.env.WATCH_REPORT_DEPENDENCIES);

const getCascadedLoader = getLazy(
() => require('internal/process/esm_loader').esmLoader
);

// Whether any user-provided CJS modules had been loaded (executed).
// Used for internal assertions.
Expand All @@ -127,7 +131,6 @@ const {
setArrowMessage,
} = require('internal/errors');
const { validateString } = require('internal/validators');
const pendingDeprecation = getOptionValue('--pending-deprecation');

const {
CHAR_BACKWARD_SLASH,
Expand All @@ -140,15 +143,7 @@ const {
isProxy
} = require('internal/util/types');

const asyncESM = require('internal/process/esm_loader');
const { enrichCJSError } = require('internal/modules/esm/translators');
const { kEvaluated } = internalBinding('module_wrap');
const {
encodedSepRegEx,
packageExportsResolve,
packageImportsResolve
} = require('internal/modules/esm/resolve');

const isWindows = process.platform === 'win32';

const relativeResolveCache = ObjectCreate(null);
Expand Down Expand Up @@ -190,13 +185,13 @@ function updateChildren(parent, child, scan) {
}

function reportModuleToWatchMode(filename) {
if (shouldReportRequiredModules && process.send) {
if (shouldReportRequiredModules() && process.send) {
process.send({ 'watch:require': [filename] });
}
}

function reportModuleNotFoundToWatchMode(basePath, extensions) {
if (shouldReportRequiredModules && process.send) {
if (shouldReportRequiredModules() && process.send) {
process.send({ 'watch:require': ArrayPrototypeMap(extensions, (ext) => path.resolve(`${basePath}${ext}`)) });
}
}
Expand All @@ -213,22 +208,6 @@ function Module(id = '', parent) {
this.children = [];
}

const builtinModules = [];
for (const { 0: id, 1: mod } of BuiltinModule.map) {
if (mod.canBeRequiredByUsers &&
BuiltinModule.canBeRequiredWithoutScheme(id)) {
ArrayPrototypePush(builtinModules, id);
}
}

const allBuiltins = new SafeSet(
ArrayPrototypeFlatMap(builtinModules, (bm) => [bm, `node:${bm}`])
);
BuiltinModule.getSchemeOnlyModuleNames().forEach((builtin) => allBuiltins.add(`node:${builtin}`));

ObjectFreeze(builtinModules);
Module.builtinModules = builtinModules;

Module._cache = ObjectCreate(null);
Module._pathCache = ObjectCreate(null);
Module._extensions = ObjectCreate(null);
Expand Down Expand Up @@ -297,26 +276,59 @@ function setModuleParent(value) {
moduleParentCache.set(this, value);
}

ObjectDefineProperty(Module.prototype, 'parent', {
__proto__: null,
get: pendingDeprecation ? deprecate(
getModuleParent,
'module.parent is deprecated due to accuracy issues. Please use ' +
'require.main to find program entry point instead.',
'DEP0144'
) : getModuleParent,
set: pendingDeprecation ? deprecate(
setModuleParent,
'module.parent is deprecated due to accuracy issues. Please use ' +
'require.main to find program entry point instead.',
'DEP0144'
) : setModuleParent,
});

let debug = require('internal/util/debuglog').debuglog('module', (fn) => {
debug = fn;
});
Module._debug = deprecate(debug, 'Module._debug is deprecated.', 'DEP0077');

const builtinModules = [];
// This function is called during pre-execution, before any user code is run.
function initializeCJS() {
const pendingDeprecation = getOptionValue('--pending-deprecation');
ObjectDefineProperty(Module.prototype, 'parent', {
__proto__: null,
get: pendingDeprecation ? deprecate(
getModuleParent,
'module.parent is deprecated due to accuracy issues. Please use ' +
'require.main to find program entry point instead.',
'DEP0144'
) : getModuleParent,
set: pendingDeprecation ? deprecate(
setModuleParent,
'module.parent is deprecated due to accuracy issues. Please use ' +
'require.main to find program entry point instead.',
'DEP0144'
) : setModuleParent,
});
Module._debug = deprecate(debug, 'Module._debug is deprecated.', 'DEP0077');

for (const { 0: id, 1: mod } of BuiltinModule.map) {
if (mod.canBeRequiredByUsers &&
BuiltinModule.canBeRequiredWithoutScheme(id)) {
ArrayPrototypePush(builtinModules, id);
}
}

const allBuiltins = new SafeSet(
ArrayPrototypeFlatMap(builtinModules, (bm) => [bm, `node:${bm}`])
);
BuiltinModule.getSchemeOnlyModuleNames().forEach((builtin) => allBuiltins.add(`node:${builtin}`));
ObjectFreeze(builtinModules);
Module.builtinModules = builtinModules;

Module.isBuiltin = function isBuiltin(moduleName) {
return allBuiltins.has(moduleName);
};

initializeCjsConditions();

if (!getEmbedderOptions().noGlobalSearchPaths) {
Module._initPaths();
}

// TODO(joyeecheung): deprecate this in favor of a proper hook?
Module.runMain =
require('internal/modules/run_main').executeUserEntryPoint;
}

// Given a module name, and a list of paths to test, returns the first
// matching file in the following precedence.
Expand All @@ -337,7 +349,6 @@ function readPackage(requestPath) {
const existing = packageJsonCache.get(jsonPath);
if (existing !== undefined) return existing;

const packageJsonReader = require('internal/modules/package_json_reader');
const result = packageJsonReader.read(jsonPath);
const json = result.containsKeys === false ? '{}' : result.string;
if (json === undefined) {
Expand Down Expand Up @@ -440,7 +451,7 @@ const realpathCache = new SafeMap();
function tryFile(requestPath, isMain) {
const rc = _stat(requestPath);
if (rc !== 0) return;
if (preserveSymlinks && !isMain) {
if (getOptionValue('--preserve-symlinks') && !isMain) {
return path.resolve(requestPath);
}
return toRealPath(requestPath);
Expand Down Expand Up @@ -511,9 +522,10 @@ function trySelf(parentPath, request) {
}

try {
const { packageExportsResolve } = require('internal/modules/esm/resolve');
return finalizeEsmResolution(packageExportsResolve(
pathToFileURL(pkgPath + '/package.json'), expansion, pkg,
pathToFileURL(parentPath), cjsConditions), parentPath, pkgPath);
pathToFileURL(parentPath), getCjsConditions()), parentPath, pkgPath);
} catch (e) {
if (e.code === 'ERR_MODULE_NOT_FOUND')
throw createEsmNotFoundErr(request, pkgPath + '/package.json');
Expand All @@ -535,9 +547,10 @@ function resolveExports(nmPath, request) {
const pkg = _readPackage(pkgPath);
if (pkg?.exports != null) {
try {
const { packageExportsResolve } = require('internal/modules/esm/resolve');
return finalizeEsmResolution(packageExportsResolve(
pathToFileURL(pkgPath + '/package.json'), '.' + expansion, pkg, null,
cjsConditions), null, pkgPath);
getCjsConditions()), null, pkgPath);
} catch (e) {
if (e.code === 'ERR_MODULE_NOT_FOUND')
throw createEsmNotFoundErr(request, pkgPath + '/package.json');
Expand Down Expand Up @@ -616,19 +629,19 @@ Module._findPath = function(request, paths, isMain) {
if (!trailingSlash) {
if (rc === 0) { // File.
if (!isMain) {
if (preserveSymlinks) {
if (getOptionValue('--preserve-symlinks')) {
filename = path.resolve(basePath);
} else {
filename = toRealPath(basePath);
}
} else if (preserveSymlinksMain) {
// For the main module, we use the preserveSymlinksMain flag instead
} else if (getOptionValue('--preserve-symlinks-main')) {
// For the main module, we use the --preserve-symlinks-main flag instead
// mainly for backward compatibility, as the preserveSymlinks flag
// historically has not applied to the main module. Most likely this
// was intended to keep .bin/ binaries working, as following those
// symlinks is usually required for the imports in the corresponding
// files to resolve; that said, in some use cases following symlinks
// causes bigger problems which is why the preserveSymlinksMain option
// causes bigger problems which is why the --preserve-symlinks-main option
// is needed.
filename = path.resolve(basePath);
} else {
Expand Down Expand Up @@ -1007,9 +1020,10 @@ Module._resolveFilename = function(request, parent, isMain, options) {
const pkg = readPackageScope(parentPath) || {};
if (pkg.data?.imports != null) {
try {
const { packageImportsResolve } = require('internal/modules/esm/resolve');
return finalizeEsmResolution(
packageImportsResolve(request, pathToFileURL(parentPath),
cjsConditions), parentPath,
getCjsConditions()), parentPath,
pkg.path);
} catch (e) {
if (e.code === 'ERR_MODULE_NOT_FOUND')
Expand Down Expand Up @@ -1051,6 +1065,7 @@ Module._resolveFilename = function(request, parent, isMain, options) {
};

function finalizeEsmResolution(resolved, parentPath, pkgPath) {
const { encodedSepRegEx } = require('internal/modules/esm/resolve');
if (RegExpPrototypeExec(encodedSepRegEx, resolved) !== null)
throw new ERR_INVALID_MODULE_SPECIFIER(
resolved, 'must not include encoded "/" or "\\" characters', parentPath);
Expand Down Expand Up @@ -1089,14 +1104,14 @@ Module.prototype.load = function(filename) {
Module._extensions[extension](this, filename);
this.loaded = true;

const esmLoader = asyncESM.esmLoader;
const cascadedLoader = getCascadedLoader();
// Create module entry at load time to snapshot exports correctly
const exports = this.exports;
// Preemptively cache
if ((module?.module === undefined ||
module.module.getStatus() < kEvaluated) &&
!esmLoader.cjsCache.has(this))
esmLoader.cjsCache.set(this, exports);
!cascadedLoader.cjsCache.has(this))
cascadedLoader.cjsCache.set(this, exports);
};


Expand All @@ -1121,17 +1136,20 @@ Module.prototype.require = function(id) {
// (needed for setting breakpoint when called with --inspect-brk)
let resolvedArgv;
let hasPausedEntry = false;

let Script;
function wrapSafe(filename, content, cjsModuleInstance) {
if (patched) {
const wrapper = Module.wrap(content);
if (Script === undefined) {
({ Script } = require('vm'));
}
const script = new Script(wrapper, {
filename,
lineOffset: 0,
importModuleDynamically: async (specifier, _, importAssertions) => {
const loader = asyncESM.esmLoader;
return loader.import(specifier, normalizeReferrerURL(filename),
importAssertions);
const cascadedLoader = getCascadedLoader();
return cascadedLoader.import(specifier, normalizeReferrerURL(filename),
importAssertions);
},
});

Expand All @@ -1155,9 +1173,9 @@ function wrapSafe(filename, content, cjsModuleInstance) {
], {
filename,
importModuleDynamically(specifier, _, importAssertions) {
const loader = asyncESM.esmLoader;
return loader.import(specifier, normalizeReferrerURL(filename),
importAssertions);
const cascadedLoader = getCascadedLoader();
return cascadedLoader.import(specifier, normalizeReferrerURL(filename),
importAssertions);
},
});

Expand All @@ -1168,8 +1186,10 @@ function wrapSafe(filename, content, cjsModuleInstance) {

return result.function;
} catch (err) {
if (process.mainModule === cjsModuleInstance)
if (process.mainModule === cjsModuleInstance) {
const { enrichCJSError } = require('internal/modules/esm/translators');
enrichCJSError(err, content);
}
throw err;
}
}
Expand All @@ -1181,10 +1201,11 @@ function wrapSafe(filename, content, cjsModuleInstance) {
Module.prototype._compile = function(content, filename) {
let moduleURL;
let redirects;
if (policy?.manifest) {
const manifest = policy()?.manifest;
if (manifest) {
moduleURL = pathToFileURL(filename);
redirects = policy.manifest.getDependencyMapper(moduleURL);
policy.manifest.assertIntegrity(moduleURL, content);
redirects = manifest.getDependencyMapper(moduleURL);
manifest.assertIntegrity(moduleURL, content);
}

const compiledWrapper = wrapSafe(filename, content, this);
Expand Down Expand Up @@ -1285,9 +1306,10 @@ Module._extensions['.js'] = function(module, filename) {
Module._extensions['.json'] = function(module, filename) {
const content = fs.readFileSync(filename, 'utf8');

if (policy?.manifest) {
const manifest = policy()?.manifest;
if (manifest) {
const moduleURL = pathToFileURL(filename);
policy.manifest.assertIntegrity(moduleURL, content);
manifest.assertIntegrity(moduleURL, content);
}

try {
Expand All @@ -1301,10 +1323,11 @@ Module._extensions['.json'] = function(module, filename) {

// Native extension for .node
Module._extensions['.node'] = function(module, filename) {
if (policy?.manifest) {
const manifest = policy()?.manifest;
if (manifest) {
const content = fs.readFileSync(filename);
const moduleURL = pathToFileURL(filename);
policy.manifest.assertIntegrity(moduleURL, content);
manifest.assertIntegrity(moduleURL, content);
}
// Be aware this doesn't use `content`
return process.dlopen(module, path.toNamespacedPath(filename));
Expand Down Expand Up @@ -1413,9 +1436,5 @@ Module.syncBuiltinESMExports = function syncBuiltinESMExports() {
}
};

Module.isBuiltin = function isBuiltin(moduleName) {
return allBuiltins.has(moduleName);
};

// Backwards compatibility
Module.Module = Module;
Loading

0 comments on commit b1a1798

Please sign in to comment.