diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69c2d642..28028f77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false # prevent test to stop if one fails matrix: - node-version: [10.x, 12.x, 14.x, 16.x, 18.x] + node-version: [10, 12, 14, 16, 18] os: [ubuntu-latest] # Skip macos-latest, windows-latest for now runs-on: ${{ matrix.os }} @@ -32,8 +32,10 @@ jobs: run: yarn install - name: Lint - if: matrix['node-version'] == '14.x' && matrix['os'] == 'ubuntu-latest' run: yarn lint - name: Build run: yarn build + + - name: Check Patches + run: ./scripts/test_patch.sh node${{ matrix.node-version }} diff --git a/README.md b/README.md index 4317475b..141760f0 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,75 @@ This project deploys multiple defense measures to ensure that the safe binaries - Easy to spot a compromise. - `pkg-fetch` package on npm is strictly permission-controlled - Only authorized Vercel employees can push new revisions to npm. + +## Contributing Updates to Patches + +### Example workflow for applying patches to a new version of Node.js (18.13.0) + +1. Clone Node.js as a sibling to your current `pkg-fetch` clone + +- `git clone https://github.com/nodejs/node.git` +- `cd node` + +2. Checkout the tag you wish to generate a patch for + +- `git checkout v18.13.0` + +3. Attempt to apply the closest patch (e.g. applying the existing patch for + 18.12.1 when trying to generate a new patch for 18.13.0) + +- `git apply ..\pkg-fetch\patches\node.v18.12.1.cpp.patch --reject` + +4. If no rejects, great! you are ready to make your new patch file. + +- `git add -A` +- `git diff --staged --src-prefix=node/ --dst-prefix=node/ > ..\pkg-fetch\patches\node.v18.13.0.cpp.patch` + +5. If rejects exist, resolve them yourself, and ensure all changes are saved, + and repeat step 4 to export the patch file + +#### Resolving Rejects + +Usually when a patch is rejected, it's because the context around the changes +was refactored slightly since the last patched version. This is not usually +complicated to resolve, but requires a human to interpret the changes since the +last version `pkg` was patched against, compared with the version you wish to +create a patch for. + +One method is to pull up the diff for the file where the rejects apply for the +changes between the last tag (e.g. v18.12.1 to use the previous example) and the +tag you want a patch for (e.g. v18.13.0 to use the previous example). Alongside +this, have the `.rej` file and go through each rejected hunk by hunk and use +your best judgement to determine how it should apply against the new tag. + +Save you results, and export the overall git diff with the commands from the +example above. + +### Checking that patches apply cleanly + +The expectation is that a patch applies cleanly, with no delta or offsets from +the source repo. + +When making a change to a patch file, it is possible to apply that patch without +building by running + +`yarn applyPatches --node-range node18` + +where the `--node-range` can be specified to apply patches for the version of +node for which you are updating patches. If unspecified, the latest node version +in [patches.json](./patches/patches.json) will be used. + +Ultimately, the patch should result in fully functional node binary, but the +`applyPatches` script can be used to quickly iterate just the application of +the patches you are updating without needing to wait for the full build to +complete. + +## Building a Binary Locally + +You can use the `yarn start` script to build the binary locally, which is helpful +when updating patches to ensure functionality before pushing patch updates for +review. + +For example: + +`yarn start --node-range node18 --arch x64 --output dist` diff --git a/lib/apply-patches.ts b/lib/apply-patches.ts new file mode 100644 index 00000000..b855b715 --- /dev/null +++ b/lib/apply-patches.ts @@ -0,0 +1,35 @@ +#!/usr/bin/env node + +import yargs from 'yargs'; + +import { log } from './log'; +import { getNodeVersion } from './index'; +import { version } from '../package.json'; +import { fetchExtractApply, prepBuildPath } from './build'; + +async function applyPatchesOnVersion(nodeRange: string, quietExtraction = false) { + await prepBuildPath(); + await fetchExtractApply(getNodeVersion(nodeRange), quietExtraction); +} + +async function main() { + const { argv } = yargs + .option('node-range', { alias: 'n', default: 'latest', type: 'string' }) + .option('quiet-extraction', { alias: 'q', type: 'boolean' }) + .version(version) + .alias('v', 'version') + .help() + .alias('h', 'help'); + + const { + 'node-range': nodeRange, + 'quiet-extraction': quietExtraction, + } = argv; + + await applyPatchesOnVersion(nodeRange, quietExtraction); +} + +main().catch((error) => { + if (!error.wasReported) log.error(error); + process.exit(2); +}); diff --git a/lib/build.ts b/lib/build.ts index 56785c03..9823fda9 100644 --- a/lib/build.ts +++ b/lib/build.ts @@ -104,7 +104,7 @@ async function tarFetch(nodeVersion: string) { await downloadUrl(`${distUrl}/${tarName}`, archivePath); } -async function tarExtract(nodeVersion: string) { +async function tarExtract(nodeVersion: string, suppressTarOutput: boolean) { log.info('Extracting Node.js source archive...'); const tarName = `node-${nodeVersion}.tar.gz`; @@ -130,7 +130,9 @@ async function tarExtract(nodeVersion: string) { const extract = tar.extract(nodePath, { strip: 1, map: (header) => { - log.info(header.name); + if (!suppressTarOutput) { + log.info(header.name); + } return header; }, }); @@ -159,6 +161,12 @@ async function applyPatches(nodeVersion: string) { } } +export async function fetchExtractApply(nodeVersion: string, quietExtraction: boolean) { + await tarFetch(nodeVersion); + await tarExtract(nodeVersion, quietExtraction); + await applyPatches(nodeVersion); +} + async function compileOnWindows( nodeVersion: string, targetArch: string, @@ -284,19 +292,20 @@ async function compile( return compileOnUnix(nodeVersion, targetArch, targetPlatform); } +export async function prepBuildPath() { + await fs.remove(buildPath); + await fs.mkdirp(nodePath); + await fs.mkdirp(nodeArchivePath); +} + export default async function build( nodeVersion: string, targetArch: string, targetPlatform: string, local: string ) { - await fs.remove(buildPath); - await fs.mkdirp(nodePath); - await fs.mkdirp(nodeArchivePath); - - await tarFetch(nodeVersion); - await tarExtract(nodeVersion); - await applyPatches(nodeVersion); + await prepBuildPath(); + await fetchExtractApply(nodeVersion, false); const output = await compile(nodeVersion, targetArch, targetPlatform); const outputHash = await hash(output); diff --git a/lib/index.ts b/lib/index.ts index 250c5846..12162da7 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -54,15 +54,23 @@ interface NeedOptions { arch: string; } -export async function need(opts: NeedOptions) { - // eslint-disable-line complexity - const { forceFetch, forceBuild, dryRun, output } = opts || {}; - let { nodeRange, platform, arch } = opts || {}; +export function satisfyingNodeVersion(nodeRange: string) { + const versions = Object.keys(patchesJson) + .filter((nv) => semver.satisfies(nv, nodeRange) || nodeRange === 'latest') + .sort((nv1, nv2) => (semver.gt(nv1, nv2) ? 1 : -1)); - if (!nodeRange) throw wasReported('nodeRange not specified'); - if (!platform) throw wasReported('platform not specified'); - if (!arch) throw wasReported('arch not specified'); + const nodeVersion = versions.pop(); + + if (!nodeVersion) { + throw wasReported( + `No available node version satisfies '${nodeRange}'` + ); + } + return nodeVersion; +} + +export function getNodeVersion(nodeRange: string) { nodeRange = abiToNodeRange(nodeRange); // 'm48' -> 'node6' if (!isValidNodeRange(nodeRange)) { @@ -73,24 +81,24 @@ export async function need(opts: NeedOptions) { nodeRange = `v${nodeRange.slice(4)}`; // 'node6' -> 'v6' for semver } - platform = toFancyPlatform(platform); // win32 -> win - arch = toFancyArch(arch); // ia32 -> x86 + const nodeVersion = satisfyingNodeVersion(nodeRange); + return nodeVersion; +} - function satisfyingNodeVersion() { - const versions = Object.keys(patchesJson) - .filter((nv) => semver.satisfies(nv, nodeRange) || nodeRange === 'latest') - .sort((nv1, nv2) => (semver.gt(nv1, nv2) ? 1 : -1)); - return versions.pop(); - } +export async function need(opts: NeedOptions) { + // eslint-disable-line complexity + const { forceFetch, forceBuild, dryRun, output, nodeRange } = opts || {}; + let { platform, arch } = opts || {}; + + if (!nodeRange) throw wasReported('nodeRange not specified'); + if (!platform) throw wasReported('platform not specified'); + if (!arch) throw wasReported('arch not specified'); - const nodeVersion = satisfyingNodeVersion(); + platform = toFancyPlatform(platform); // win32 -> win + arch = toFancyArch(arch); // ia32 -> x86 - if (!nodeVersion) { - throw wasReported( - `No available node version satisfies '${opts.nodeRange}'` - ); - } + const nodeVersion = getNodeVersion(nodeRange); const fetched = localPlace({ from: 'fetched', diff --git a/package.json b/package.json index 9240a4f5..c4aab3fd 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,8 @@ "lint": "eslint lib", "prepare": "npm run build", "prepublishOnly": "npm run lint", - "start": "node lib-es5/bin.js" + "start": "node lib-es5/bin.js", + "applyPatches": "node lib-es5/apply-patches.js" }, "prettier": { "singleQuote": true diff --git a/patches/node.v10.24.1.cpp.patch b/patches/node.v10.24.1.cpp.patch index 8533d454..a9fe8b46 100644 --- a/patches/node.v10.24.1.cpp.patch +++ b/patches/node.v10.24.1.cpp.patch @@ -306,7 +306,7 @@ if (importModuleDynamically !== undefined) { --- node/src/inspector_agent.cc +++ node/src/inspector_agent.cc -@@ -705,12 +705,10 @@ +@@ -712,12 +712,10 @@ CHECK_EQ(0, uv_async_init(parent_env_->event_loop(), &start_io_thread_async, StartIoThreadAsyncCallback)); diff --git a/scripts/test_patch.sh b/scripts/test_patch.sh new file mode 100755 index 00000000..d21dd75a --- /dev/null +++ b/scripts/test_patch.sh @@ -0,0 +1,53 @@ +node_range=$1 + +if [ -z "$node_range" ]; then + echo "usage: ./test_patch.sh " + echo " where nodeVersion is of the form 'node18' or 'node16'" + exit 1 +fi + +echo "Applying patches for $node_range" + +command="npm run applyPatches -- --node-range $node_range --quiet-extraction" +output=$($command) +status=$? + +if [ $status -ne 0 ]; then + echo -e "Command failed:\n$command\n$output" + exit 1 +fi + +echo "Checking output" + +expected_include_strings=("fetching" "extracting" "applying patches" "patching file") +failing_strings=("failed" "offset" "rejects") + +found_all_expected_strings=true +for s in "${expected_include_strings[@]}"; do + if [[ "${output,,}" != *"${s,,}"* ]]; then + found_all_expected_strings=false + echo -e "ERROR: Did not find \"$s\" in output" + fi +done + +if [ "$found_all_expected_strings" = false ]; then + echo -e "\nDid not find the expected text when applying patches.\n\nOutput:\n$output" + exit 1 +fi + +line_errors=0 +while IFS= read -r line; do + for fString in "${failing_strings[@]}"; do + if [[ "${line,,}" == *"${fString,,}"* ]]; then + echo "ERROR: Found \"$fString\" in line: \"$line\""; + line_errors=$((line_errors + 1)) + fi + done +done <<< "$output" + +if [ $line_errors -gt 0 ]; then + echo "ERROR: errors found while attempting to apply patches" + exit $line_errors +fi + +echo "All checks complete"