Add applyPatches script and some basic patch documentation

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false # prevent test to stop if one fails
       matrix:
-        node-version: [10.x, 12.x, 14.x, 16.x, 18.x]
+        node-version: [10, 12, 14, 16, 18]
         os: [ubuntu-latest] # Skip macos-latest, windows-latest for now
 
     runs-on: ${{ matrix.os }}
@@ -32,8 +32,10 @@ jobs:
         run: yarn install
 
       - name: Lint
-        if: matrix['node-version'] == '14.x' && matrix['os'] == 'ubuntu-latest'
         run: yarn lint
 
       - name: Build
         run: yarn build
+
+      - name: Check Patches
+        run: ./scripts/test_patch.sh node${{ matrix.node-version }}

README.md

@@ -43,3 +43,75 @@ This project deploys multiple defense measures to ensure that the safe binaries
   - Easy to spot a compromise.
 - `pkg-fetch` package on npm is strictly permission-controlled
   - Only authorized Vercel employees can push new revisions to npm.
+
+## Contributing Updates to Patches
+
+### Example workflow for applying patches to a new version of Node.js (18.13.0)
+
+1. Clone Node.js as a sibling to your current `pkg-fetch` clone
+
+- `git clone https://github.com/nodejs/node.git`
+- `cd node`
+
+2. Checkout the tag you wish to generate a patch for
+
+- `git checkout v18.13.0`
+
+3. Attempt to apply the closest patch (e.g. applying the existing patch for
+   18.12.1 when trying to generate a new patch for 18.13.0)
+
+- `git apply ..\pkg-fetch\patches\node.v18.12.1.cpp.patch --reject`
+
+4. If no rejects, great! you are ready to make your new patch file.
+
+- `git add -A`
+- `git diff --staged --src-prefix=node/ --dst-prefix=node/ > ..\pkg-fetch\patches\node.v18.13.0.cpp.patch`
+
+5. If rejects exist, resolve them yourself, and ensure all changes are saved,
+   and repeat step 4 to export the patch file
+
+#### Resolving Rejects
+
+Usually when a patch is rejected, it's because the context around the changes
+was refactored slightly since the last patched version. This is not usually
+complicated to resolve, but requires a human to interpret the changes since the
+last version `pkg` was patched against, compared with the version you wish to
+create a patch for.
+
+One method is to pull up the diff for the file where the rejects apply for the
+changes between the last tag (e.g. v18.12.1 to use the previous example) and the
+tag you want a patch for (e.g. v18.13.0 to use the previous example). Alongside
+this, have the `.rej` file and go through each rejected hunk by hunk and use
+your best judgement to determine how it should apply against the new tag.
+
+Save you results, and export the overall git diff with the commands from the
+example above.
+
+### Checking that patches apply cleanly
+
+The expectation is that a patch applies cleanly, with no delta or offsets from
+the source repo.
+
+When making a change to a patch file, it is possible to apply that patch without
+building by running
+
+`yarn applyPatches --node-range node18`
+
+where the `--node-range` can be specified to apply patches for the version of
+node for which you are updating patches. If unspecified, the latest node version
+in [patches.json](./patches/patches.json) will be used.
+
+Ultimately, the patch should result in fully functional node binary, but the
+`applyPatches` script can be used to quickly iterate just the application of
+the patches you are updating without needing to wait for the full build to
+complete.
+
+## Building a Binary Locally
+
+You can use the `yarn start` script to build the binary locally, which is helpful
+when updating patches to ensure functionality before pushing patch updates for
+review.
+
+For example:
+
+`yarn start --node-range node18 --arch x64 --output dist`

lib/apply-patches.ts

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+
+import yargs from 'yargs';
+
+import { log } from './log';
+import { getNodeVersion } from './index';
+import { version } from '../package.json';
+import { fetchExtractApply, prepBuildPath } from './build';
+
+async function applyPatchesOnVersion(nodeRange: string, quietExtraction = false) {
+  await prepBuildPath();
+  await fetchExtractApply(getNodeVersion(nodeRange), quietExtraction);
+}
+
+async function main() {
+  const { argv } = yargs
+    .option('node-range', { alias: 'n', default: 'latest', type: 'string' })
+    .option('quiet-extraction', { alias: 'q', type: 'boolean' })
+    .version(version)
+    .alias('v', 'version')
+    .help()
+    .alias('h', 'help');
+
+  const {
+    'node-range': nodeRange,
+    'quiet-extraction': quietExtraction,
+  } = argv;
+
+  await applyPatchesOnVersion(nodeRange, quietExtraction);
+}
+
+main().catch((error) => {
+  if (!error.wasReported) log.error(error);
+  process.exit(2);
+});

lib/build.ts

@@ -104,7 +104,7 @@ async function tarFetch(nodeVersion: string) {
   await downloadUrl(`${distUrl}/${tarName}`, archivePath);
 }
 
-async function tarExtract(nodeVersion: string) {
+async function tarExtract(nodeVersion: string, suppressTarOutput: boolean) {
   log.info('Extracting Node.js source archive...');
 
   const tarName = `node-${nodeVersion}.tar.gz`;
@@ -130,7 +130,9 @@ async function tarExtract(nodeVersion: string) {
   const extract = tar.extract(nodePath, {
     strip: 1,
     map: (header) => {
-      log.info(header.name);
+      if (!suppressTarOutput) {
+        log.info(header.name);
+      }
       return header;
     },
   });
@@ -159,6 +161,12 @@ async function applyPatches(nodeVersion: string) {
   }
 }
 
+export async function fetchExtractApply(nodeVersion: string, quietExtraction: boolean) {
+  await tarFetch(nodeVersion);
+  await tarExtract(nodeVersion, quietExtraction);
+  await applyPatches(nodeVersion);
+}
+
 async function compileOnWindows(
   nodeVersion: string,
   targetArch: string,
@@ -284,19 +292,20 @@ async function compile(
   return compileOnUnix(nodeVersion, targetArch, targetPlatform);
 }
 
+export async function prepBuildPath() {
+  await fs.remove(buildPath);
+  await fs.mkdirp(nodePath);
+  await fs.mkdirp(nodeArchivePath);
+}
+
 export default async function build(
   nodeVersion: string,
   targetArch: string,
   targetPlatform: string,
   local: string
 ) {
-  await fs.remove(buildPath);
-  await fs.mkdirp(nodePath);
-  await fs.mkdirp(nodeArchivePath);
-
-  await tarFetch(nodeVersion);
-  await tarExtract(nodeVersion);
-  await applyPatches(nodeVersion);
+  await prepBuildPath();
+  await fetchExtractApply(nodeVersion, false);
 
   const output = await compile(nodeVersion, targetArch, targetPlatform);
   const outputHash = await hash(output);

lib/index.ts

@@ -54,15 +54,23 @@ interface NeedOptions {
   arch: string;
 }
 
-export async function need(opts: NeedOptions) {
-  // eslint-disable-line complexity
-  const { forceFetch, forceBuild, dryRun, output } = opts || {};
-  let { nodeRange, platform, arch } = opts || {};
+export function satisfyingNodeVersion(nodeRange: string) {
+  const versions = Object.keys(patchesJson)
+    .filter((nv) => semver.satisfies(nv, nodeRange) || nodeRange === 'latest')
+    .sort((nv1, nv2) => (semver.gt(nv1, nv2) ? 1 : -1));
 
-  if (!nodeRange) throw wasReported('nodeRange not specified');
-  if (!platform) throw wasReported('platform not specified');
-  if (!arch) throw wasReported('arch not specified');
+  const nodeVersion = versions.pop();
+
+  if (!nodeVersion) {
+    throw wasReported(
+      `No available node version satisfies '${nodeRange}'`
+    );
+  }
 
+  return nodeVersion;
+}
+
+export function getNodeVersion(nodeRange: string) {
   nodeRange = abiToNodeRange(nodeRange); // 'm48' -> 'node6'
 
   if (!isValidNodeRange(nodeRange)) {
@@ -73,24 +81,24 @@ export async function need(opts: NeedOptions) {
     nodeRange = `v${nodeRange.slice(4)}`; // 'node6' -> 'v6' for semver
   }
 
-  platform = toFancyPlatform(platform); // win32 -> win
-  arch = toFancyArch(arch); // ia32 -> x86
+  const nodeVersion = satisfyingNodeVersion(nodeRange);
+  return nodeVersion;
+}
 
-  function satisfyingNodeVersion() {
-    const versions = Object.keys(patchesJson)
-      .filter((nv) => semver.satisfies(nv, nodeRange) || nodeRange === 'latest')
-      .sort((nv1, nv2) => (semver.gt(nv1, nv2) ? 1 : -1));
 
-    return versions.pop();
-  }
+export async function need(opts: NeedOptions) {
+  // eslint-disable-line complexity
+  const { forceFetch, forceBuild, dryRun, output, nodeRange } = opts || {};
+  let { platform, arch } = opts || {};
+
+  if (!nodeRange) throw wasReported('nodeRange not specified');
+  if (!platform) throw wasReported('platform not specified');
+  if (!arch) throw wasReported('arch not specified');
 
-  const nodeVersion = satisfyingNodeVersion();
+  platform = toFancyPlatform(platform); // win32 -> win
+  arch = toFancyArch(arch); // ia32 -> x86
 
-  if (!nodeVersion) {
-    throw wasReported(
-      `No available node version satisfies '${opts.nodeRange}'`
-    );
-  }
+  const nodeVersion = getNodeVersion(nodeRange);
 
   const fetched = localPlace({
     from: 'fetched',

package.json

@@ -50,7 +50,8 @@
     "lint": "eslint lib",
     "prepare": "npm run build",
     "prepublishOnly": "npm run lint",
-    "start": "node lib-es5/bin.js"
+    "start": "node lib-es5/bin.js",
+    "applyPatches": "node lib-es5/apply-patches.js"
   },
   "prettier": {
     "singleQuote": true

patches/node.v10.24.1.cpp.patch

@@ -306,7 +306,7 @@
      if (importModuleDynamically !== undefined) {
 --- node/src/inspector_agent.cc
 +++ node/src/inspector_agent.cc
-@@ -705,12 +705,10 @@
+@@ -712,12 +712,10 @@
      CHECK_EQ(0, uv_async_init(parent_env_->event_loop(),
                                &start_io_thread_async,
                                StartIoThreadAsyncCallback));

scripts/test_patch.sh

@@ -0,0 +1,53 @@
+node_range=$1
+
+if [ -z "$node_range" ]; then
+  echo "usage: ./test_patch.sh <nodeVersion>"
+  echo "       where nodeVersion is of the form 'node18' or 'node16'"
+  exit 1
+fi
+
+echo "Applying patches for $node_range"
+
+command="npm run applyPatches -- --node-range $node_range --quiet-extraction"
+output=$($command)
+status=$?
+
+if [ $status -ne 0 ]; then
+  echo -e "Command failed:\n$command\n$output"
+  exit 1
+fi
+
+echo "Checking output"
+
+expected_include_strings=("fetching" "extracting" "applying patches" "patching file")
+failing_strings=("failed" "offset" "rejects")
+
+found_all_expected_strings=true
+for s in "${expected_include_strings[@]}"; do
+  if [[ "${output,,}" != *"${s,,}"* ]]; then
+    found_all_expected_strings=false
+    echo -e "ERROR: Did not find \"$s\" in output"
+  fi
+done
+
+if [ "$found_all_expected_strings" = false ]; then
+  echo -e "\nDid not find the expected text when applying patches.\n\nOutput:\n$output"
+  exit 1
+fi
+
+line_errors=0
+while IFS= read -r line; do
+  for fString in "${failing_strings[@]}"; do
+    if [[ "${line,,}" == *"${fString,,}"* ]]; then
+      echo "ERROR: Found \"$fString\" in line: \"$line\"";
+      line_errors=$((line_errors + 1))
+    fi
+  done
+done <<< "$output"
+
+if [ $line_errors -gt 0 ]; then
+  echo "ERROR: errors found while attempting to apply patches"
+  exit $line_errors
+fi
+
+echo "All checks complete"