core(fetcher): fetch over protocol

lighthouse-core/gather/fetcher.js

@@ -28,6 +28,10 @@ class Fetcher {
   }
 
   /**
+   * Chrome M92 and above:
+   * We use `Network.loadNetworkResource` to fetch each resource.
+   *
+   * Chrome <M92:
    * The Fetch domain accepts patterns for controlling what requests are intercepted, but we
    * enable the domain for all patterns and filter events at a lower level to support multiple
    * concurrent usages. Reasons for this:
@@ -41,7 +45,7 @@ class Fetcher {
    * So instead we have one global `Fetch.enable` / `Fetch.requestPaused` pair, and allow specific
    * urls to be intercepted via `fetcher._setOnRequestPausedHandler`.
    */
-  async enableRequestInterception() {
+  async enable() {
     if (this._enabled) return;
 
     this._enabled = true;
@@ -51,7 +55,7 @@ class Fetcher {
     await this.driver.on('Fetch.requestPaused', this._onRequestPaused);
   }
 
-  async disableRequestInterception() {
+  async disable() {
     if (!this._enabled) return;
 
     this._enabled = false;
@@ -84,19 +88,108 @@ class Fetcher {
   }
 
   /**
-   * Requires that `driver.enableRequestInterception` has been called.
+   * Requires that `fetcher.enable` has been called.
    *
    * Fetches any resource in a way that circumvents CORS.
    *
    * @param {string} url
-   * @param {{timeout: number}} options timeout is in ms
+   * @param {{timeout: number}=} options timeout is in ms
    * @return {Promise<string>}
    */
-  async fetchResource(url, {timeout = 500}) {
+  async fetchResource(url, options = {timeout: 500}) {
     if (!this._enabled) {
-      throw new Error('Must call `enableRequestInterception` before using fetchResource');
+      throw new Error('Must call `enable` before using fetchResource');
+    }
+
+    // `Network.loadNetworkResource` was introduced in M88.
+    // The long timeout bug with `IO.read` was fixed in M92:
+    // https://bugs.chromium.org/p/chromium/issues/detail?id=1191757
+    const milestone = await this.driver.getBrowserVersion().then(v => v.milestone);
+    if (milestone >= 92) {
+      return await this._fetchResourceOverProtocol(url, options);
+    }
+    return await this._fetchResourceIframe(url, options);
+  }
+
+  /**
+   * @param {string} handle
+   * @param {{timeout: number}=} options,
+   * @return {Promise<string>}
+   */
+  async _readIOStream(handle, options = {timeout: 500}) {
+    const startTime = Date.now();
+
+    let ioResponse;
+    let data = '';
+    while (!ioResponse || !ioResponse.eof) {
+      const elapsedTime = Date.now() - startTime;
+      if (elapsedTime > options.timeout) {
+        throw new Error('Waiting for the end of the IO stream exceeded the allotted time.');
+      }
+      ioResponse = await this.driver.sendCommand('IO.read', {handle});
+      const responseData = ioResponse.base64Encoded ?
+        Buffer.from(ioResponse.data, 'base64').toString('utf-8') :
+        ioResponse.data;
+      data = data.concat(responseData);
+    }
+
+    return data;
+  }
+
+  /**
+   * @param {string} url
+   */
+  async _fetchStream(url) {
+    await this.driver.sendCommand('Network.enable');
+    const frameTreeResponse = await this.driver.sendCommand('Page.getFrameTree');
+    const networkResponse = await this.driver.sendCommand('Network.loadNetworkResource', {
+      frameId: frameTreeResponse.frameTree.frame.id,
+      url,
+      options: {
+        disableCache: true,
+        includeCredentials: true,
+      },
+    });
+    await this.driver.sendCommand('Network.disable');
+
+    if (!networkResponse.resource.success || !networkResponse.resource.stream) {
+      const statusCode = networkResponse.resource.httpStatusCode || '';
+      throw new Error(`Loading network resource failed (${statusCode})`);
     }
 
+    return networkResponse.resource.stream;
+  }
+
+  /**
+   * @param {string} url
+   * @param {{timeout: number}=} options timeout is in ms
+   * @return {Promise<string>}
+   */
+  async _fetchResourceOverProtocol(url, options = {timeout: 500}) {
+    const startTime = Date.now();
+
+    /** @type {NodeJS.Timeout} */
+    let timeoutHandle;
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutHandle = setTimeout(() => {
+        reject(new Error('Timed out fetching resource'));
+      }, options.timeout);
+    });
+
+    const fetchStreamPromise = this._fetchStream(url);
+    const stream = await Promise.race([fetchStreamPromise, timeoutPromise])
+      .finally(() => clearTimeout(timeoutHandle));
+
+    return await this._readIOStream(stream, {timeout: options.timeout - (Date.now() - startTime)});
+  }
+
+  /**
+   * Fetches resource by injecting an iframe into the page.
+   * @param {string} url
+   * @param {{timeout: number}=} options timeout is in ms
+   * @return {Promise<string>}
+   */
+  async _fetchResourceIframe(url, options = {timeout: 500}) {
     /** @type {Promise<string>} */
     const requestInterceptionPromise = new Promise((resolve, reject) => {
       /** @param {LH.Crdp.Fetch.RequestPausedEvent} event */
@@ -167,7 +260,7 @@ class Fetcher {
     /** @type {Promise<never>} */
     const timeoutPromise = new Promise((_, reject) => {
       const errorMessage = 'Timed out fetching resource.';
-      timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), timeout);
+      timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), options.timeout);
     });
 
     const racePromise = Promise.race([

lighthouse-core/gather/gather-runner.js

@@ -163,7 +163,7 @@ class GatherRunner {
       // Disable fetcher, in case a gatherer enabled it.
       // This cleanup should be removed once the only usage of
       // fetcher (fetching arbitrary URLs) is replaced by new protocol support.
-      await driver.fetcher.disableRequestInterception();
+      await driver.fetcher.disable();
 
       await driver.disconnect();
     } catch (err) {
@@ -683,7 +683,7 @@ class GatherRunner {
         // Noop if fetcher was never enabled.
         // This cleanup should be removed once the only usage of
         // fetcher (fetching arbitrary URLs) is replaced by new protocol support.
-        await driver.fetcher.disableRequestInterception();
+        await driver.fetcher.disable();
       }
 
       await GatherRunner.disposeDriver(driver, options);

lighthouse-core/gather/gatherers/source-maps.js

@@ -131,7 +131,7 @@ class SourceMaps extends Gatherer {
     driver.off('Debugger.scriptParsed', this.onScriptParsed);
     await driver.sendCommand('Debugger.disable');
 
-    await driver.fetcher.enableRequestInterception();
+    await driver.fetcher.enable();
     const eventProcessPromises = this._scriptParsedEvents
       .map((event) => this._retrieveMapFromScriptParsedEvent(driver, event));
     return Promise.all(eventProcessPromises);

lighthouse-core/lib/network-request.js

@@ -66,6 +66,7 @@ const RESOURCE_TYPES = {
   Manifest: 'Manifest',
   SignedExchange: 'SignedExchange',
   Ping: 'Ping',
+  Preflight: 'Preflight',
   CSPViolationReport: 'CSPViolationReport',
 };
 

lighthouse-core/test/gather/driver-test.js

@@ -262,6 +262,7 @@ describe('.gotoURL', () => {
       id: 'ABC', loaderId: '', securityOrigin: '', mimeType: 'text/html', domainAndRegistry: '',
       secureContextType: /** @type {'Secure'} */ ('Secure'),
       crossOriginIsolatedContextType: /** @type {'Isolated'} */ ('Isolated'),
+      gatedAPIFeatures: [],
     };
     navigate({...baseFrame, url: 'http://example.com'});
     navigate({...baseFrame, url: 'https://example.com'});

lighthouse-core/test/gather/fake-driver.js

@@ -14,7 +14,7 @@ function makeFakeDriver({protocolGetVersionResponse}) {
   return {
     get fetcher() {
       return {
-        disableRequestInterception: () => Promise.resolve(),
+        disable: () => Promise.resolve(),
       };
     },
     getBrowserVersion() {

lighthouse-core/test/gather/fetcher-test.js

@@ -0,0 +1,178 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const Fetcher = require('../../gather/fetcher.js');
+const Driver = require('../../gather/driver.js');
+const Connection = require('../../gather/connections/connection.js');
+const {createMockSendCommandFn} = require('../test-utils.js');
+
+/* eslint-env jest */
+
+/** @type {Connection} */
+let connectionStub;
+/** @type {Driver} */
+let driver;
+/** @type {Fetcher} */
+let fetcher;
+/** @type {number} */
+let browserMilestone;
+
+beforeEach(() => {
+  connectionStub = new Connection();
+  driver = new Driver(connectionStub);
+  fetcher = new Fetcher(driver);
+  browserMilestone = 92;
+  driver.getBrowserVersion = jest.fn().mockImplementation(() => {
+    return Promise.resolve({milestone: browserMilestone});
+  });
+});
+
+describe('.fetchResource', () => {
+  beforeEach(() => {
+    fetcher._enabled = true;
+    fetcher._fetchResourceOverProtocol = jest.fn().mockReturnValue(Promise.resolve('PROTOCOL'));
+    fetcher._fetchResourceIframe = jest.fn().mockReturnValue(Promise.resolve('IFRAME'));
+  });
+
+  it('throws if fetcher not enabled', async () => {
+    fetcher._enabled = false;
+    const resultPromise = fetcher.fetchResource('https://example.com');
+    await expect(resultPromise).rejects.toThrow(/Must call `enable`/);
+  });
+
+  it('calls fetchResourceOverProtocol in newer chrome', async () => {
+    const result = await fetcher.fetchResource('https://example.com');
+    expect(result).toEqual('PROTOCOL');
+    expect(fetcher._fetchResourceOverProtocol).toHaveBeenCalled();
+    expect(fetcher._fetchResourceIframe).not.toHaveBeenCalled();
+  });
+
+  it('calls fetchResourceIframe in chrome before M92', async () => {
+    browserMilestone = 91;
+    const result = await fetcher.fetchResource('https://example.com');
+    expect(result).toEqual('IFRAME');
+    expect(fetcher._fetchResourceOverProtocol).not.toHaveBeenCalled();
+    expect(fetcher._fetchResourceIframe).toHaveBeenCalled();
+  });
+});
+
+describe('._readIOStream', () => {
+  it('reads contents of stream', async () => {
+    connectionStub.sendCommand = createMockSendCommandFn()
+      .mockResponse('IO.read', {data: 'Hello World!', eof: true, base64Encoded: false});
+
+    const data = await fetcher._readIOStream('1');
+    expect(data).toEqual('Hello World!');
+  });
+
+  it('combines multiple reads', async () => {
+    connectionStub.sendCommand = createMockSendCommandFn()
+      .mockResponse('IO.read', {data: 'Hello ', eof: false, base64Encoded: false})
+      .mockResponse('IO.read', {data: 'World', eof: false, base64Encoded: false})
+      .mockResponse('IO.read', {data: '!', eof: true, base64Encoded: false});
+
+    const data = await fetcher._readIOStream('1');
+    expect(data).toEqual('Hello World!');
+  });
+
+  it('decodes if base64', async () => {
+    const buffer = Buffer.from('Hello World!').toString('base64');
+    connectionStub.sendCommand = createMockSendCommandFn()
+      .mockResponse('IO.read', {data: buffer, eof: true, base64Encoded: true});
+
+    const data = await fetcher._readIOStream('1');
+    expect(data).toEqual('Hello World!');
+  });
+
+  it('decodes multiple base64 reads', async () => {
+    const buffer1 = Buffer.from('Hello ').toString('base64');
+    const buffer2 = Buffer.from('World!').toString('base64');
+    connectionStub.sendCommand = createMockSendCommandFn()
+      .mockResponse('IO.read', {data: buffer1, eof: false, base64Encoded: true})
+      .mockResponse('IO.read', {data: buffer2, eof: true, base64Encoded: true});
+
+    const data = await fetcher._readIOStream('1');
+    expect(data).toEqual('Hello World!');
+  });
+
+  it('throws on timeout', async () => {
+    connectionStub.sendCommand = jest.fn()
+      .mockReturnValue(Promise.resolve({data: 'No stop', eof: false, base64Encoded: false}));
+
+    const dataPromise = fetcher._readIOStream('1', {timeout: 50});
+    await expect(dataPromise).rejects.toThrowError(/Waiting for the end of the IO stream/);
+  });
+});
+
+describe('._fetchResourceOverProtocol', () => {
+  /** @type {string} */
+  let streamContents;
+
+  beforeEach(() => {
+    streamContents = 'STREAM CONTENTS';
+    fetcher._readIOStream = jest.fn().mockImplementation(() => {
+      return Promise.resolve(streamContents);
+    });
+  });
+
+  it('fetches a file', async () => {
+    connectionStub.sendCommand = createMockSendCommandFn()
+      .mockResponse('Network.enable')
+      .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}})
+      .mockResponse('Network.loadNetworkResource', {
+        resource: {success: true, httpStatusCode: 200, stream: '1'},
+      })
+      .mockResponse('Network.disable');
+
+    const data = await fetcher._fetchResourceOverProtocol('https://example.com');
+    expect(data).toEqual(streamContents);
+  });
+
+  it('throws when resource could not be fetched', async () => {
+    connectionStub.sendCommand = createMockSendCommandFn()
+      .mockResponse('Network.enable')
+      .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}})
+      .mockResponse('Network.loadNetworkResource', {
+        resource: {success: false, httpStatusCode: 404},
+      })
+      .mockResponse('Network.disable');
+
+    const dataPromise = fetcher._fetchResourceOverProtocol('https://example.com');
+    await expect(dataPromise).rejects.toThrowError(/Loading network resource failed/);
+  });
+
+  it('throws on timeout', async () => {
+    connectionStub.sendCommand = createMockSendCommandFn()
+      .mockResponse('Network.enable')
+      .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}})
+      .mockResponse('Network.loadNetworkResource', {
+        resource: {success: false, httpStatusCode: 404},
+      }, 100)
+      .mockResponse('Network.disable');
+
+    const dataPromise = fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 50});
+    await expect(dataPromise).rejects.toThrowError(/Timed out fetching resource/);
+  });
+
+  it('uses remaining time on _readIOStream', async () => {
+    connectionStub.sendCommand = createMockSendCommandFn()
+      .mockResponse('Network.enable')
+      .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}})
+      .mockResponse('Network.loadNetworkResource', {
+        resource: {success: true, httpStatusCode: 200, stream: '1'},
+      }, 500)
+      .mockResponse('Network.disable');
+
+    let timeout;
+    fetcher._readIOStream = jest.fn().mockImplementation((_, options) => {
+      timeout = options.timeout;
+    });
+
+    await fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 1000});
+    expect(timeout).toBeCloseTo(500, -2);
+  });
+});

lighthouse-core/test/gather/gatherers/source-maps-test.js

@@ -45,6 +45,7 @@ describe('SourceMaps gatherer', () => {
     const sendCommandMock = createMockSendCommandFn()
       .mockResponse('Debugger.enable', {})
       .mockResponse('Debugger.disable', {})
+      .mockResponse('Network.enable', {})
       .mockResponse('Fetch.enable', {})
       .mockResponse('Fetch.disable', {});
     const fetchMock = jest.fn();

package.json

@@ -129,7 +129,7 @@
     "cpy": "^7.0.1",
     "cross-env": "^7.0.2",
     "csv-validator": "^0.0.3",
-    "devtools-protocol": "0.0.805376",
+    "devtools-protocol": "0.0.859327",
     "eslint": "^7.23.0",
     "eslint-config-google": "^0.9.1",
     "eslint-plugin-local-rules": "0.1.0",