diff --git a/lighthouse-cli/test/smokehouse/test-definitions/source-maps/expectations.js b/lighthouse-cli/test/smokehouse/test-definitions/source-maps/expectations.js index 8a4cd0df868f..5fe34e75c91f 100644 --- a/lighthouse-cli/test/smokehouse/test-definitions/source-maps/expectations.js +++ b/lighthouse-cli/test/smokehouse/test-definitions/source-maps/expectations.js @@ -14,6 +14,9 @@ const map = JSON.parse(mapJson); /** * @type {Array} * Expected Lighthouse audit values for seo tests + * + * We have experienced timeouts in the past when fetching source maps. + * We should verify the timing issue in Chromium if this gets flaky. */ const expectations = [ { diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 04cb48c9f733..62755027ccf7 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -28,6 +28,10 @@ class Fetcher { } /** + * Chrome M92 and above: + * We use `Network.loadNetworkResource` to fetch each resource. + * + * Chrome } */ - async fetchResource(url, {timeout = 500}) { + async fetchResource(url, options = {timeout: 500}) { if (!this._enabled) { - throw new Error('Must call `enableRequestInterception` before using fetchResource'); + throw new Error('Must call `enable` before using fetchResource'); + } + + // `Network.loadNetworkResource` was introduced in M88. + // The long timeout bug with `IO.read` was fixed in M92: + // https://bugs.chromium.org/p/chromium/issues/detail?id=1191757 + const milestone = await this.driver.getBrowserVersion().then(v => v.milestone); + if (milestone >= 92) { + return await this._fetchResourceOverProtocol(url, options); + } + return await this._fetchResourceIframe(url, options); + } + + /** + * @param {string} handle + * @param {{timeout: number}=} options, + * @return {Promise} + */ + async _readIOStream(handle, options = {timeout: 500}) { + const startTime = Date.now(); + + let ioResponse; + let data = ''; + while (!ioResponse || !ioResponse.eof) { + const elapsedTime = Date.now() - startTime; + if (elapsedTime > options.timeout) { + throw new Error('Waiting for the end of the IO stream exceeded the allotted time.'); + } + ioResponse = await this.driver.sendCommand('IO.read', {handle}); + const responseData = ioResponse.base64Encoded ? + Buffer.from(ioResponse.data, 'base64').toString('utf-8') : + ioResponse.data; + data = data.concat(responseData); } + return data; + } + + /** + * @param {string} url + * @return {Promise} + */ + async _loadNetworkResource(url) { + await this.driver.sendCommand('Network.enable'); + const frameTreeResponse = await this.driver.sendCommand('Page.getFrameTree'); + const networkResponse = await this.driver.sendCommand('Network.loadNetworkResource', { + frameId: frameTreeResponse.frameTree.frame.id, + url, + options: { + disableCache: true, + includeCredentials: true, + }, + }); + await this.driver.sendCommand('Network.disable'); + + if (!networkResponse.resource.success || !networkResponse.resource.stream) { + const statusCode = networkResponse.resource.httpStatusCode || ''; + throw new Error(`Loading network resource failed (${statusCode})`); + } + + return networkResponse.resource.stream; + } + + /** + * @param {string} url + * @param {{timeout: number}} options timeout is in ms + * @return {Promise} + */ + async _fetchResourceOverProtocol(url, options) { + const startTime = Date.now(); + + /** @type {NodeJS.Timeout} */ + let timeoutHandle; + const timeoutPromise = new Promise((_, reject) => { + timeoutHandle = setTimeout(() => { + reject(new Error('Timed out fetching resource')); + }, options.timeout); + }); + + const fetchStreamPromise = this._loadNetworkResource(url); + const stream = await Promise.race([fetchStreamPromise, timeoutPromise]) + .finally(() => clearTimeout(timeoutHandle)); + + return await this._readIOStream(stream, {timeout: options.timeout - (Date.now() - startTime)}); + } + + /** + * Fetches resource by injecting an iframe into the page. + * @param {string} url + * @param {{timeout: number}} options timeout is in ms + * @return {Promise} + */ + async _fetchResourceIframe(url, options) { /** @type {Promise} */ const requestInterceptionPromise = new Promise((resolve, reject) => { /** @param {LH.Crdp.Fetch.RequestPausedEvent} event */ @@ -167,7 +261,7 @@ class Fetcher { /** @type {Promise} */ const timeoutPromise = new Promise((_, reject) => { const errorMessage = 'Timed out fetching resource.'; - timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), timeout); + timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), options.timeout); }); const racePromise = Promise.race([ diff --git a/lighthouse-core/gather/gather-runner.js b/lighthouse-core/gather/gather-runner.js index cca8a8e07c4c..6b0145f07649 100644 --- a/lighthouse-core/gather/gather-runner.js +++ b/lighthouse-core/gather/gather-runner.js @@ -163,7 +163,7 @@ class GatherRunner { // Disable fetcher, in case a gatherer enabled it. // This cleanup should be removed once the only usage of // fetcher (fetching arbitrary URLs) is replaced by new protocol support. - await driver.fetcher.disableRequestInterception(); + await driver.fetcher.disable(); await driver.disconnect(); } catch (err) { @@ -683,7 +683,7 @@ class GatherRunner { // Noop if fetcher was never enabled. // This cleanup should be removed once the only usage of // fetcher (fetching arbitrary URLs) is replaced by new protocol support. - await driver.fetcher.disableRequestInterception(); + await driver.fetcher.disable(); } await GatherRunner.disposeDriver(driver, options); diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js index 52a5a07b3dd8..3c1566f161a3 100644 --- a/lighthouse-core/gather/gatherers/source-maps.js +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -131,7 +131,7 @@ class SourceMaps extends Gatherer { driver.off('Debugger.scriptParsed', this.onScriptParsed); await driver.sendCommand('Debugger.disable'); - await driver.fetcher.enableRequestInterception(); + await driver.fetcher.enable(); const eventProcessPromises = this._scriptParsedEvents .map((event) => this._retrieveMapFromScriptParsedEvent(driver, event)); return Promise.all(eventProcessPromises); diff --git a/lighthouse-core/lib/network-request.js b/lighthouse-core/lib/network-request.js index 9a889f87d68d..efbd3e51ecf1 100644 --- a/lighthouse-core/lib/network-request.js +++ b/lighthouse-core/lib/network-request.js @@ -66,6 +66,7 @@ const RESOURCE_TYPES = { Manifest: 'Manifest', SignedExchange: 'SignedExchange', Ping: 'Ping', + Preflight: 'Preflight', CSPViolationReport: 'CSPViolationReport', }; diff --git a/lighthouse-core/test/gather/driver-test.js b/lighthouse-core/test/gather/driver-test.js index b6855d6f9f51..0153bb301763 100644 --- a/lighthouse-core/test/gather/driver-test.js +++ b/lighthouse-core/test/gather/driver-test.js @@ -262,6 +262,7 @@ describe('.gotoURL', () => { id: 'ABC', loaderId: '', securityOrigin: '', mimeType: 'text/html', domainAndRegistry: '', secureContextType: /** @type {'Secure'} */ ('Secure'), crossOriginIsolatedContextType: /** @type {'Isolated'} */ ('Isolated'), + gatedAPIFeatures: [], }; navigate({...baseFrame, url: 'http://example.com'}); navigate({...baseFrame, url: 'https://example.com'}); diff --git a/lighthouse-core/test/gather/fake-driver.js b/lighthouse-core/test/gather/fake-driver.js index 555464093234..c469f7b6ed72 100644 --- a/lighthouse-core/test/gather/fake-driver.js +++ b/lighthouse-core/test/gather/fake-driver.js @@ -14,7 +14,7 @@ function makeFakeDriver({protocolGetVersionResponse}) { return { get fetcher() { return { - disableRequestInterception: () => Promise.resolve(), + disable: () => Promise.resolve(), }; }, getBrowserVersion() { diff --git a/lighthouse-core/test/gather/fetcher-test.js b/lighthouse-core/test/gather/fetcher-test.js new file mode 100644 index 000000000000..5787d124fd7f --- /dev/null +++ b/lighthouse-core/test/gather/fetcher-test.js @@ -0,0 +1,178 @@ +/** + * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const Fetcher = require('../../gather/fetcher.js'); +const Driver = require('../../gather/driver.js'); +const Connection = require('../../gather/connections/connection.js'); +const {createMockSendCommandFn} = require('../test-utils.js'); + +/* eslint-env jest */ + +/** @type {Connection} */ +let connectionStub; +/** @type {Driver} */ +let driver; +/** @type {Fetcher} */ +let fetcher; +/** @type {number} */ +let browserMilestone; + +beforeEach(() => { + connectionStub = new Connection(); + driver = new Driver(connectionStub); + fetcher = new Fetcher(driver); + browserMilestone = 92; + driver.getBrowserVersion = jest.fn().mockImplementation(() => { + return Promise.resolve({milestone: browserMilestone}); + }); +}); + +describe('.fetchResource', () => { + beforeEach(() => { + fetcher._enabled = true; + fetcher._fetchResourceOverProtocol = jest.fn().mockReturnValue(Promise.resolve('PROTOCOL')); + fetcher._fetchResourceIframe = jest.fn().mockReturnValue(Promise.resolve('IFRAME')); + }); + + it('throws if fetcher not enabled', async () => { + fetcher._enabled = false; + const resultPromise = fetcher.fetchResource('https://example.com'); + await expect(resultPromise).rejects.toThrow(/Must call `enable`/); + }); + + it('calls fetchResourceOverProtocol in newer chrome', async () => { + const result = await fetcher.fetchResource('https://example.com'); + expect(result).toEqual('PROTOCOL'); + expect(fetcher._fetchResourceOverProtocol).toHaveBeenCalled(); + expect(fetcher._fetchResourceIframe).not.toHaveBeenCalled(); + }); + + it('calls fetchResourceIframe in chrome before M92', async () => { + browserMilestone = 91; + const result = await fetcher.fetchResource('https://example.com'); + expect(result).toEqual('IFRAME'); + expect(fetcher._fetchResourceOverProtocol).not.toHaveBeenCalled(); + expect(fetcher._fetchResourceIframe).toHaveBeenCalled(); + }); +}); + +describe('._readIOStream', () => { + it('reads contents of stream', async () => { + connectionStub.sendCommand = createMockSendCommandFn() + .mockResponse('IO.read', {data: 'Hello World!', eof: true, base64Encoded: false}); + + const data = await fetcher._readIOStream('1'); + expect(data).toEqual('Hello World!'); + }); + + it('combines multiple reads', async () => { + connectionStub.sendCommand = createMockSendCommandFn() + .mockResponse('IO.read', {data: 'Hello ', eof: false, base64Encoded: false}) + .mockResponse('IO.read', {data: 'World', eof: false, base64Encoded: false}) + .mockResponse('IO.read', {data: '!', eof: true, base64Encoded: false}); + + const data = await fetcher._readIOStream('1'); + expect(data).toEqual('Hello World!'); + }); + + it('decodes if base64', async () => { + const buffer = Buffer.from('Hello World!').toString('base64'); + connectionStub.sendCommand = createMockSendCommandFn() + .mockResponse('IO.read', {data: buffer, eof: true, base64Encoded: true}); + + const data = await fetcher._readIOStream('1'); + expect(data).toEqual('Hello World!'); + }); + + it('decodes multiple base64 reads', async () => { + const buffer1 = Buffer.from('Hello ').toString('base64'); + const buffer2 = Buffer.from('World!').toString('base64'); + connectionStub.sendCommand = createMockSendCommandFn() + .mockResponse('IO.read', {data: buffer1, eof: false, base64Encoded: true}) + .mockResponse('IO.read', {data: buffer2, eof: true, base64Encoded: true}); + + const data = await fetcher._readIOStream('1'); + expect(data).toEqual('Hello World!'); + }); + + it('throws on timeout', async () => { + connectionStub.sendCommand = jest.fn() + .mockReturnValue(Promise.resolve({data: 'No stop', eof: false, base64Encoded: false})); + + const dataPromise = fetcher._readIOStream('1', {timeout: 50}); + await expect(dataPromise).rejects.toThrowError(/Waiting for the end of the IO stream/); + }); +}); + +describe('._fetchResourceOverProtocol', () => { + /** @type {string} */ + let streamContents; + + beforeEach(() => { + streamContents = 'STREAM CONTENTS'; + fetcher._readIOStream = jest.fn().mockImplementation(() => { + return Promise.resolve(streamContents); + }); + }); + + it('fetches a file', async () => { + connectionStub.sendCommand = createMockSendCommandFn() + .mockResponse('Network.enable') + .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}}) + .mockResponse('Network.loadNetworkResource', { + resource: {success: true, httpStatusCode: 200, stream: '1'}, + }) + .mockResponse('Network.disable'); + + const data = await fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 500}); + expect(data).toEqual(streamContents); + }); + + it('throws when resource could not be fetched', async () => { + connectionStub.sendCommand = createMockSendCommandFn() + .mockResponse('Network.enable') + .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}}) + .mockResponse('Network.loadNetworkResource', { + resource: {success: false, httpStatusCode: 404}, + }) + .mockResponse('Network.disable'); + + const dataPromise = fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 500}); + await expect(dataPromise).rejects.toThrowError(/Loading network resource failed/); + }); + + it('throws on timeout', async () => { + connectionStub.sendCommand = createMockSendCommandFn() + .mockResponse('Network.enable') + .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}}) + .mockResponse('Network.loadNetworkResource', { + resource: {success: false, httpStatusCode: 404}, + }, 100) + .mockResponse('Network.disable'); + + const dataPromise = fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 50}); + await expect(dataPromise).rejects.toThrowError(/Timed out fetching resource/); + }); + + it('uses remaining time on _readIOStream', async () => { + connectionStub.sendCommand = createMockSendCommandFn() + .mockResponse('Network.enable') + .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}}) + .mockResponse('Network.loadNetworkResource', { + resource: {success: true, httpStatusCode: 200, stream: '1'}, + }, 500) + .mockResponse('Network.disable'); + + let timeout; + fetcher._readIOStream = jest.fn().mockImplementation((_, options) => { + timeout = options.timeout; + }); + + await fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 1000}); + expect(timeout).toBeCloseTo(500, -2); + }); +}); diff --git a/lighthouse-core/test/gather/gatherers/source-maps-test.js b/lighthouse-core/test/gather/gatherers/source-maps-test.js index 9dc5a780b9e7..8ee4f4199b25 100644 --- a/lighthouse-core/test/gather/gatherers/source-maps-test.js +++ b/lighthouse-core/test/gather/gatherers/source-maps-test.js @@ -45,6 +45,7 @@ describe('SourceMaps gatherer', () => { const sendCommandMock = createMockSendCommandFn() .mockResponse('Debugger.enable', {}) .mockResponse('Debugger.disable', {}) + .mockResponse('Network.enable', {}) .mockResponse('Fetch.enable', {}) .mockResponse('Fetch.disable', {}); const fetchMock = jest.fn(); diff --git a/package.json b/package.json index cc674538fe44..a6e9f7fc5663 100644 --- a/package.json +++ b/package.json @@ -129,7 +129,7 @@ "cpy": "^7.0.1", "cross-env": "^7.0.2", "csv-validator": "^0.0.3", - "devtools-protocol": "0.0.805376", + "devtools-protocol": "0.0.859327", "eslint": "^7.23.0", "eslint-config-google": "^0.9.1", "eslint-plugin-local-rules": "0.1.0", diff --git a/yarn.lock b/yarn.lock index 7fb9db5a45ac..7da40a0e8034 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2919,10 +2919,10 @@ detective@^5.2.0: defined "^1.0.0" minimist "^1.1.1" -devtools-protocol@0.0.805376: - version "0.0.805376" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.805376.tgz#7ea29e412bfea69e9f2e77bcbafe83c898ad23bd" - integrity sha512-hZBiZTkVOAiWN7eI3oL1ftYtSi/HN8qn7/QYtDUNf9qVCG9/8pt+KyhL3Qoat6nXgiYiyreaz0mr6iB9Edw/sw== +devtools-protocol@0.0.859327: + version "0.0.859327" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.859327.tgz#4ea4a9cfc8a4ba492f3ba57b4109d2c5fe99474a" + integrity sha512-vRlLFY8Y2p3UnuDPRF0tsjHgXFM9JTS8T/p2F2YB9ukfpV+HByJU4kJc/Ks2/PozQ2bIQqmcYhDrSaFvs1Cy8Q== diff-sequences@^26.6.2: version "26.6.2"