core(fr): add base fraggle rock snapshot runner

jest.config.js

@@ -22,6 +22,7 @@ module.exports = {
   testMatch: [
     '**/lighthouse-core/**/*-test.js',
     '**/lighthouse-cli/**/*-test.js',
+    '**/lighthouse-core/test/fraggle-rock/**/*-test-pptr.js',
     '**/lighthouse-treemap/**/*-test.js',
     '**/lighthouse-treemap/**/*-test-pptr.js',
     '**/lighthouse-viewer/**/*-test.js',

lighthouse-cli/test/fixtures/static-server.js

@@ -26,12 +26,18 @@ class Server {
     this._server = http.createServer(this._requestHandler.bind(this));
     /** @type {(data: string) => string=} */
     this._dataTransformer = undefined;
+    this._baseDir = __dirname;
   }
 
   getPort() {
     return this._server.address().port;
   }
 
+  /** @param {string} baseDir */
+  setBaseDir(baseDir) {
+    this._baseDir = baseDir;
+  }
+
   /**
    * @param {=number} port
    * @param {=string} hostname
@@ -64,8 +70,7 @@ class Server {
     const requestUrl = parseURL(request.url);
     const filePath = requestUrl.pathname;
     const queryString = requestUrl.search && parseQueryString(requestUrl.search.slice(1));
-    let absoluteFilePath = path.join(__dirname, filePath);
-
+    let absoluteFilePath = path.join(this._baseDir, filePath);
     const sendResponse = (statusCode, data) => {
       // Used by Smokerider.
       if (this._dataTransformer) data = this._dataTransformer(data);
@@ -227,6 +232,7 @@ if (require.main === module) {
   console.log(`offline: listening on http://localhost:${offlinePort}`);
 } else {
   module.exports = {
+    Server,
     server: serverForOnline,
     serverForOffline,
   };

lighthouse-core/fraggle-rock/api.js

@@ -0,0 +1,85 @@
+/**
+ * @license Copyright 2020 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const _ = require('lodash');
+const Driver = require('./gather/driver.js');
+const Runner = require('../runner.js');
+const Config = require('../config/config.js');
+
+/**
+ * @param {LH.Gatherer.GathererInstance} gatherer
+ * @return {gatherer is LH.Gatherer.FRGathererInstance}
+ */
+function isFRGatherer(gatherer) {
+  // TODO(FR-COMPAT): use configuration on gatherer.meta to detect interface compatibility
+  return gatherer.name === 'Accessibility';
+}
+
+/** @param {{page: import('puppeteer').Page, config?: LH.Config.Json}} options */
+async function snapshot(options) {
+  const config = new Config(options.config);
+  const driver = new Driver(options.page);
+  await driver.connect();
+
+  const url = await options.page.url();
+
+  return Runner.run(
+    async () => {
+      /** @type {LH.BaseArtifacts} */
+      const baseArtifacts = {
+        fetchTime: new Date().toJSON(),
+        LighthouseRunWarnings: [],
+        URL: {requestedUrl: url, finalUrl: url},
+        Timing: [],
+        Stacks: [],
+        settings: config.settings,
+        // TODO(FR-COMPAT): convert these to regular artifacts
+        HostFormFactor: 'mobile',
+        TestedAsMobileDevice: true,
+        HostUserAgent: 'unknown',
+        NetworkUserAgent: 'unknown',
+        BenchmarkIndex: 0,
+        InstallabilityErrors: {errors: []},
+        traces: {},
+        devtoolsLogs: {},
+        WebAppManifest: null,
+        PageLoadError: null,
+      };
+
+      const gatherers = (config.passes || [])
+        .map(pass => pass.gatherers)
+        .reduce((a, b) => a.concat(b), []);
+
+      /** @type {Partial<LH.GathererArtifacts>} */
+      const artifacts = {};
+
+      for (const {instance} of gatherers) {
+        // TODO(FR-COMPAT): use configuration on gatherer.meta to detect snapshot support
+        if (!isFRGatherer(instance)) continue;
+
+        /** @type {keyof LH.GathererArtifacts} */
+        const artifactName = instance.name;
+        const artifact = await Promise.resolve()
+          .then(() => instance.afterPass({driver}))
+          .catch(err => err);
+
+        // @ts-expect-error tsc can't yet express that artifactName is not a union of types.
+        artifacts[artifactName] = artifact;
+      }
+
+      return /** @type {LH.Artifacts} */ ({...baseArtifacts, ...artifacts}); // Cast to drop Partial<>
+    },
+    {
+      url,
+      config,
+    }
+  );
+}
+
+module.exports = {
+  snapshot,
+};

lighthouse-core/gather/gatherers/accessibility.js

@@ -92,6 +92,9 @@ function runA11yChecks() {
   });
 }
 
+/**
+ * @implements {LH.Gatherer.FRGathererInstance}
+ */
 class Accessibility extends Gatherer {
   /**
    * @param {LH.Gatherer.FRTransitionalContext} passContext

lighthouse-core/gather/gatherers/gatherer.js

@@ -5,8 +5,6 @@
  */
 'use strict';
 
-/** @typedef {void|LH.GathererArtifacts[keyof LH.GathererArtifacts]} PhaseResult */
-
 /**
  * Base class for all gatherers; defines pass lifecycle methods. The artifact
  * from the gatherer is the last not-undefined value returned by a lifecycle
@@ -16,6 +14,8 @@
  * If an Error is thrown (or a Promise that rejects on an Error),
  * the runner will treat it as an error internal to the gatherer and
  * continue execution of any remaining gatherers.
+ *
+ * @implements {LH.Gatherer.GathererInstance}
  */
 class Gatherer {
   /**
@@ -31,15 +31,15 @@ class Gatherer {
   /**
    * Called before navigation to target url.
    * @param {LH.Gatherer.PassContext} passContext
-   * @return {PhaseResult|Promise<PhaseResult>}
+   * @return {LH.Gatherer.PhaseResult}
    */
   beforePass(passContext) { }
 
   /**
    * Called after target page is loaded. If a trace is enabled for this pass,
    * the trace is still being recorded.
    * @param {LH.Gatherer.PassContext} passContext
-   * @return {PhaseResult|Promise<PhaseResult>}
+   * @return {LH.Gatherer.PhaseResult}
    */
   pass(passContext) { }
 
@@ -49,7 +49,7 @@ class Gatherer {
    * and record of network activity are provided in `loadData`.
    * @param {LH.Gatherer.PassContext} passContext
    * @param {LH.Gatherer.LoadData} loadData
-   * @return {PhaseResult|Promise<PhaseResult>}
+   * @return {LH.Gatherer.PhaseResult}
    */
   afterPass(passContext, loadData) { }
 

lighthouse-core/test/fixtures/fraggle-rock/snapshot-basic/onclick.html

@@ -0,0 +1,40 @@
+<!doctype html>
+<!--
+ * Copyright 2020 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+-->
+<html lang="en">
+<head>
+<title>Interactive Onclick Tester</title>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0">
+</head>
+<body>
+  Hello, Fraggle Rock!
+
+  This page has no accessibility violations until the mouse is clicked.
+
+  <template id="click-target">
+    <button>Click to add violations</button>
+  </template>
+
+  <template id="violations">
+    <input type="text" />
+  </template>
+
+  <script>
+    function addTemplate(selector) {
+      /** @type {HTMLTemplateElement} */
+      const template = document.querySelector(selector);
+      document.body.appendChild(template.content.cloneNode(true));
+    }
+
+    document.addEventListener('click', () => {
+      addTemplate('#violations');
+    });
+
+    addTemplate('#click-target');
+  </script>
+</body>
+</html>

lighthouse-core/test/fraggle-rock/api-test-pptr.js

@@ -0,0 +1,81 @@
+/**
+ * @license Copyright 2020 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-env jest */
+
+const path = require('path');
+const lighthouse = require('../../fraggle-rock/api.js');
+const puppeteer = require('puppeteer');
+const StaticServer = require('../../../lighthouse-cli/test/fixtures/static-server.js').Server;
+
+jest.setTimeout(90e3);
+
+describe('Fraggle Rock API', () => {
+  /** @type {InstanceType<StaticServer>} */
+  let server;
+  /** @type {import('puppeteer').Browser} */
+  let browser;
+  /** @type {import('puppeteer').Page} */
+  let page;
+
+  beforeAll(async () => {
+    server = new StaticServer();
+    await server.listen(0, '127.0.0.1');
+    browser = await puppeteer.launch({
+      headless: true,
+    });
+  });
+
+  beforeEach(async () => {
+    page = await browser.newPage();
+  });
+
+  afterEach(async () => {
+    await page.close();
+  });
+
+  afterAll(async () => {
+    await browser.close();
+    await server.close();
+  });
+
+  describe('snapshot', () => {
+    beforeEach(() => {
+      server.setBaseDir(path.join(__dirname, '../fixtures/fraggle-rock/snapshot-basic'));
+    });
+
+    it('should compute accessibility results on the page as-is', async () => {
+      await page.goto(`http://localhost:${server.getPort()}/onclick.html`);
+      // Wait for the javascript to run
+      await page.waitForSelector('button');
+      await page.click('button');
+      // Wait for the violations to appear
+      await page.waitForSelector('input');
+
+      const result = await lighthouse.snapshot({page});
+      if (!result) throw new Error('Lighthouse failed to produce a result');
+
+      const {lhr} = result;
+      const accessibility = lhr.categories.accessibility;
+      expect(accessibility.score).toBeLessThan(1);
+
+      const auditResults = accessibility.auditRefs.map(ref => lhr.audits[ref.id]);
+      const irrelevantDisplayModes = new Set(['notApplicable', 'manual']);
+      const applicableAudits = auditResults
+        .filter(audit => !irrelevantDisplayModes.has(audit.scoreDisplayMode));
+
+      const erroredAudits = applicableAudits
+        .filter(audit => audit.score === null);
+      expect(erroredAudits).toHaveLength(0);
+
+      const failedAuditIds = applicableAudits
+        .filter(audit => audit.score !== null && audit.score < 1)
+        .map(audit => audit.id);
+      expect(failedAuditIds).toContain('label');
+    });
+  });
+});

tsconfig.json

@@ -34,6 +34,7 @@
     "lighthouse-core/test/gather/driver/execution-context-test.js",
     "lighthouse-core/test/fraggle-rock/gather/session-test.js",
     "lighthouse-core/test/fraggle-rock/gather/driver-test.js",
+    "lighthouse-core/test/fraggle-rock/api-test-pptr.js",
     "lighthouse-core/test/gather/driver-test.js",
     "lighthouse-core/test/gather/gather-runner-test.js",
     "lighthouse-core/test/audits/script-treemap-data-test.js"

types/config.d.ts

@@ -4,9 +4,12 @@
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 
-import Gatherer = require('../lighthouse-core/gather/gatherers/gatherer.js');
 import Audit = require('../lighthouse-core/audits/audit.js');
 
+interface ClassOf<T> {
+  new (): T;
+}
+
 declare global {
   module LH {
     module Config {
@@ -41,12 +44,13 @@ declare global {
         path: string;
         options?: {};
       } | {
-        implementation: typeof Gatherer;
+        implementation: ClassOf<Gatherer.GathererInstance>;
         options?: {};
       } | {
-        instance: InstanceType<typeof Gatherer>;
+        instance: Gatherer.GathererInstance;
         options?: {};
-      } | Gatherer | typeof Gatherer | string;
+      } | Gatherer.GathererInstance | ClassOf<Gatherer.GathererInstance> | string;
+
 
       export interface CategoryJson {
         title: string | IcuMessage;
@@ -87,8 +91,8 @@ declare global {
       }
 
       export interface GathererDefn {
-        implementation?: typeof Gatherer;
-        instance: InstanceType<typeof Gatherer>;
+        implementation?: ClassOf<Gatherer.GathererInstance>;
+        instance: Gatherer.GathererInstance;
         path?: string;
       }
 
@@ -125,4 +129,4 @@ declare global {
 }
 
 // empty export to keep file a module
-export {}
+export {};

types/gatherer.d.ts

@@ -51,6 +51,21 @@ declare global {
       trace?: Trace;
     }
 
+    type PhaseResult_ = void|LH.GathererArtifacts[keyof LH.GathererArtifacts]
+    export type PhaseResult = PhaseResult_ | Promise<PhaseResult_>
+
+    export interface GathererInstance {
+      name: keyof LH.GathererArtifacts;
+      beforePass(context: LH.Gatherer.PassContext): PhaseResult;
+      pass(context: LH.Gatherer.PassContext): PhaseResult;
+      afterPass(context: LH.Gatherer.PassContext, loadData: LH.Gatherer.LoadData): PhaseResult;
+    }
+
+    export interface FRGathererInstance {
+      name: keyof LH.GathererArtifacts;
+      afterPass(context: FRTransitionalContext): PhaseResult;
+    }
+
     namespace Simulation {
       export type GraphNode = import('../lighthouse-core/lib/dependency-graph/base-node').Node;
       export type GraphNetworkNode = _NetworkNode;