From 45607bb72f79181d8fdddf9b5fc5de1182a4dfdf Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 11:36:20 +0000 Subject: [PATCH 01/13] Docs Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- docs/playwright.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/playwright.md b/docs/playwright.md index 315033955b2..a4c7d1b1886 100644 --- a/docs/playwright.md +++ b/docs/playwright.md @@ -77,6 +77,9 @@ test.use({ ``` The appropriate homeserver will be launched by the Playwright worker and reused for all tests which match the worker configuration. +Due to homeservers being reused between tests, please use unique names for any rooms put into the room directory as +they may be visible from other tests, the suggested approach is to use `testInfo.testId` within the name or lodash's uniqueId. +We remove public rooms from the directory between tests but deleting users doesn't have a homeserver agnostic solution. The logs from testcontainers will be attached to any reports output from Playwright. ## Writing Tests From 1b8f62abe2487be07017c3cb05c4cc36161a13c2 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 11:37:11 +0000 Subject: [PATCH 02/13] Avoid reusing user1234 Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .../forgot-password/forgot-password.spec.ts | 53 ++++++++++++------- .../one-to-one-chat/one-to-one-chat.spec.ts | 4 +- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/playwright/e2e/forgot-password/forgot-password.spec.ts b/playwright/e2e/forgot-password/forgot-password.spec.ts index 71475e892eb..af4e6def7ed 100644 --- a/playwright/e2e/forgot-password/forgot-password.spec.ts +++ b/playwright/e2e/forgot-password/forgot-password.spec.ts @@ -6,16 +6,25 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import { expect, test } from "../../element-web-test"; +import { expect, test as base } from "../../element-web-test"; import { selectHomeserver } from "../utils"; import { emailHomeserver } from "../../plugins/homeserver/synapse/emailHomeserver.ts"; import { isDendrite } from "../../plugins/homeserver/dendrite"; +import { Credentials } from "../../plugins/homeserver"; -const username = "user1234"; -// this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen. -const password = "oETo7MPf0o"; const email = "user@nowhere.dummy"; +const test = base.extend<{ credentials: Pick }>({ + // eslint-disable-next-line no-empty-pattern + credentials: async ({}, use, testInfo) => { + await use({ + username: `user_${testInfo.testId}`, + // this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen. + password: "oETo7MPf0o", + }); + }, +}); + test.use(emailHomeserver); test.use({ config: { @@ -45,31 +54,35 @@ test.describe("Forgot Password", () => { await expect(page.getByRole("main")).toMatchScreenshot("forgot-password.png"); }); - test("renders email verification dialog properly", { tag: "@screenshot" }, async ({ page, homeserver }) => { - const user = await homeserver.registerUser(username, password); + test( + "renders email verification dialog properly", + { tag: "@screenshot" }, + async ({ page, homeserver, credentials }) => { + const user = await homeserver.registerUser(credentials.username, credentials.password); - await homeserver.setThreepid(user.userId, "email", email); + await homeserver.setThreepid(user.userId, "email", email); - await page.goto("/"); + await page.goto("/"); - await page.getByRole("link", { name: "Sign in" }).click(); - await selectHomeserver(page, homeserver.baseUrl); + await page.getByRole("link", { name: "Sign in" }).click(); + await selectHomeserver(page, homeserver.baseUrl); - await page.getByRole("button", { name: "Forgot password?" }).click(); + await page.getByRole("button", { name: "Forgot password?" }).click(); - await page.getByRole("textbox", { name: "Email address" }).fill(email); + await page.getByRole("textbox", { name: "Email address" }).fill(email); - await page.getByRole("button", { name: "Send email" }).click(); + await page.getByRole("button", { name: "Send email" }).click(); - await page.getByRole("button", { name: "Next" }).click(); + await page.getByRole("button", { name: "Next" }).click(); - await page.getByRole("textbox", { name: "New Password", exact: true }).fill(password); - await page.getByRole("textbox", { name: "Confirm new password", exact: true }).fill(password); + await page.getByRole("textbox", { name: "New Password", exact: true }).fill(credentials.password); + await page.getByRole("textbox", { name: "Confirm new password", exact: true }).fill(credentials.password); - await page.getByRole("button", { name: "Reset password" }).click(); + await page.getByRole("button", { name: "Reset password" }).click(); - await expect(page.getByRole("button", { name: "Resend" })).toBeInViewport(); + await expect(page.getByRole("button", { name: "Resend" })).toBeInViewport(); - await expect(page.locator(".mx_Dialog")).toMatchScreenshot("forgot-password-verify-email.png"); - }); + await expect(page.locator(".mx_Dialog")).toMatchScreenshot("forgot-password-verify-email.png"); + }, + ); }); diff --git a/playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts b/playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts index deefb305dbc..8a4401f5f23 100644 --- a/playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts +++ b/playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts @@ -17,8 +17,8 @@ const test = base.extend<{ test.describe("1:1 chat room", () => { test.use({ displayName: "Jeff", - user2: async ({ homeserver }, use) => { - const credentials = await homeserver.registerUser("user1234", "p4s5W0rD", "Timmy"); + user2: async ({ homeserver }, use, testInfo) => { + const credentials = await homeserver.registerUser(`user2_${testInfo.testId}`, "p4s5W0rD", "Timmy"); await use(credentials); }, }); From b203e4b2a546d3edd3d7aab0ee4bf5947c915c88 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 11:37:20 +0000 Subject: [PATCH 03/13] Fix stale-screenshot-reporter.ts Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/stale-screenshot-reporter.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/playwright/stale-screenshot-reporter.ts b/playwright/stale-screenshot-reporter.ts index dc934827c1d..5c0e42ca2b2 100644 --- a/playwright/stale-screenshot-reporter.ts +++ b/playwright/stale-screenshot-reporter.ts @@ -20,10 +20,13 @@ const snapshotRoot = path.join(__dirname, "snapshots"); class StaleScreenshotReporter implements Reporter { private screenshots = new Set(); + private failing = false; private success = true; public onTestEnd(test: TestCase): void { - if (!test.ok()) return; + if (!test.ok()) { + this.failing = true; + } for (const annotation of test.annotations) { if (annotation.type === "_screenshot") { this.screenshots.add(annotation.description); @@ -40,6 +43,8 @@ class StaleScreenshotReporter implements Reporter { } public async onExit(): Promise { + if (this.failing) return; + const screenshotFiles = new Set(await glob(`**/*.png`, { cwd: snapshotRoot })); for (const screenshot of screenshotFiles) { if (screenshot.split("-").at(-1) !== "linux.png") { From 3eeb2216f9a79c124131b0631bf4920282167103 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 11:37:49 +0000 Subject: [PATCH 04/13] Clean up public rooms between tests on reused homeserver Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/services.ts | 1 + .../testcontainers/HomeserverContainer.ts | 6 +- playwright/testcontainers/dendrite.ts | 8 +- playwright/testcontainers/synapse.ts | 133 +++++++++++++----- 4 files changed, 104 insertions(+), 44 deletions(-) diff --git a/playwright/services.ts b/playwright/services.ts index b480cbc4054..1e2501dc2de 100644 --- a/playwright/services.ts +++ b/playwright/services.ts @@ -136,5 +136,6 @@ export const test = base.extend<{}, Services>({ await logger.testStarted(testInfo); await use(context); await logger.testFinished(testInfo); + await homeserver.onTestFinished(testInfo); }, }); diff --git a/playwright/testcontainers/HomeserverContainer.ts b/playwright/testcontainers/HomeserverContainer.ts index 09eea7da77a..73c9882418c 100644 --- a/playwright/testcontainers/HomeserverContainer.ts +++ b/playwright/testcontainers/HomeserverContainer.ts @@ -6,17 +6,17 @@ Please see LICENSE files in the repository root for full details. */ import { AbstractStartedContainer, GenericContainer } from "testcontainers"; -import { APIRequestContext } from "@playwright/test"; +import { APIRequestContext, TestInfo } from "@playwright/test"; -import { StartedSynapseContainer } from "./synapse.ts"; import { HomeserverInstance } from "../plugins/homeserver"; export interface HomeserverContainer extends GenericContainer { withConfigField(key: string, value: any): this; withConfig(config: Partial): this; - start(): Promise; + start(): Promise; } export interface StartedHomeserverContainer extends AbstractStartedContainer, HomeserverInstance { setRequest(request: APIRequestContext): void; + onTestFinished(testInfo: TestInfo): Promise; } diff --git a/playwright/testcontainers/dendrite.ts b/playwright/testcontainers/dendrite.ts index 629ea70c65c..517b0f7a642 100644 --- a/playwright/testcontainers/dendrite.ts +++ b/playwright/testcontainers/dendrite.ts @@ -235,7 +235,7 @@ export class DendriteContainer extends GenericContainer implements HomeserverCon return this; } - public override async start(): Promise { + public override async start(): Promise { this.withCopyContentToContainer([ { target: "/etc/dendrite/dendrite.yaml", @@ -244,8 +244,7 @@ export class DendriteContainer extends GenericContainer implements HomeserverCon ]); const container = await super.start(); - // Surprisingly, Dendrite implements the same register user Admin API Synapse, so we can just extend it - return new StartedSynapseContainer( + return new StartedDendriteContainer( container, `http://${container.getHost()}:${container.getMappedPort(8008)}`, this.config.client_api.registration_shared_secret, @@ -258,3 +257,6 @@ export class PineconeContainer extends DendriteContainer { super("matrixdotorg/dendrite-demo-pinecone:main", "/usr/bin/dendrite-demo-pinecone"); } } + +// Surprisingly, Dendrite implements the same register user Admin API Synapse, so we can just extend it +export class StartedDendriteContainer extends StartedSynapseContainer {} diff --git a/playwright/testcontainers/synapse.ts b/playwright/testcontainers/synapse.ts index 5111a6f0a66..27249b75952 100644 --- a/playwright/testcontainers/synapse.ts +++ b/playwright/testcontainers/synapse.ts @@ -6,7 +6,7 @@ Please see LICENSE files in the repository root for full details. */ import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers"; -import { APIRequestContext } from "@playwright/test"; +import { APIRequestContext, TestInfo } from "@playwright/test"; import crypto from "node:crypto"; import * as YAML from "yaml"; import { set } from "lodash"; @@ -138,6 +138,8 @@ const DEFAULT_CONFIG = { }, }; +type Verb = "GET" | "POST" | "PUT" | "DELETE"; + export type SynapseConfigOptions = Partial; export class SynapseContainer extends GenericContainer implements HomeserverContainer { @@ -228,8 +230,8 @@ export class SynapseContainer extends GenericContainer implements HomeserverCont } export class StartedSynapseContainer extends AbstractStartedContainer implements StartedHomeserverContainer { - private adminToken?: string; - private request?: APIRequestContext; + private adminTokenPromise?: Promise; + protected _request?: APIRequestContext; constructor( container: StartedTestContainer, @@ -240,7 +242,24 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements } public setRequest(request: APIRequestContext): void { - this.request = request; + this._request = request; + } + + public async onTestFinished(testInfo: TestInfo): Promise { + // Clean up the server to prevent rooms leaking between tests + await this.deletePublicRooms(); + } + + protected async deletePublicRooms(): Promise { + // We hide the rooms from the room directory to save time between tests and for portability between homeservers + const { chunk: rooms } = await this.request<{ + chunk: { room_id: string }[]; + }>("GET", "v3/publicRooms", {}); + await Promise.all( + rooms.map((room) => + this.request("PUT", `v3/directory/list/room/${room.room_id}`, { visibility: "private" }), + ), + ); } private async registerUserInternal( @@ -250,12 +269,12 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements admin = false, ): Promise { const url = `${this.baseUrl}/_synapse/admin/v1/register`; - const { nonce } = await this.request.get(url).then((r) => r.json()); + const { nonce } = await this._request.get(url).then((r) => r.json()); const mac = crypto .createHmac("sha1", this.registrationSharedSecret) .update(`${nonce}\0${username}\0${password}\0${admin ? "" : "not"}admin`) .digest("hex"); - const res = await this.request.post(url, { + const res = await this._request.post(url, { data: { nonce, username, @@ -282,23 +301,76 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements }; } + protected async getAdminToken(): Promise { + if (this.adminTokenPromise === undefined) { + this.adminTokenPromise = this.registerUserInternal( + "admin", + "totalyinsecureadminpassword", + undefined, + true, + ).then((res) => res.accessToken); + } + return this.adminTokenPromise; + } + + private async adminRequest(verb: "GET", path: string, data?: never): Promise; + private async adminRequest(verb: Verb, path: string, data?: object): Promise; + private async adminRequest(verb: Verb, path: string, data?: object): Promise { + const adminToken = await this.getAdminToken(); + const url = `${this.baseUrl}/_synapse/admin/${path}`; + const res = await this._request.fetch(url, { + data, + method: verb, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + if (!res.ok()) { + throw await res.json(); + } + + return res.json(); + } + + public async request(verb: "GET", path: string, data?: never): Promise; + public async request(verb: Verb, path: string, data?: object): Promise; + public async request(verb: Verb, path: string, data?: object): Promise { + const token = await this.getAdminToken(); + const url = `${this.baseUrl}/_matrix/client/${path}`; + const res = await this._request.fetch(url, { + data, + method: verb, + headers: { + Authorization: `Bearer ${token}`, + }, + }); + + if (!res.ok()) { + throw await res.json(); + } + + return res.json(); + } + public registerUser(username: string, password: string, displayName?: string): Promise { return this.registerUserInternal(username, password, displayName, false); } public async loginUser(userId: string, password: string): Promise { - const url = `${this.baseUrl}/_matrix/client/v3/login`; - const res = await this.request.post(url, { - data: { - type: "m.login.password", - identifier: { - type: "m.id.user", - user: userId, - }, - password: password, + const json = await this.request<{ + access_token: string; + user_id: string; + device_id: string; + home_server: string; + }>("POST", "v3/login", { + type: "m.login.password", + identifier: { + type: "m.id.user", + user: userId, }, + password: password, }); - const json = await res.json(); return { password, @@ -311,28 +383,13 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements } public async setThreepid(userId: string, medium: string, address: string): Promise { - if (this.adminToken === undefined) { - const result = await this.registerUserInternal("admin", "totalyinsecureadminpassword", undefined, true); - this.adminToken = result.accessToken; - } - - const url = `${this.baseUrl}/_synapse/admin/v2/users/${userId}`; - const res = await this.request.put(url, { - data: { - threepids: [ - { - medium, - address, - }, - ], - }, - headers: { - Authorization: `Bearer ${this.adminToken}`, - }, + await this.adminRequest("PUT", `v2/users/${userId}`, { + threepids: [ + { + medium, + address, + }, + ], }); - - if (!res.ok()) { - throw await res.json(); - } } } From b7b650e0fc581aabdfb38721ff4ad8f09b315475 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 11:38:12 +0000 Subject: [PATCH 05/13] Deflake spotlight when homeserver is reused Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/spotlight/spotlight.spec.ts | 195 +++++++++++---------- 1 file changed, 100 insertions(+), 95 deletions(-) diff --git a/playwright/e2e/spotlight/spotlight.spec.ts b/playwright/e2e/spotlight/spotlight.spec.ts index d1bb3dec258..da35ca57b35 100644 --- a/playwright/e2e/spotlight/spotlight.spec.ts +++ b/playwright/e2e/spotlight/spotlight.spec.ts @@ -6,8 +6,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import type { AccountDataEvents } from "matrix-js-sdk/src/matrix"; -import { test, expect } from "../../element-web-test"; +import type { AccountDataEvents, Visibility } from "matrix-js-sdk/src/matrix"; +import { test as base, expect } from "../../element-web-test"; import { Filter } from "../../pages/Spotlight"; import { Bot } from "../../pages/bot"; import type { Locator, Page } from "@playwright/test"; @@ -38,41 +38,37 @@ async function startDM(app: ElementAppPage, page: Page, name: string): Promise { - const bot1Name = "BotBob"; - let bot1: Bot; - - const bot2Name = "ByteBot"; - let bot2: Bot; - - const room1Name = "247"; - let room1Id: string; - - const room2Name = "Lounge"; - let room2Id: string; - - const room3Name = "Public"; - let room3Id: string; - - test.use({ - displayName: "Jim", - }); - - test.beforeEach(async ({ page, homeserver, app, user }) => { - bot1 = new Bot(page, homeserver, { displayName: bot1Name, autoAcceptInvites: true }); - bot2 = new Bot(page, homeserver, { displayName: bot2Name, autoAcceptInvites: true }); - const Visibility = await page.evaluate(() => (window as any).matrixcs.Visibility); - - room1Id = await app.client.createRoom({ name: room1Name, visibility: Visibility.Public }); - - await bot1.joinRoom(room1Id); - const bot1UserId = await bot1.evaluate((client) => client.getUserId()); - room2Id = await bot2.createRoom({ name: room2Name, visibility: Visibility.Public }); - await bot2.inviteUser(room2Id, bot1UserId); - - room3Id = await bot2.createRoom({ - name: room3Name, - visibility: Visibility.Public, +type RoomRef = { name: string; roomId: string }; +const test = base.extend<{ + bot1: Bot; + bot2: Bot; + room1: RoomRef; + room2: RoomRef; + room3: RoomRef; +}>({ + bot1: async ({ page, homeserver }, use, testInfo) => { + const bot = new Bot(page, homeserver, { displayName: `BotBob_${testInfo.testId}`, autoAcceptInvites: true }); + await use(bot); + }, + bot2: async ({ page, homeserver }, use, testInfo) => { + const bot = new Bot(page, homeserver, { displayName: `ByteBot_${testInfo.testId}`, autoAcceptInvites: true }); + await use(bot); + }, + room1: async ({ app }, use) => { + const name = "247"; + const roomId = await app.client.createRoom({ name, visibility: "public" as Visibility }); + await use({ name, roomId }); + }, + room2: async ({ bot2 }, use) => { + const name = "Lounge"; + const roomId = await bot2.createRoom({ name, visibility: "public" as Visibility }); + await use({ name, roomId }); + }, + room3: async ({ bot2 }, use) => { + const name = "Public"; + const roomId = await bot2.createRoom({ + name, + visibility: "public" as Visibility, initial_state: [ { type: "m.room.history_visibility", @@ -83,9 +79,21 @@ test.describe("Spotlight", () => { }, ], }); - await bot2.inviteUser(room3Id, bot1UserId); + await use({ name, roomId }); + }, +}); + +test.describe("Spotlight", () => { + test.use({ + displayName: "Jim", + }); + + test.beforeEach(async ({ page, user, bot1, bot2, room1, room2, room3 }) => { + await bot1.joinRoom(room1.roomId); + await bot2.inviteUser(room2.roomId, bot1.credentials.userId); + await bot2.inviteUser(room3.roomId, bot1.credentials.userId); - await page.goto("/#/room/" + room1Id); + await page.goto(`/#/room/${room1.roomId}`); await expect(page.locator(".mx_RoomSublist_skeletonUI")).not.toBeAttached(); }); @@ -117,69 +125,69 @@ test.describe("Spotlight", () => { await expect(spotlight.dialog.locator(".mx_SpotlightDialog_filter")).not.toBeAttached(); }); - test("should find joined rooms", async ({ page, app }) => { + test("should find joined rooms", async ({ page, app, room1 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle - await spotlight.search(room1Name); + await spotlight.search(room1.name); const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(room1Name); + await expect(resultLocator.first()).toContainText(room1.name); await resultLocator.first().click(); - await expect(page).toHaveURL(new RegExp(`#/room/${room1Id}`)); - await expect(roomHeaderName(page)).toContainText(room1Name); + await expect(page).toHaveURL(new RegExp(`#/room/${room1.roomId}`)); + await expect(roomHeaderName(page)).toContainText(room1.name); }); - test("should find known public rooms", async ({ page, app }) => { + test("should find known public rooms", async ({ page, app, room1 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.PublicRooms); - await spotlight.search(room1Name); + await spotlight.search(room1.name); const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(room1Name); + await expect(resultLocator.first()).toContainText(room1.name); await expect(resultLocator.first()).toContainText("View"); await resultLocator.first().click(); - await expect(page).toHaveURL(new RegExp(`#/room/${room1Id}`)); - await expect(roomHeaderName(page)).toContainText(room1Name); + await expect(page).toHaveURL(new RegExp(`#/room/${room1.roomId}`)); + await expect(roomHeaderName(page)).toContainText(room1.name); }); - test("should find unknown public rooms", async ({ page, app }) => { + test("should find unknown public rooms", async ({ page, app, room2 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.PublicRooms); - await spotlight.search(room2Name); + await spotlight.search(room2.name); const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(room2Name); + await expect(resultLocator.first()).toContainText(room2.name); await expect(resultLocator.first()).toContainText("Join"); await resultLocator.first().click(); - await expect(page).toHaveURL(new RegExp(`#/room/${room2Id}`)); + await expect(page).toHaveURL(new RegExp(`#/room/${room2.roomId}`)); await expect(page.locator(".mx_RoomView_MessageList")).toHaveCount(1); - await expect(roomHeaderName(page)).toContainText(room2Name); + await expect(roomHeaderName(page)).toContainText(room2.name); }); - test("should find unknown public world readable rooms", async ({ page, app }) => { + test("should find unknown public world readable rooms", async ({ page, app, room3 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.PublicRooms); - await spotlight.search(room3Name); + await spotlight.search(room3.name); const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(room3Name); + await expect(resultLocator.first()).toContainText(room3.name); await expect(resultLocator.first()).toContainText("View"); await resultLocator.first().click(); - await expect(page).toHaveURL(new RegExp(`#/room/${room3Id}`)); + await expect(page).toHaveURL(new RegExp(`#/room/${room3.roomId}`)); await page.getByRole("button", { name: "Join the discussion" }).click(); - await expect(roomHeaderName(page)).toHaveText(room3Name); + await expect(roomHeaderName(page)).toHaveText(room3.name); }); // TODO: We currently can’t test finding rooms on other homeservers/other protocols // We obviously don’t have federation or bridges in local e2e tests - test.skip("should find unknown public rooms on other homeservers", async ({ page, app }) => { + test.skip("should find unknown public rooms on other homeservers", async ({ page, app, room3 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.PublicRooms); - await spotlight.search(room3Name); + await spotlight.search(room3.name); await page.locator("[aria-haspopup=true][role=button]").click(); await page @@ -194,20 +202,20 @@ test.describe("Spotlight", () => { const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(room3Name); - await expect(resultLocator.first()).toContainText(room3Id); + await expect(resultLocator.first()).toContainText(room3.name); + await expect(resultLocator.first()).toContainText(room3.roomId); }); - test("should find known people", async ({ page, app }) => { + test("should find known people", async ({ page, app, bot1 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.People); - await spotlight.search(bot1Name); + await spotlight.search(bot1.credentials.displayName); const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(bot1Name); + await expect(resultLocator.first()).toContainText(bot1.credentials.displayName); await resultLocator.first().click(); - await expect(roomHeaderName(page)).toHaveText(bot1Name); + await expect(roomHeaderName(page)).toHaveText(bot1.credentials.displayName); }); /** @@ -217,42 +225,41 @@ test.describe("Spotlight", () => { * * https://github.com/matrix-org/synapse/issues/16472 */ - test("should find unknown people", async ({ page, app }) => { + test("should find unknown people", async ({ page, app, bot2 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.People); - await spotlight.search(bot2Name); + await spotlight.search(bot2.credentials.displayName); const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(bot2Name); + await expect(resultLocator.first()).toContainText(bot2.credentials.displayName); await resultLocator.first().click(); - await expect(roomHeaderName(page)).toHaveText(bot2Name); + await expect(roomHeaderName(page)).toHaveText(bot2.credentials.displayName); }); - test("should find group DMs by usernames or user ids", async ({ page, app }) => { + test("should find group DMs by usernames or user ids", async ({ page, app, bot1, bot2, room1 }) => { // First we want to share a room with both bots to ensure we’ve got their usernames cached - const bot2UserId = await bot2.evaluate((client) => client.getUserId()); - await app.client.inviteUser(room1Id, bot2UserId); + await app.client.inviteUser(room1.roomId, bot2.credentials.userId); // Starting a DM with ByteBot (will be turned into a group dm later) let spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.People); - await spotlight.search(bot2Name); + await spotlight.search(bot2.credentials.displayName); let resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(bot2Name); + await expect(resultLocator.first()).toContainText(bot2.credentials.displayName); await resultLocator.first().click(); // Send first message to actually start DM - await expect(roomHeaderName(page)).toHaveText(bot2Name); + await expect(roomHeaderName(page)).toHaveText(bot2.credentials.displayName); const locator = page.getByRole("textbox", { name: "Send a message…" }); await locator.fill("Hey!"); await locator.press("Enter"); // Assert DM exists by checking for the first message and the room being in the room list await expect(page.locator(".mx_EventTile_body").filter({ hasText: "Hey!" })).toBeAttached({ timeout: 3000 }); - await expect(page.getByRole("group", { name: "People" })).toContainText(bot2Name); + await expect(page.getByRole("group", { name: "People" })).toContainText(bot2.credentials.displayName); // Invite BotBob into existing DM with ByteBot const dmRooms = await app.client.evaluate((client, userId) => { @@ -260,18 +267,17 @@ test.describe("Spotlight", () => { .getAccountData("m.direct" as keyof AccountDataEvents) ?.getContent>(); return map[userId] ?? []; - }, bot2UserId); + }, bot2.credentials.userId); expect(dmRooms).toHaveLength(1); const groupDmName = await app.client.evaluate((client, id) => client.getRoom(id).name, dmRooms[0]); - const bot1UserId = await bot1.evaluate((client) => client.getUserId()); - await app.client.inviteUser(dmRooms[0], bot1UserId); + await app.client.inviteUser(dmRooms[0], bot1.credentials.userId); await expect(roomHeaderName(page).first()).toContainText(groupDmName); await expect(page.getByRole("group", { name: "People" }).first()).toContainText(groupDmName); // Search for BotBob by id, should return group DM and user spotlight = await app.openSpotlight(); await spotlight.filter(Filter.People); - await spotlight.search(bot1UserId); + await spotlight.search(bot1.credentials.userId); await page.waitForTimeout(1000); // wait for the dialog to settle resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(2); @@ -284,7 +290,7 @@ test.describe("Spotlight", () => { // Search for ByteBot by id, should return group DM and user spotlight = await app.openSpotlight(); await spotlight.filter(Filter.People); - await spotlight.search(bot2UserId); + await spotlight.search(bot2.credentials.userId); await page.waitForTimeout(1000); // wait for the dialog to settle resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(2); @@ -297,11 +303,10 @@ test.describe("Spotlight", () => { }); // Test against https://github.com/vector-im/element-web/issues/22851 - test("should show each person result only once", async ({ page, app }) => { + test("should show each person result only once", async ({ page, app, bot1 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.People); - const bot1UserId = await bot1.evaluate((client) => client.getUserId()); // 2 rounds of search to simulate the bug conditions. Specifically, the first search // should have 1 result (not 2) and the second search should also have 1 result (instead @@ -310,24 +315,24 @@ test.describe("Spotlight", () => { // We search for user ID to trigger the profile lookup within the dialog. for (let i = 0; i < 2; i++) { console.log("Iteration: " + i); - await spotlight.search(bot1UserId); + await spotlight.search(bot1.credentials.userId); await page.waitForTimeout(1000); // wait for the dialog to settle const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(bot1UserId); + await expect(resultLocator.first()).toContainText(bot1.credentials.userId); } }); - test("should allow opening group chat dialog", async ({ page, app }) => { + test("should allow opening group chat dialog", async ({ page, app, bot2 }) => { const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.People); - await spotlight.search(bot2Name); + await spotlight.search(bot2.credentials.displayName); await page.waitForTimeout(3000); // wait for the dialog to settle const resultLocator = spotlight.results; await expect(resultLocator).toHaveCount(1); - await expect(resultLocator.first()).toContainText(bot2Name); + await expect(resultLocator.first()).toContainText(bot2.credentials.displayName); await expect(spotlight.dialog.locator(".mx_SpotlightDialog_startGroupChat")).toContainText( "Start a group chat", @@ -336,18 +341,18 @@ test.describe("Spotlight", () => { await expect(page.getByRole("dialog")).toContainText("Direct Messages"); }); - test("should close spotlight after starting a DM", async ({ page, app }) => { - await startDM(app, page, bot1Name); + test("should close spotlight after starting a DM", async ({ page, app, bot1 }) => { + await startDM(app, page, bot1.credentials.displayName); await expect(page.locator(".mx_SpotlightDialog")).toHaveCount(0); }); - test("should show the same user only once", async ({ page, app }) => { - await startDM(app, page, bot1Name); + test("should show the same user only once", async ({ page, app, bot1 }) => { + await startDM(app, page, bot1.credentials.displayName); await page.goto("/#/home"); const spotlight = await app.openSpotlight(); await page.waitForTimeout(500); // wait for the dialog to settle await spotlight.filter(Filter.People); - await spotlight.search(bot1Name); + await spotlight.search(bot1.credentials.displayName); await page.waitForTimeout(3000); // wait for the dialog to settle await expect(spotlight.dialog.locator(".mx_Spinner")).not.toBeAttached(); const resultLocator = spotlight.results; From c25c8a3b9acf36ea1087f672f7ccbccb99d19e7f Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 11:48:26 +0000 Subject: [PATCH 06/13] Deflake more tests using existing username Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/crypto/backups-mas.spec.ts | 17 +++++++++++------ playwright/e2e/login/login-sso.spec.ts | 4 ++-- playwright/e2e/login/soft_logout_oauth.spec.ts | 4 ++-- playwright/e2e/login/utils.ts | 5 +++-- playwright/e2e/oidc/oidc-native.spec.ts | 10 ++++++++-- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/playwright/e2e/crypto/backups-mas.spec.ts b/playwright/e2e/crypto/backups-mas.spec.ts index 614bde50646..1838f9e234b 100644 --- a/playwright/e2e/crypto/backups-mas.spec.ts +++ b/playwright/e2e/crypto/backups-mas.spec.ts @@ -19,19 +19,19 @@ test.use(masHomeserver); test.describe("Encryption state after registration", () => { test.skip(isDendrite, "does not yet support MAS"); - test("Key backup is enabled by default", async ({ page, mailhogClient, app }) => { + test("Key backup is enabled by default", async ({ page, mailhogClient, app }, testInfo) => { await page.goto("/#/login"); await page.getByRole("button", { name: "Continue" }).click(); - await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!"); + await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!"); await app.settings.openUserSettings("Security & Privacy"); await expect(page.getByText("This session is backing up your keys.")).toBeVisible(); }); - test("user is prompted to set up recovery", async ({ page, mailhogClient, app }) => { + test("user is prompted to set up recovery", async ({ page, mailhogClient, app }, testInfo) => { await page.goto("/#/login"); await page.getByRole("button", { name: "Continue" }).click(); - await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!"); + await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!"); await page.getByRole("button", { name: "Add room" }).click(); await page.getByRole("menuitem", { name: "New room" }).click(); @@ -45,8 +45,13 @@ test.describe("Encryption state after registration", () => { test.describe("Key backup reset from elsewhere", () => { test.skip(isDendrite, "does not yet support MAS"); - test("Key backup is disabled when reset from elsewhere", async ({ page, mailhogClient, request, homeserver }) => { - const testUsername = "alice"; + test("Key backup is disabled when reset from elsewhere", async ({ + page, + mailhogClient, + request, + homeserver, + }, testInfo) => { + const testUsername = `alice_${testInfo.testId}`; const testPassword = "Pa$sW0rD!"; // there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake diff --git a/playwright/e2e/login/login-sso.spec.ts b/playwright/e2e/login/login-sso.spec.ts index fbe190b9358..22428af5028 100644 --- a/playwright/e2e/login/login-sso.spec.ts +++ b/playwright/e2e/login/login-sso.spec.ts @@ -17,13 +17,13 @@ test.use(legacyOAuthHomeserver); test.describe("SSO login", () => { test.skip(isDendrite, "does not yet support SSO"); - test("logs in with SSO and lands on the home screen", async ({ page, homeserver }) => { + test("logs in with SSO and lands on the home screen", async ({ page, homeserver }, testInfo) => { // If this test fails with a screen showing "Timeout connecting to remote server", it is most likely due to // your firewall settings: Synapse is unable to reach the OIDC server. // // If you are using ufw, try something like: // sudo ufw allow in on docker0 // - await doTokenRegistration(page, homeserver); + await doTokenRegistration(page, homeserver, testInfo); }); }); diff --git a/playwright/e2e/login/soft_logout_oauth.spec.ts b/playwright/e2e/login/soft_logout_oauth.spec.ts index 19b1fc0124c..f6814d0cf4a 100644 --- a/playwright/e2e/login/soft_logout_oauth.spec.ts +++ b/playwright/e2e/login/soft_logout_oauth.spec.ts @@ -26,8 +26,8 @@ test.use({ test.use(legacyOAuthHomeserver); test.describe("Soft logout with SSO user", () => { test.use({ - user: async ({ page, homeserver }, use) => { - const user = await doTokenRegistration(page, homeserver); + user: async ({ page, homeserver }, use, testInfo) => { + const user = await doTokenRegistration(page, homeserver, testInfo); // Eventually, we should end up at the home screen. await expect(page).toHaveURL(/\/#\/home$/); diff --git a/playwright/e2e/login/utils.ts b/playwright/e2e/login/utils.ts index cc98d8819a2..e7121159f0f 100644 --- a/playwright/e2e/login/utils.ts +++ b/playwright/e2e/login/utils.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import { Page, expect } from "@playwright/test"; +import { Page, expect, TestInfo } from "@playwright/test"; import { Credentials, HomeserverInstance } from "../../plugins/homeserver"; @@ -15,6 +15,7 @@ import { Credentials, HomeserverInstance } from "../../plugins/homeserver"; export async function doTokenRegistration( page: Page, homeserver: HomeserverInstance, + testInfo: TestInfo, ): Promise { await page.goto("/#/login"); @@ -35,7 +36,7 @@ export async function doTokenRegistration( // Synapse prompts us to pick a user ID await expect(page.getByRole("heading", { name: "Create your account" })).toBeVisible(); - await page.getByRole("textbox", { name: "Username (required)" }).fill("alice"); + await page.getByRole("textbox", { name: "Username (required)" }).fill(`alice_${testInfo.testId}`); // wait for username validation to start, and complete await expect(page.locator("#field-username-output")).toHaveText(""); diff --git a/playwright/e2e/oidc/oidc-native.spec.ts b/playwright/e2e/oidc/oidc-native.spec.ts index 63cf0a5b59f..eb268c5cccc 100644 --- a/playwright/e2e/oidc/oidc-native.spec.ts +++ b/playwright/e2e/oidc/oidc-native.spec.ts @@ -17,7 +17,13 @@ test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.skip(isDendrite, "does not yet support MAS"); test.slow(); // trace recording takes a while here - test("can register the oauth2 client and an account", async ({ context, page, homeserver, mailhogClient, mas }) => { + test("can register the oauth2 client and an account", async ({ + context, + page, + homeserver, + mailhogClient, + mas, + }, testInfo) => { const tokenUri = `${mas.baseUrl}/oauth2/token`; const tokenApiPromise = page.waitForRequest( (request) => request.url() === tokenUri && request.postDataJSON()["grant_type"] === "authorization_code", @@ -25,7 +31,7 @@ test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => { await page.goto("/#/login"); await page.getByRole("button", { name: "Continue" }).click(); - await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!"); + await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!"); // Eventually, we should end up at the home screen. await expect(page).toHaveURL(/\/#\/home$/, { timeout: 10000 }); From db47e6890332f1aadc5d8d86e73b64505b99664b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 11:48:37 +0000 Subject: [PATCH 07/13] Clean mailhog between tests Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/services.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playwright/services.ts b/playwright/services.ts index 1e2501dc2de..a083304fa13 100644 --- a/playwright/services.ts +++ b/playwright/services.ts @@ -131,11 +131,12 @@ export const test = base.extend<{}, Services>({ { scope: "worker" }, ], - context: async ({ logger, context, request, homeserver }, use, testInfo) => { + context: async ({ logger, context, request, homeserver, mailhogClient }, use, testInfo) => { homeserver.setRequest(request); await logger.testStarted(testInfo); await use(context); await logger.testFinished(testInfo); await homeserver.onTestFinished(testInfo); + await mailhogClient.deleteAll(); }, }); From 8f68dbb4f340214fa3fc9c0aac7ad9789108dc01 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 12:51:32 +0000 Subject: [PATCH 08/13] Fix more flakes Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/spotlight/spotlight.spec.ts | 10 +- playwright/services.ts | 2 + .../testcontainers/HomeserverContainer.ts | 2 + playwright/testcontainers/dendrite.ts | 2 +- playwright/testcontainers/mas.ts | 91 ++++++++++++--- playwright/testcontainers/synapse.ts | 105 ++++++------------ playwright/testcontainers/utils.ts | 68 +++++++++++- 7 files changed, 189 insertions(+), 91 deletions(-) diff --git a/playwright/e2e/spotlight/spotlight.spec.ts b/playwright/e2e/spotlight/spotlight.spec.ts index da35ca57b35..d4451b9b5c1 100644 --- a/playwright/e2e/spotlight/spotlight.spec.ts +++ b/playwright/e2e/spotlight/spotlight.spec.ts @@ -366,28 +366,28 @@ test.describe("Spotlight", () => { await spotlight.search("b"); let resultLocator = spotlight.results; - await expect(resultLocator).toHaveCount(2); + await expect(resultLocator.count()).resolves.toBeGreaterThan(2); await expect(resultLocator.first()).toHaveAttribute("aria-selected", "true"); await expect(resultLocator.last()).toHaveAttribute("aria-selected", "false"); await spotlight.searchBox.press("ArrowDown"); resultLocator = spotlight.results; await expect(resultLocator.first()).toHaveAttribute("aria-selected", "false"); - await expect(resultLocator.last()).toHaveAttribute("aria-selected", "true"); + await expect(resultLocator.nth(2)).toHaveAttribute("aria-selected", "true"); await spotlight.searchBox.press("ArrowDown"); resultLocator = spotlight.results; await expect(resultLocator.first()).toHaveAttribute("aria-selected", "false"); - await expect(resultLocator.last()).toHaveAttribute("aria-selected", "false"); + await expect(resultLocator.nth(2)).toHaveAttribute("aria-selected", "false"); await spotlight.searchBox.press("ArrowUp"); resultLocator = spotlight.results; await expect(resultLocator.first()).toHaveAttribute("aria-selected", "false"); - await expect(resultLocator.last()).toHaveAttribute("aria-selected", "true"); + await expect(resultLocator.nth(2)).toHaveAttribute("aria-selected", "true"); await spotlight.searchBox.press("ArrowUp"); resultLocator = spotlight.results; await expect(resultLocator.first()).toHaveAttribute("aria-selected", "true"); - await expect(resultLocator.last()).toHaveAttribute("aria-selected", "false"); + await expect(resultLocator.nth(2)).toHaveAttribute("aria-selected", "false"); }); }); diff --git a/playwright/services.ts b/playwright/services.ts index a083304fa13..4fe05b1282d 100644 --- a/playwright/services.ts +++ b/playwright/services.ts @@ -116,6 +116,8 @@ export const test = base.extend<{}, Services>({ .withConfig(synapseConfigOptions) .start(); + container.setMatrixAuthenticationService(mas); + await use(container); await container.stop(); }, diff --git a/playwright/testcontainers/HomeserverContainer.ts b/playwright/testcontainers/HomeserverContainer.ts index 73c9882418c..e825b6e5544 100644 --- a/playwright/testcontainers/HomeserverContainer.ts +++ b/playwright/testcontainers/HomeserverContainer.ts @@ -9,6 +9,7 @@ import { AbstractStartedContainer, GenericContainer } from "testcontainers"; import { APIRequestContext, TestInfo } from "@playwright/test"; import { HomeserverInstance } from "../plugins/homeserver"; +import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts"; export interface HomeserverContainer extends GenericContainer { withConfigField(key: string, value: any): this; @@ -18,5 +19,6 @@ export interface HomeserverContainer extends GenericContainer { export interface StartedHomeserverContainer extends AbstractStartedContainer, HomeserverInstance { setRequest(request: APIRequestContext): void; + setMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): void; onTestFinished(testInfo: TestInfo): Promise; } diff --git a/playwright/testcontainers/dendrite.ts b/playwright/testcontainers/dendrite.ts index 517b0f7a642..ce786d15c18 100644 --- a/playwright/testcontainers/dendrite.ts +++ b/playwright/testcontainers/dendrite.ts @@ -258,5 +258,5 @@ export class PineconeContainer extends DendriteContainer { } } -// Surprisingly, Dendrite implements the same register user Admin API Synapse, so we can just extend it +// Surprisingly, Dendrite implements the same register user Synapse Admin API, so we can just extend it export class StartedDendriteContainer extends StartedSynapseContainer {} diff --git a/playwright/testcontainers/mas.ts b/playwright/testcontainers/mas.ts index d15f619dbc6..e60cb652982 100644 --- a/playwright/testcontainers/mas.ts +++ b/playwright/testcontainers/mas.ts @@ -11,6 +11,8 @@ import * as YAML from "yaml"; import { getFreePort } from "../plugins/utils/port.ts"; import { deepCopy } from "../plugins/utils/object.ts"; +import { Credentials } from "../plugins/homeserver"; +import { ClientServerApi } from "./utils.ts"; const DEFAULT_CONFIG = { http: { @@ -18,18 +20,11 @@ const DEFAULT_CONFIG = { { name: "web", resources: [ - { - name: "discovery", - }, - { - name: "human", - }, - { - name: "oauth", - }, - { - name: "compat", - }, + { name: "discovery" }, + { name: "human" }, + { name: "oauth" }, + { name: "compat" }, + { name: "adminapi" }, { name: "graphql", playground: true, @@ -172,9 +167,12 @@ const DEFAULT_CONFIG = { export class MatrixAuthenticationServiceContainer extends GenericContainer { private config: typeof DEFAULT_CONFIG; + private readonly args = ["-c", "/config/config.yaml"]; constructor(db: StartedPostgreSqlContainer) { - super("ghcr.io/element-hq/matrix-authentication-service:0.12.0"); + // We rely on `mas-cli manage add-email` which isn't in a release yet + // https://github.com/element-hq/matrix-authentication-service/pull/3235 + super("ghcr.io/element-hq/matrix-authentication-service:sha-0b90c33"); this.config = deepCopy(DEFAULT_CONFIG); this.config.database.username = db.getUsername(); @@ -182,7 +180,7 @@ export class MatrixAuthenticationServiceContainer extends GenericContainer { this.withExposedPorts(8080, 8081) .withWaitStrategy(Wait.forHttp("/health", 8081)) - .withCommand(["server", "--config", "/config/config.yaml"]); + .withCommand(["server", ...this.args]); } public withConfig(config: object): this { @@ -210,15 +208,78 @@ export class MatrixAuthenticationServiceContainer extends GenericContainer { }, ]); - return new StartedMatrixAuthenticationServiceContainer(await super.start(), `http://localhost:${port}`); + return new StartedMatrixAuthenticationServiceContainer( + await super.start(), + `http://localhost:${port}`, + this.args, + ); } } export class StartedMatrixAuthenticationServiceContainer extends AbstractStartedContainer { + private adminTokenPromise?: Promise; + constructor( container: StartedTestContainer, public readonly baseUrl: string, + private readonly args: string[], ) { super(container); } + + public async getAdminToken(csApi: ClientServerApi): Promise { + if (this.adminTokenPromise === undefined) { + this.adminTokenPromise = this.registerUserInternal( + csApi, + "admin", + "totalyinsecureadminpassword", + undefined, + true, + ).then((res) => res.accessToken); + } + return this.adminTokenPromise; + } + + private async registerUserInternal( + csApi: ClientServerApi, + username: string, + password: string, + displayName?: string, + admin = false, + ): Promise { + const args: string[] = []; + if (admin) args.push("-a"); + await this.exec([ + "mas-cli", + "manage", + "register-user", + ...this.args, + ...args, + "-y", + "-p", + password, + "-d", + displayName ?? "", + username, + ]); + + return csApi.loginUser(username, password); + } + + public async registerUser( + csApi: ClientServerApi, + username: string, + password: string, + displayName?: string, + ): Promise { + return this.registerUserInternal(csApi, username, password, displayName, false); + } + + public async setThreepid(username: string, medium: string, address: string): Promise { + if (medium !== "email") { + throw new Error("Only email threepids are supported by MAS"); + } + + await this.exec(["mas-cli", "manage", "add-email", ...this.args, username, address]); + } } diff --git a/playwright/testcontainers/synapse.ts b/playwright/testcontainers/synapse.ts index 27249b75952..05b5702328a 100644 --- a/playwright/testcontainers/synapse.ts +++ b/playwright/testcontainers/synapse.ts @@ -16,6 +16,8 @@ import { randB64Bytes } from "../plugins/utils/rand.ts"; import { Credentials } from "../plugins/homeserver"; import { deepCopy } from "../plugins/utils/object.ts"; import { HomeserverContainer, StartedHomeserverContainer } from "./HomeserverContainer.ts"; +import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts"; +import { Api, ClientServerApi, Verb } from "./utils.ts"; const TAG = "develop@sha256:b69222d98abe9625d46f5d3cb01683d5dc173ae339215297138392cfeec935d9"; @@ -138,8 +140,6 @@ const DEFAULT_CONFIG = { }, }; -type Verb = "GET" | "POST" | "PUT" | "DELETE"; - export type SynapseConfigOptions = Partial; export class SynapseContainer extends GenericContainer implements HomeserverContainer { @@ -231,7 +231,10 @@ export class SynapseContainer extends GenericContainer implements HomeserverCont export class StartedSynapseContainer extends AbstractStartedContainer implements StartedHomeserverContainer { private adminTokenPromise?: Promise; + private _mas?: StartedMatrixAuthenticationServiceContainer; protected _request?: APIRequestContext; + protected csApi: ClientServerApi; + protected adminApi: Api; constructor( container: StartedTestContainer, @@ -239,10 +242,17 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements private readonly registrationSharedSecret: string, ) { super(container); + this.csApi = new ClientServerApi(this.baseUrl); + this.adminApi = new Api(`${this.baseUrl}/_synapse/admin/`); } public setRequest(request: APIRequestContext): void { this._request = request; + this.csApi.setRequest(request); + } + + public setMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): void { + this._mas = mas; } public async onTestFinished(testInfo: TestInfo): Promise { @@ -251,13 +261,14 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements } protected async deletePublicRooms(): Promise { + const token = await this.getAdminToken(); // We hide the rooms from the room directory to save time between tests and for portability between homeservers - const { chunk: rooms } = await this.request<{ + const { chunk: rooms } = await this.csApi.request<{ chunk: { room_id: string }[]; - }>("GET", "v3/publicRooms", {}); + }>("GET", "v3/publicRooms", token, {}); await Promise.all( rooms.map((room) => - this.request("PUT", `v3/directory/list/room/${room.room_id}`, { visibility: "private" }), + this.csApi.request("PUT", `v3/directory/list/room/${room.room_id}`, token, { visibility: "private" }), ), ); } @@ -268,13 +279,18 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements displayName?: string, admin = false, ): Promise { - const url = `${this.baseUrl}/_synapse/admin/v1/register`; - const { nonce } = await this._request.get(url).then((r) => r.json()); + const path = `v1/register`; + const { nonce } = await this.adminApi.request<{ nonce: string }>("GET", path, undefined, {}); const mac = crypto .createHmac("sha1", this.registrationSharedSecret) .update(`${nonce}\0${username}\0${password}\0${admin ? "" : "not"}admin`) .digest("hex"); - const res = await this._request.post(url, { + const data = await this.adminApi.request<{ + home_server: string; + access_token: string; + user_id: string; + device_id: string; + }>("POST", path, undefined, { data: { nonce, username, @@ -285,11 +301,6 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements }, }); - if (!res.ok()) { - throw await res.json(); - } - - const data = await res.json(); return { homeServer: data.home_server, accessToken: data.access_token, @@ -303,6 +314,10 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements protected async getAdminToken(): Promise { if (this.adminTokenPromise === undefined) { + if (this._mas) { + return (this.adminTokenPromise = this._mas.getAdminToken(this.csApi)); + } + this.adminTokenPromise = this.registerUserInternal( "admin", "totalyinsecureadminpassword", @@ -317,72 +332,24 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements private async adminRequest(verb: Verb, path: string, data?: object): Promise; private async adminRequest(verb: Verb, path: string, data?: object): Promise { const adminToken = await this.getAdminToken(); - const url = `${this.baseUrl}/_synapse/admin/${path}`; - const res = await this._request.fetch(url, { - data, - method: verb, - headers: { - Authorization: `Bearer ${adminToken}`, - }, - }); - - if (!res.ok()) { - throw await res.json(); - } - - return res.json(); - } - - public async request(verb: "GET", path: string, data?: never): Promise; - public async request(verb: Verb, path: string, data?: object): Promise; - public async request(verb: Verb, path: string, data?: object): Promise { - const token = await this.getAdminToken(); - const url = `${this.baseUrl}/_matrix/client/${path}`; - const res = await this._request.fetch(url, { - data, - method: verb, - headers: { - Authorization: `Bearer ${token}`, - }, - }); - - if (!res.ok()) { - throw await res.json(); - } - - return res.json(); + return this.adminApi.request(verb, path, adminToken, data); } public registerUser(username: string, password: string, displayName?: string): Promise { + if (this._mas) { + return this._mas.registerUser(this.csApi, username, password, displayName); + } return this.registerUserInternal(username, password, displayName, false); } public async loginUser(userId: string, password: string): Promise { - const json = await this.request<{ - access_token: string; - user_id: string; - device_id: string; - home_server: string; - }>("POST", "v3/login", { - type: "m.login.password", - identifier: { - type: "m.id.user", - user: userId, - }, - password: password, - }); - - return { - password, - accessToken: json.access_token, - userId: json.user_id, - deviceId: json.device_id, - homeServer: json.home_server, - username: userId.slice(1).split(":")[0], - }; + return this.csApi.loginUser(userId, password); } public async setThreepid(userId: string, medium: string, address: string): Promise { + if (this._mas) { + return this._mas.setThreepid(userId, medium, address); + } await this.adminRequest("PUT", `v2/users/${userId}`, { threepids: [ { diff --git a/playwright/testcontainers/utils.ts b/playwright/testcontainers/utils.ts index 1339e9c2fc1..487562f8af9 100644 --- a/playwright/testcontainers/utils.ts +++ b/playwright/testcontainers/utils.ts @@ -5,10 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import { TestInfo } from "@playwright/test"; +import { APIRequestContext, TestInfo } from "@playwright/test"; import { Readable } from "stream"; import stripAnsi from "strip-ansi"; +import { Credentials } from "../plugins/homeserver"; + export class ContainerLogger { private logs: Record = {}; @@ -41,3 +43,67 @@ export class ContainerLogger { } } } + +export type Verb = "GET" | "POST" | "PUT" | "DELETE"; + +export class Api { + private _request?: APIRequestContext; + + constructor(private readonly baseUrl: string) {} + + public setRequest(request: APIRequestContext): void { + this._request = request; + } + + public async request(verb: "GET", path: string, token?: string, data?: never): Promise; + public async request(verb: Verb, path: string, token?: string, data?: object): Promise; + public async request(verb: Verb, path: string, token?: string, data?: object): Promise { + const url = `${this.baseUrl}/_matrix/client/${path}`; + const res = await this._request.fetch(url, { + data, + method: verb, + headers: token + ? { + Authorization: `Bearer ${token}`, + } + : undefined, + }); + + if (!res.ok()) { + throw await res.json(); + } + + return res.json(); + } +} + +export class ClientServerApi extends Api { + constructor(baseUrl: string) { + super(`${baseUrl}/_matrix/client/`); + } + + public async loginUser(userId: string, password: string): Promise { + const json = await this.request<{ + access_token: string; + user_id: string; + device_id: string; + home_server: string; + }>("POST", "v3/login", undefined, { + type: "m.login.password", + identifier: { + type: "m.id.user", + user: userId, + }, + password: password, + }); + + return { + password, + accessToken: json.access_token, + userId: json.user_id, + deviceId: json.device_id, + homeServer: json.home_server, + username: userId.slice(1).split(":")[0], + }; + } +} From 548d45af886600bda199db20936ac0b4cacfe1c4 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 13:17:58 +0000 Subject: [PATCH 09/13] Fix missing _request Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/testcontainers/synapse.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/playwright/testcontainers/synapse.ts b/playwright/testcontainers/synapse.ts index 05b5702328a..ff1b0002327 100644 --- a/playwright/testcontainers/synapse.ts +++ b/playwright/testcontainers/synapse.ts @@ -249,6 +249,7 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements public setRequest(request: APIRequestContext): void { this._request = request; this.csApi.setRequest(request); + this.adminApi.setRequest(request); } public setMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): void { From 8b3ffb4b3fc07c464bc366348c8d9a2ac1253bb9 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 13:59:18 +0000 Subject: [PATCH 10/13] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/testcontainers/synapse.ts | 24 +++++++++++------------- playwright/testcontainers/utils.ts | 6 +++--- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/playwright/testcontainers/synapse.ts b/playwright/testcontainers/synapse.ts index ff1b0002327..8c5fafdbd5d 100644 --- a/playwright/testcontainers/synapse.ts +++ b/playwright/testcontainers/synapse.ts @@ -243,7 +243,7 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements ) { super(container); this.csApi = new ClientServerApi(this.baseUrl); - this.adminApi = new Api(`${this.baseUrl}/_synapse/admin/`); + this.adminApi = new Api(`${this.baseUrl}/_synapse/admin`); } public setRequest(request: APIRequestContext): void { @@ -266,10 +266,10 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements // We hide the rooms from the room directory to save time between tests and for portability between homeservers const { chunk: rooms } = await this.csApi.request<{ chunk: { room_id: string }[]; - }>("GET", "v3/publicRooms", token, {}); + }>("GET", "/v3/publicRooms", token, {}); await Promise.all( rooms.map((room) => - this.csApi.request("PUT", `v3/directory/list/room/${room.room_id}`, token, { visibility: "private" }), + this.csApi.request("PUT", `/v3/directory/list/room/${room.room_id}`, token, { visibility: "private" }), ), ); } @@ -280,7 +280,7 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements displayName?: string, admin = false, ): Promise { - const path = `v1/register`; + const path = "/v1/register"; const { nonce } = await this.adminApi.request<{ nonce: string }>("GET", path, undefined, {}); const mac = crypto .createHmac("sha1", this.registrationSharedSecret) @@ -292,14 +292,12 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements user_id: string; device_id: string; }>("POST", path, undefined, { - data: { - nonce, - username, - password, - mac, - admin, - displayname: displayName, - }, + nonce, + username, + password, + mac, + admin, + displayname: displayName, }); return { @@ -351,7 +349,7 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements if (this._mas) { return this._mas.setThreepid(userId, medium, address); } - await this.adminRequest("PUT", `v2/users/${userId}`, { + await this.adminRequest("PUT", `/v2/users/${userId}`, { threepids: [ { medium, diff --git a/playwright/testcontainers/utils.ts b/playwright/testcontainers/utils.ts index 487562f8af9..c48c655c2e5 100644 --- a/playwright/testcontainers/utils.ts +++ b/playwright/testcontainers/utils.ts @@ -58,7 +58,7 @@ export class Api { public async request(verb: "GET", path: string, token?: string, data?: never): Promise; public async request(verb: Verb, path: string, token?: string, data?: object): Promise; public async request(verb: Verb, path: string, token?: string, data?: object): Promise { - const url = `${this.baseUrl}/_matrix/client/${path}`; + const url = `${this.baseUrl}${path}`; const res = await this._request.fetch(url, { data, method: verb, @@ -79,7 +79,7 @@ export class Api { export class ClientServerApi extends Api { constructor(baseUrl: string) { - super(`${baseUrl}/_matrix/client/`); + super(`${baseUrl}/_matrix/client`); } public async loginUser(userId: string, password: string): Promise { @@ -88,7 +88,7 @@ export class ClientServerApi extends Api { user_id: string; device_id: string; home_server: string; - }>("POST", "v3/login", undefined, { + }>("POST", "/v3/login", undefined, { type: "m.login.password", identifier: { type: "m.id.user", From 71f06cd2588700cfbb68b7ba84a37aaf24df5726 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 17:51:40 +0000 Subject: [PATCH 11/13] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/csAPI.ts | 29 +++---- playwright/e2e/login/login-consent.spec.ts | 6 ++ playwright/e2e/oidc/index.ts | 2 +- playwright/e2e/oidc/oidc-native.spec.ts | 9 +- .../e2e/sliding-sync/sliding-sync.spec.ts | 1 - playwright/e2e/spotlight/spotlight.spec.ts | 10 +-- playwright/element-web-test.ts | 2 +- playwright/plugins/homeserver/index.ts | 3 + playwright/services.ts | 3 +- .../testcontainers/HomeserverContainer.ts | 2 +- playwright/testcontainers/dendrite.ts | 5 ++ playwright/testcontainers/mas.ts | 80 +++++++++++++---- playwright/testcontainers/synapse.ts | 86 +++++++++++++------ playwright/testcontainers/utils.ts | 4 +- 14 files changed, 167 insertions(+), 75 deletions(-) diff --git a/playwright/e2e/csAPI.ts b/playwright/e2e/csAPI.ts index d55816fb6a2..78362506ef2 100644 --- a/playwright/e2e/csAPI.ts +++ b/playwright/e2e/csAPI.ts @@ -9,24 +9,24 @@ import { APIRequestContext } from "playwright-core"; import { KeyBackupInfo } from "matrix-js-sdk/src/crypto-api"; import { HomeserverInstance } from "../plugins/homeserver"; +import { ClientServerApi } from "../testcontainers/utils.ts"; /** * A small subset of the Client-Server API used to manipulate the state of the * account on the homeserver independently of the client under test. */ -export class TestClientServerAPI { +export class TestClientServerAPI extends ClientServerApi { public constructor( - private request: APIRequestContext, - private homeserver: HomeserverInstance, + request: APIRequestContext, + homeserver: HomeserverInstance, private accessToken: string, - ) {} + ) { + super(`${homeserver.baseUrl}/_matrix/client`); + this.setRequest(request); + } public async getCurrentBackupInfo(): Promise { - const res = await this.request.get(`${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version`, { - headers: { Authorization: `Bearer ${this.accessToken}` }, - }); - - return await res.json(); + return this.request("GET", `/v3/room_keys/version`, this.accessToken); } /** @@ -34,15 +34,6 @@ export class TestClientServerAPI { * @param version The version to delete */ public async deleteBackupVersion(version: string): Promise { - const res = await this.request.delete( - `${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version/${version}`, - { - headers: { Authorization: `Bearer ${this.accessToken}` }, - }, - ); - - if (!res.ok) { - throw new Error(`Failed to delete backup version: ${res.status}`); - } + await this.request("DELETE", `/v3/room_keys/version/${version}`, this.accessToken); } } diff --git a/playwright/e2e/login/login-consent.spec.ts b/playwright/e2e/login/login-consent.spec.ts index ab70e1d1869..d7d5861a02e 100644 --- a/playwright/e2e/login/login-consent.spec.ts +++ b/playwright/e2e/login/login-consent.spec.ts @@ -77,6 +77,9 @@ async function login(page: Page, homeserver: HomeserverInstance, credentials: Cr await page.getByRole("button", { name: "Sign in" }).click(); } +// This test suite uses the same userId for all tests in the suite +// due to DEVICE_SIGNING_KEYS_BODY being specific to that userId, +// so we restart the Synapse container to make it forget everything. test.use(consentHomeserver); test.use({ config: { @@ -97,6 +100,9 @@ test.use({ ...credentials, displayName, }); + + // Restart the homeserver to wipe its in-memory db so we can reuse the same user ID without cross-signing prompts + await homeserver.restart(); }, }); diff --git a/playwright/e2e/oidc/index.ts b/playwright/e2e/oidc/index.ts index 7e9b03ee6ac..bfd49b496a0 100644 --- a/playwright/e2e/oidc/index.ts +++ b/playwright/e2e/oidc/index.ts @@ -33,7 +33,7 @@ export async function registerAccountMas( expect(messages.items).toHaveLength(1); }).toPass(); expect(messages.items[0].to).toEqual(`${username} <${email}>`); - const [code] = messages.items[0].text.match(/(\d{6})/); + const [, code] = messages.items[0].text.match(/Your verification code to confirm this email address is: (\d{6})/); await page.getByRole("textbox", { name: "6-digit code" }).fill(code); await page.getByRole("button", { name: "Continue" }).click(); diff --git a/playwright/e2e/oidc/oidc-native.spec.ts b/playwright/e2e/oidc/oidc-native.spec.ts index eb268c5cccc..a50730ce747 100644 --- a/playwright/e2e/oidc/oidc-native.spec.ts +++ b/playwright/e2e/oidc/oidc-native.spec.ts @@ -24,6 +24,8 @@ test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => { mailhogClient, mas, }, testInfo) => { + await page.clock.install(); + const tokenUri = `${mas.baseUrl}/oauth2/token`; const tokenApiPromise = page.waitForRequest( (request) => request.url() === tokenUri && request.postDataJSON()["grant_type"] === "authorization_code", @@ -31,11 +33,14 @@ test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => { await page.goto("/#/login"); await page.getByRole("button", { name: "Continue" }).click(); - await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!"); + + const userId = `alice_${testInfo.testId}`; + await registerAccountMas(page, mailhogClient, userId, "alice@email.com", "Pa$sW0rD!"); // Eventually, we should end up at the home screen. await expect(page).toHaveURL(/\/#\/home$/, { timeout: 10000 }); - await expect(page.getByRole("heading", { name: "Welcome alice", exact: true })).toBeVisible(); + await expect(page.getByRole("heading", { name: `Welcome ${userId}`, exact: true })).toBeVisible(); + await page.clock.runFor(20000); // run the timer so we see the token request const tokenApiRequest = await tokenApiPromise; expect(tokenApiRequest.postDataJSON()["grant_type"]).toBe("authorization_code"); diff --git a/playwright/e2e/sliding-sync/sliding-sync.spec.ts b/playwright/e2e/sliding-sync/sliding-sync.spec.ts index 29a612ccd32..1ab7909a478 100644 --- a/playwright/e2e/sliding-sync/sliding-sync.spec.ts +++ b/playwright/e2e/sliding-sync/sliding-sync.spec.ts @@ -108,7 +108,6 @@ test.describe("Sliding Sync", () => { await page.getByRole("menuitemradio", { name: "A-Z" }).dispatchEvent("click"); await expect(page.locator(".mx_StyledRadioButton_checked").getByText("A-Z")).toBeVisible(); - await page.pause(); await checkOrder(["Apple", "Orange", "Pineapple", "Test Room"], page); }); diff --git a/playwright/e2e/spotlight/spotlight.spec.ts b/playwright/e2e/spotlight/spotlight.spec.ts index d4451b9b5c1..da35ca57b35 100644 --- a/playwright/e2e/spotlight/spotlight.spec.ts +++ b/playwright/e2e/spotlight/spotlight.spec.ts @@ -366,28 +366,28 @@ test.describe("Spotlight", () => { await spotlight.search("b"); let resultLocator = spotlight.results; - await expect(resultLocator.count()).resolves.toBeGreaterThan(2); + await expect(resultLocator).toHaveCount(2); await expect(resultLocator.first()).toHaveAttribute("aria-selected", "true"); await expect(resultLocator.last()).toHaveAttribute("aria-selected", "false"); await spotlight.searchBox.press("ArrowDown"); resultLocator = spotlight.results; await expect(resultLocator.first()).toHaveAttribute("aria-selected", "false"); - await expect(resultLocator.nth(2)).toHaveAttribute("aria-selected", "true"); + await expect(resultLocator.last()).toHaveAttribute("aria-selected", "true"); await spotlight.searchBox.press("ArrowDown"); resultLocator = spotlight.results; await expect(resultLocator.first()).toHaveAttribute("aria-selected", "false"); - await expect(resultLocator.nth(2)).toHaveAttribute("aria-selected", "false"); + await expect(resultLocator.last()).toHaveAttribute("aria-selected", "false"); await spotlight.searchBox.press("ArrowUp"); resultLocator = spotlight.results; await expect(resultLocator.first()).toHaveAttribute("aria-selected", "false"); - await expect(resultLocator.nth(2)).toHaveAttribute("aria-selected", "true"); + await expect(resultLocator.last()).toHaveAttribute("aria-selected", "true"); await spotlight.searchBox.press("ArrowUp"); resultLocator = spotlight.results; await expect(resultLocator.first()).toHaveAttribute("aria-selected", "true"); - await expect(resultLocator.nth(2)).toHaveAttribute("aria-selected", "false"); + await expect(resultLocator.last()).toHaveAttribute("aria-selected", "false"); }); }); diff --git a/playwright/element-web-test.ts b/playwright/element-web-test.ts index 0c6392fdc21..12b86c1f69f 100644 --- a/playwright/element-web-test.ts +++ b/playwright/element-web-test.ts @@ -41,7 +41,7 @@ const CONFIG_JSON: Partial = { }, }; -interface CredentialsWithDisplayName extends Credentials { +export interface CredentialsWithDisplayName extends Credentials { displayName: string; } diff --git a/playwright/plugins/homeserver/index.ts b/playwright/plugins/homeserver/index.ts index 50dea472f78..9e54e0aa916 100644 --- a/playwright/plugins/homeserver/index.ts +++ b/playwright/plugins/homeserver/index.ts @@ -6,8 +6,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ +import { ClientServerApi } from "../../testcontainers/utils.ts"; + export interface HomeserverInstance { readonly baseUrl: string; + readonly csApi: ClientServerApi; /** * Register a user on the given Homeserver using the shared registration secret. diff --git a/playwright/services.ts b/playwright/services.ts index 4fe05b1282d..0d9ece05291 100644 --- a/playwright/services.ts +++ b/playwright/services.ts @@ -114,10 +114,9 @@ export const test = base.extend<{}, Services>({ .withNetworkAliases("homeserver") .withLogConsumer(logger.getConsumer("synapse")) .withConfig(synapseConfigOptions) + .withMatrixAuthenticationService(mas) .start(); - container.setMatrixAuthenticationService(mas); - await use(container); await container.stop(); }, diff --git a/playwright/testcontainers/HomeserverContainer.ts b/playwright/testcontainers/HomeserverContainer.ts index e825b6e5544..259ecb7fe0a 100644 --- a/playwright/testcontainers/HomeserverContainer.ts +++ b/playwright/testcontainers/HomeserverContainer.ts @@ -14,11 +14,11 @@ import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts"; export interface HomeserverContainer extends GenericContainer { withConfigField(key: string, value: any): this; withConfig(config: Partial): this; + withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this; start(): Promise; } export interface StartedHomeserverContainer extends AbstractStartedContainer, HomeserverInstance { setRequest(request: APIRequestContext): void; - setMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): void; onTestFinished(testInfo: TestInfo): Promise; } diff --git a/playwright/testcontainers/dendrite.ts b/playwright/testcontainers/dendrite.ts index ce786d15c18..c358ff15852 100644 --- a/playwright/testcontainers/dendrite.ts +++ b/playwright/testcontainers/dendrite.ts @@ -13,6 +13,7 @@ import { randB64Bytes } from "../plugins/utils/rand.ts"; import { StartedSynapseContainer } from "./synapse.ts"; import { deepCopy } from "../plugins/utils/object.ts"; import { HomeserverContainer } from "./HomeserverContainer.ts"; +import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts"; const DEFAULT_CONFIG = { version: 2, @@ -235,6 +236,10 @@ export class DendriteContainer extends GenericContainer implements HomeserverCon return this; } + public withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this { + throw new Error("Dendrite does not support MAS."); + } + public override async start(): Promise { this.withCopyContentToContainer([ { diff --git a/playwright/testcontainers/mas.ts b/playwright/testcontainers/mas.ts index e60cb652982..697ef374a18 100644 --- a/playwright/testcontainers/mas.ts +++ b/playwright/testcontainers/mas.ts @@ -5,14 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers"; +import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait, ExecResult } from "testcontainers"; import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import * as YAML from "yaml"; import { getFreePort } from "../plugins/utils/port.ts"; import { deepCopy } from "../plugins/utils/object.ts"; import { Credentials } from "../plugins/homeserver"; -import { ClientServerApi } from "./utils.ts"; const DEFAULT_CONFIG = { http: { @@ -227,10 +226,9 @@ export class StartedMatrixAuthenticationServiceContainer extends AbstractStarted super(container); } - public async getAdminToken(csApi: ClientServerApi): Promise { + public async getAdminToken(): Promise { if (this.adminTokenPromise === undefined) { this.adminTokenPromise = this.registerUserInternal( - csApi, "admin", "totalyinsecureadminpassword", undefined, @@ -240,20 +238,24 @@ export class StartedMatrixAuthenticationServiceContainer extends AbstractStarted return this.adminTokenPromise; } - private async registerUserInternal( - csApi: ClientServerApi, + private async manage(cmd: string, ...args: string[]): Promise { + const result = await this.exec(["mas-cli", "manage", cmd, ...this.args, ...args]); + if (result.exitCode !== 0) { + throw new Error(`Failed mas-cli manage ${cmd}: ${result.output}`); + } + return result; + } + + private async manageRegisterUser( username: string, password: string, displayName?: string, admin = false, - ): Promise { + ): Promise { const args: string[] = []; if (admin) args.push("-a"); - await this.exec([ - "mas-cli", - "manage", + const result = await this.manage( "register-user", - ...this.args, ...args, "-y", "-p", @@ -261,18 +263,62 @@ export class StartedMatrixAuthenticationServiceContainer extends AbstractStarted "-d", displayName ?? "", username, - ]); + ); - return csApi.loginUser(username, password); + const registerLines = result.output.trim().split("\n"); + const userId = registerLines + .find((line) => line.includes("Matrix ID: ")) + ?.split(": ") + .pop(); + + if (!userId) { + throw new Error(`Failed to register user: ${result.output}`); + } + + return userId; } - public async registerUser( - csApi: ClientServerApi, + private async manageIssueCompatibilityToken( + username: string, + admin = false, + ): Promise<{ accessToken: string; deviceId: string }> { + const args: string[] = []; + if (admin) args.push("--yes-i-want-to-grant-synapse-admin-privileges"); + const result = await this.manage("issue-compatibility-token", ...args, username); + + const parts = result.output.trim().split(/\s+/); + const accessToken = parts.find((part) => part.startsWith("mct_")); + const deviceId = parts.find((part) => part.startsWith("compat_session.device="))?.split("=")[1]; + + if (!accessToken || !deviceId) { + throw new Error(`Failed to issue compatibility token: ${result.output}`); + } + + return { accessToken, deviceId }; + } + + private async registerUserInternal( username: string, password: string, displayName?: string, + admin = false, ): Promise { - return this.registerUserInternal(csApi, username, password, displayName, false); + const userId = await this.manageRegisterUser(username, password, displayName, admin); + const { deviceId, accessToken } = await this.manageIssueCompatibilityToken(username, admin); + + return { + userId, + accessToken, + deviceId, + homeServer: userId.slice(1).split(":").slice(1).join(":"), + displayName, + username, + password, + }; + } + + public async registerUser(username: string, password: string, displayName?: string): Promise { + return this.registerUserInternal(username, password, displayName, false); } public async setThreepid(username: string, medium: string, address: string): Promise { @@ -280,6 +326,6 @@ export class StartedMatrixAuthenticationServiceContainer extends AbstractStarted throw new Error("Only email threepids are supported by MAS"); } - await this.exec(["mas-cli", "manage", "add-email", ...this.args, username, address]); + await this.manage("add-email", username, address); } } diff --git a/playwright/testcontainers/synapse.ts b/playwright/testcontainers/synapse.ts index 8c5fafdbd5d..74625ca3b98 100644 --- a/playwright/testcontainers/synapse.ts +++ b/playwright/testcontainers/synapse.ts @@ -5,7 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers"; +import { + AbstractStartedContainer, + GenericContainer, + ImageName, + RestartOptions, + StartedTestContainer, + Wait, +} from "testcontainers"; import { APIRequestContext, TestInfo } from "@playwright/test"; import crypto from "node:crypto"; import * as YAML from "yaml"; @@ -144,6 +151,7 @@ export type SynapseConfigOptions = Partial; export class SynapseContainer extends GenericContainer implements HomeserverContainer { private config: typeof DEFAULT_CONFIG; + private mas?: StartedMatrixAuthenticationServiceContainer; constructor() { super(`ghcr.io/element-hq/synapse:${TAG}`); @@ -203,6 +211,11 @@ export class SynapseContainer extends GenericContainer implements HomeserverCont return this; } + public withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this { + this.mas = mas; + return this; + } + public override async start(): Promise { // Synapse config public_baseurl needs to know what URL it'll be accessed from, so we have to map the port manually const port = await getFreePort(); @@ -221,20 +234,26 @@ export class SynapseContainer extends GenericContainer implements HomeserverCont }, ]); - return new StartedSynapseContainer( - await super.start(), - `http://localhost:${port}`, - this.config.registration_shared_secret, - ); + const container = await super.start(); + const baseUrl = `http://localhost:${port}`; + if (this.mas) { + return new StartedSynapseWithMasContainer( + container, + baseUrl, + this.config.registration_shared_secret, + this.mas, + ); + } + + return new StartedSynapseContainer(container, baseUrl, this.config.registration_shared_secret); } } export class StartedSynapseContainer extends AbstractStartedContainer implements StartedHomeserverContainer { - private adminTokenPromise?: Promise; - private _mas?: StartedMatrixAuthenticationServiceContainer; + protected adminTokenPromise?: Promise; protected _request?: APIRequestContext; - protected csApi: ClientServerApi; - protected adminApi: Api; + protected readonly adminApi: Api; + public readonly csApi: ClientServerApi; constructor( container: StartedTestContainer, @@ -242,8 +261,13 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements private readonly registrationSharedSecret: string, ) { super(container); - this.csApi = new ClientServerApi(this.baseUrl); this.adminApi = new Api(`${this.baseUrl}/_synapse/admin`); + this.csApi = new ClientServerApi(this.baseUrl); + } + + public restart(options?: Partial): Promise { + this.adminTokenPromise = undefined; + return super.restart(options); } public setRequest(request: APIRequestContext): void { @@ -252,10 +276,6 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements this.adminApi.setRequest(request); } - public setMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): void { - this._mas = mas; - } - public async onTestFinished(testInfo: TestInfo): Promise { // Clean up the server to prevent rooms leaking between tests await this.deletePublicRooms(); @@ -313,10 +333,6 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements protected async getAdminToken(): Promise { if (this.adminTokenPromise === undefined) { - if (this._mas) { - return (this.adminTokenPromise = this._mas.getAdminToken(this.csApi)); - } - this.adminTokenPromise = this.registerUserInternal( "admin", "totalyinsecureadminpassword", @@ -335,9 +351,6 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements } public registerUser(username: string, password: string, displayName?: string): Promise { - if (this._mas) { - return this._mas.registerUser(this.csApi, username, password, displayName); - } return this.registerUserInternal(username, password, displayName, false); } @@ -346,9 +359,6 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements } public async setThreepid(userId: string, medium: string, address: string): Promise { - if (this._mas) { - return this._mas.setThreepid(userId, medium, address); - } await this.adminRequest("PUT", `/v2/users/${userId}`, { threepids: [ { @@ -359,3 +369,29 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements }); } } + +export class StartedSynapseWithMasContainer extends StartedSynapseContainer { + constructor( + container: StartedTestContainer, + baseUrl: string, + registrationSharedSecret: string, + private readonly mas: StartedMatrixAuthenticationServiceContainer, + ) { + super(container, baseUrl, registrationSharedSecret); + } + + protected async getAdminToken(): Promise { + if (this.adminTokenPromise === undefined) { + this.adminTokenPromise = this.mas.getAdminToken(); + } + return this.adminTokenPromise; + } + + public registerUser(username: string, password: string, displayName?: string): Promise { + return this.mas.registerUser(username, password, displayName); + } + + public async setThreepid(userId: string, medium: string, address: string): Promise { + return this.mas.setThreepid(userId, medium, address); + } +} diff --git a/playwright/testcontainers/utils.ts b/playwright/testcontainers/utils.ts index c48c655c2e5..f4fe7f6d318 100644 --- a/playwright/testcontainers/utils.ts +++ b/playwright/testcontainers/utils.ts @@ -70,7 +70,9 @@ export class Api { }); if (!res.ok()) { - throw await res.json(); + throw new Error( + `Request to ${url} failed with status ${res.status()}: ${JSON.stringify(await res.json())}`, + ); } return res.json(); From 1a98f8ceadbf02c48083fb599446d444f6b1a233 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 18:07:43 +0000 Subject: [PATCH 12/13] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/crypto/backups-mas.spec.ts | 3 +-- playwright/e2e/csAPI.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/playwright/e2e/crypto/backups-mas.spec.ts b/playwright/e2e/crypto/backups-mas.spec.ts index 1838f9e234b..b51f7372558 100644 --- a/playwright/e2e/crypto/backups-mas.spec.ts +++ b/playwright/e2e/crypto/backups-mas.spec.ts @@ -67,8 +67,7 @@ test.describe("Key backup reset from elsewhere", () => { await page.getByRole("textbox", { name: "Name" }).fill("test room"); await page.getByRole("button", { name: "Create room" }).click(); - // @ts-ignore - this runs in the browser scope where mxMatrixClientPeg is a thing. Here, it is not. - const accessToken = await page.evaluate(() => mxMatrixClientPeg.get().getAccessToken()); + const accessToken = await page.evaluate(() => window.mxMatrixClientPeg.get().getAccessToken()); const csAPI = new TestClientServerAPI(request, homeserver, accessToken); diff --git a/playwright/e2e/csAPI.ts b/playwright/e2e/csAPI.ts index 78362506ef2..f171ded5e3b 100644 --- a/playwright/e2e/csAPI.ts +++ b/playwright/e2e/csAPI.ts @@ -21,7 +21,7 @@ export class TestClientServerAPI extends ClientServerApi { homeserver: HomeserverInstance, private accessToken: string, ) { - super(`${homeserver.baseUrl}/_matrix/client`); + super(homeserver.baseUrl); this.setRequest(request); } From 0f6f0471596257bb97cf8f70f884580f2c620aa6 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 10 Jan 2025 18:09:29 +0000 Subject: [PATCH 13/13] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/share-dialog/share-dialog.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playwright/e2e/share-dialog/share-dialog.spec.ts b/playwright/e2e/share-dialog/share-dialog.spec.ts index d5424a681d6..58574a46ffe 100644 --- a/playwright/e2e/share-dialog/share-dialog.spec.ts +++ b/playwright/e2e/share-dialog/share-dialog.spec.ts @@ -23,7 +23,7 @@ test.describe("Share dialog", () => { const dialog = page.getByRole("dialog", { name: "Share room" }); await expect(dialog.getByText(`https://matrix.to/#/${room.roomId}`)).toBeVisible(); - expect(dialog).toMatchScreenshot("share-dialog-room.png", { + await expect(dialog).toMatchScreenshot("share-dialog-room.png", { // QRCode and url changes at every run mask: [page.locator(".mx_QRCode"), page.locator(".mx_ShareDialog_top > span")], }); @@ -40,7 +40,7 @@ test.describe("Share dialog", () => { const dialog = page.getByRole("dialog", { name: "Share User" }); await expect(dialog.getByText(`https://matrix.to/#/${user.userId}`)).toBeVisible(); - expect(dialog).toMatchScreenshot("share-dialog-user.png", { + await expect(dialog).toMatchScreenshot("share-dialog-user.png", { // QRCode changes at every run mask: [page.locator(".mx_QRCode")], }); @@ -57,7 +57,7 @@ test.describe("Share dialog", () => { const dialog = page.getByRole("dialog", { name: "Share Room Message" }); await expect(dialog.getByRole("checkbox", { name: "Link to selected message" })).toBeChecked(); - expect(dialog).toMatchScreenshot("share-dialog-event.png", { + await expect(dialog).toMatchScreenshot("share-dialog-event.png", { // QRCode and url changes at every run mask: [page.locator(".mx_QRCode"), page.locator(".mx_ShareDialog_top > span")], });