From cc871a66c03a22bd813f1d17c98d95152fa83a2d Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 8 Jan 2025 09:09:42 +0000 Subject: [PATCH 1/3] Fix playwright-image-updates.yaml workflow Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/CODEOWNERS | 2 +- .github/workflows/playwright-image-updates.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index af52a6b77d1..695a94254e9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -15,5 +15,5 @@ /src/i18n/strings /src/i18n/strings/en_EN.json @element-hq/element-web-reviewers # Ignore the synapse plugin as this is updated by GHA for docker image updating -/playwright/plugins/homeserver/synapse/index.ts +/playwright/testcontainers/synapse.ts diff --git a/.github/workflows/playwright-image-updates.yaml b/.github/workflows/playwright-image-updates.yaml index ef2dfa11327..e5e2f739c09 100644 --- a/.github/workflows/playwright-image-updates.yaml +++ b/.github/workflows/playwright-image-updates.yaml @@ -17,7 +17,7 @@ jobs: docker pull "$IMAGE" INSPECT=$(docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE") DIGEST=${INSPECT#*@} - sed -i "s/const DOCKER_TAG.*/const DOCKER_TAG = \"develop@$DIGEST\";/" playwright/plugins/homeserver/synapse/index.ts + sed -i "s/const TAG.*/const TAG = \"develop@$DIGEST\";/" playwright/testcontainers/synapse.ts env: IMAGE: ghcr.io/element-hq/synapse:develop From 30c82d8714eda9639b57d710a4bb1e67304158ee Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 8 Jan 2025 09:12:33 +0000 Subject: [PATCH 2/3] Add `X-Run-All-Tests` label for running all tests Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/labels.yml | 3 +++ .github/workflows/end-to-end-tests.yaml | 12 ++++++------ docs/playwright.md | 2 ++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 80c5408c1e3..c8a34c47716 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -210,6 +210,9 @@ - name: "X-Upcoming-Release-Blocker" description: "This does not affect the current release cycle but will affect the next one" color: "e99695" +- name: "X-Run-All-Tests" + description: "When applied to PRs, it'll run the full gamut of end-to-end tests on the PR" + color: "ff7979" - name: "Z-Actions" color: "ededed" - name: "Z-Cache-Confusion" diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 6afabdb1fe6..4232993fa0b 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -114,13 +114,13 @@ jobs: - Chrome - Firefox - WebKit - isCron: - - ${{ github.event_name == 'schedule' }} - # Skip the Firefox & Safari runs unless this was a cron trigger + runAllTests: + - ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'X-Run-All-Tests') }} + # Skip the Firefox & Safari runs unless this was a cron trigger or PR has X-Run-All-Tests label exclude: - - isCron: false + - runAllTests: false project: Firefox - - isCron: false + - runAllTests: false project: WebKit steps: - uses: actions/checkout@v4 @@ -170,7 +170,7 @@ jobs: yarn playwright test \ --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \ --project="${{ matrix.project }}" \ - ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }} + ${{ (github.event_name == 'pull_request' && matrix.runAllTests == false ) && '--grep-invert @mergequeue' || '' }} - name: Upload blob report to GitHub Actions Artifacts if: always() diff --git a/docs/playwright.md b/docs/playwright.md index fe44a06ff17..cd19c41b6cc 100644 --- a/docs/playwright.md +++ b/docs/playwright.md @@ -227,6 +227,8 @@ has to be disabled in Playwright on Firefox & Webkit to retain routing functiona Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available there at this time. +If you wish to run all tests in a PR, you can give it the label `X-Run-All-Tests`. + ## Supporter container runtimes We use testcontainers to spin up various instances of Synapse, Matrix Authentication Service, and more. From 0f5e135224e50dd07242e0a656d448c3da21a21b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 8 Jan 2025 09:15:20 +0000 Subject: [PATCH 3/3] Ignore failing tests in stale-screenshot-reporter.ts to avoid confusing errors Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/stale-screenshot-reporter.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/playwright/stale-screenshot-reporter.ts b/playwright/stale-screenshot-reporter.ts index 3e38f78ca93..dc934827c1d 100644 --- a/playwright/stale-screenshot-reporter.ts +++ b/playwright/stale-screenshot-reporter.ts @@ -23,6 +23,7 @@ class StaleScreenshotReporter implements Reporter { private success = true; public onTestEnd(test: TestCase): void { + if (!test.ok()) return; for (const annotation of test.annotations) { if (annotation.type === "_screenshot") { this.screenshots.add(annotation.description);