Attach the browser console to a failing e2e test (#525)

e2e/ARCHITECTURE.md

@@ -414,6 +414,11 @@ pnpm test                                             # starts and stops the con
 
 Every test skips when docker is unavailable, rather than failing.
 
+A failing test attaches `browser-console`, the console errors and uncaught exceptions its pages
+raised. `use.trace` and playwright's own reporting only cover contexts playwright made itself, and
+the harness makes its own, so without that attachment a failure carries the DOM snapshot and
+nothing the app said — which is unreadable when what failed is the app rendering its error page.
+
 One engine per run. These specs exercise sockets, auth and sync, so running them under three
 engines adds little coverage. `E2E_BROWSER=webkit pnpm test` runs the whole
 suite under another one. The container listens on a fixed port and cannot be sharded, so

e2e/harness/index.ts

@@ -1,3 +1,4 @@
+import {writeFile} from "node:fs/promises"
 import {npubEncode} from "nostr-tools/nip19"
 import {test as base, expect} from "@playwright/test"
 import type {BrowserContext, BrowserContextOptions, Page} from "@playwright/test"
@@ -193,6 +194,7 @@ export const test = base.extend<HarnessFixtures, HarnessWorkerFixtures>({
     await zooid.reset()
 
     const contexts: BrowserContext[] = []
+    const browserLog: string[] = []
 
     let scenario: Maybe<Scenario>
 
@@ -218,6 +220,21 @@ export const test = base.extend<HarnessFixtures, HarnessWorkerFixtures>({
 
       contexts.push(context)
 
+      // `use.trace` and the built-in reporting only cover contexts playwright made itself, so a
+      // failure in a context opened here arrives with the app's own account of it thrown away —
+      // which is how a spec that caught the app on its 500 page had nothing to say about why.
+      const who = user?.name ?? "anonymous"
+
+      context.on("console", message => {
+        if (["error", "warning"].includes(message.type())) {
+          browserLog.push(`[${who}] ${message.type()}: ${message.text()}`)
+        }
+      })
+
+      context.on("weberror", error => {
+        browserLog.push(`[${who}] uncaught: ${error.error().stack ?? error.error().message}`)
+      })
+
       // Playwright matches the most recently registered route first and every mock falls through
       // what it doesn't recognize, so the block-all goes in before the mocks, and all of it before
       // the page navigates.
@@ -277,6 +294,15 @@ export const test = base.extend<HarnessFixtures, HarnessWorkerFixtures>({
       },
     })
 
+    if (browserLog.length > 0 && testInfo.status !== testInfo.expectedStatus) {
+      // A path rather than a body: the list reporter truncates an inline attachment, and the
+      // nightly run on the box keeps test-results and nothing else.
+      const path = testInfo.outputPath("browser-console.log")
+
+      await writeFile(path, browserLog.join("\n"))
+      await testInfo.attach("browser-console", {path, contentType: "text/plain"})
+    }
+
     for (const context of contexts) {
       await context.close()
     }