diff --git a/e2e/USER_STORIES.md b/e2e/USER_STORIES.md index e2c6b9a6..19a5b082 100644 --- a/e2e/USER_STORIES.md +++ b/e2e/USER_STORIES.md @@ -438,6 +438,19 @@ Acceptance: quote of the shared message. - Sending posts the quoted message in the destination, visible to bob there. +### US-119 — Have a message read out loud + +As alice, I want a message read to me, so that I can take in what was said +without looking at the screen. + +Acceptance: + +- "Read Out Loud" on a message with no OpenRouter key saved asks for one, the + same prompt dictation uses. +- Once a key is saved, the same menu item puts a player at the bottom of the + app naming whose message is being read. +- The player plays, pauses, scrubs, and closes, and closing it takes it away. + ### US-115 — Connect a wallet while sending a zap As alice, I want to connect a wallet from the zap dialog and go on zapping, so diff --git a/e2e/harness/index.ts b/e2e/harness/index.ts index 852b8c34..8b26f4c9 100644 --- a/e2e/harness/index.ts +++ b/e2e/harness/index.ts @@ -38,6 +38,7 @@ export { mockBlossom, mockDufflepud, mockLivekit, + mockOpenRouterSpeech, } from "./net/http" export type {DufflepudFixtures, HostingFixtures, HostingHandle, HostingRecord} from "./net/http" export type {WebLnInfo} from "./app/webln" diff --git a/e2e/harness/net/http.ts b/e2e/harness/net/http.ts index 2a9b2f82..28bee34e 100644 --- a/e2e/harness/net/http.ts +++ b/e2e/harness/net/http.ts @@ -17,6 +17,9 @@ const HOSTING_ORIGIN = "https://api.hosting.coracle.social" // Hard-coded in src/app.html, so every navigation asks for it whatever the scenario is doing. const PLAUSIBLE_ORIGIN = "https://plausible.coracle.social" +// Transcription and speech both go here, against whichever key the user has saved. +const OPENROUTER_ORIGIN = "https://openrouter.ai" + // Where the hosting api sends a browser to pay. `.test` resolves nowhere and the block-all aborts // the navigation, so a spec sees the redirect without one leaving. const CHECKOUT_ORIGIN = "https://checkout.test" @@ -210,6 +213,45 @@ export const mockDufflepud = (context: BrowserContext, fixtures: DufflepudFixtur return route.fallback() }) +// Silence as a wav, built rather than inlined so a spec can ask for a length and then assert the +// duration the player reads off it. 16 bit mono pcm is the shortest header a browser will decode. +const silence = (seconds: number) => { + const rate = 8000 + const bytes = seconds * rate * 2 + const wav = Buffer.alloc(44 + bytes) + + wav.write("RIFF", 0) + wav.writeUInt32LE(36 + bytes, 4) + wav.write("WAVEfmt ", 8) + wav.writeUInt32LE(16, 16) + wav.writeUInt16LE(1, 20) + wav.writeUInt16LE(1, 22) + wav.writeUInt32LE(rate, 24) + wav.writeUInt32LE(rate * 2, 28) + wav.writeUInt16LE(2, 32) + wav.writeUInt16LE(16, 34) + wav.write("data", 36) + wav.writeUInt32LE(bytes, 40) + + return wav +} + +/** + * OpenRouter's text to speech, answering every request with the same silence. The array it returns + * collects what the app asked to have read, in the order it asked. + */ +export const mockOpenRouterSpeech = async (context: BrowserContext, seconds = 3) => { + const spoken: string[] = [] + + await context.route(`${OPENROUTER_ORIGIN}/api/v1/audio/speech`, route => { + spoken.push(JSON.parse(route.request().postData() ?? "{}").input) + + return route.fulfill({contentType: "audio/wav", body: silence(seconds)}) + }) + + return spoken +} + export type BlossomOptions = { // The blossom server the scenario expects an upload to land on, e.g. a space's own url. server: string diff --git a/e2e/specs/rooms.spec.ts b/e2e/specs/rooms.spec.ts index dc576d13..f94e8281 100644 --- a/e2e/specs/rooms.spec.ts +++ b/e2e/specs/rooms.spec.ts @@ -2,7 +2,16 @@ import {DAY, HOUR, MINUTE, WEEK, bech32ToHex} from "@welshman/lib" import {getLnUrl} from "@welshman/util" import {MessagingRelayList, Profile, RelayList, displayPubkey} from "@welshman/domain" import type {Locator, Page} from "@playwright/test" -import {expect, makeTestUser, mockDufflepud, roomPath, spacePath, test, users} from "../harness" +import { + expect, + makeTestUser, + mockDufflepud, + mockOpenRouterSpeech, + roomPath, + spacePath, + test, + users, +} from "../harness" import type {SeededSpace, TestUser} from "../harness" // A handle to a seeded event. SeededEvent isn't exported from the harness, and only its id and @@ -855,6 +864,70 @@ test("US-028 share a message somewhere else", async ({seed, as}) => { await expect(message(bob, "heads up")).toContainText("the dock is closed on sunday") }) +test("US-119 have a message read out loud", async ({seed, as}) => { + const scenario = await seed(({relay, user, at}) => { + const space = relay("space") + + space.room("general", {name: "General"}) + space.join(user.alice, "general") + space.join(user.bob, "general") + space.profile(user.bob, {name: "Bob Barnacle"}) + space.message(user.bob, "general", "the dock is closed on sunday", at(2, HOUR)) + + seedChatter(space, user.alice) + }) + + const {url} = scenario.space("space") + const alice = await as(users.alice, roomPath(url, "general")) + const spoken = await mockOpenRouterSpeech(alice.context()) + + await expect(message(alice, "the dock is closed on sunday")).toBeVisible() + + // With no key saved, reading a message asks for one the way dictation does. + await openMessageMenu(alice, "the dock is closed on sunday") + await alice.getByRole("button", {name: "Read Out Loud"}).click() + + const enable = dialog(alice, "Enable read out loud?") + + await enable.locator('input[name="flotilla-openrouter-key"]').fill("sk-or-test") + await enable.getByRole("button", {name: "Enable read out loud"}).click() + + await expect(alice.getByRole("alert")).toContainText("Read out loud is ready to use!") + + await openMessageMenu(alice, "the dock is closed on sunday") + await alice.getByRole("button", {name: "Read Out Loud"}).click() + + await expect(alice.getByText("a message from Bob Barnacle")).toBeVisible() + + // Only what the message says is sent, so the nostr uri wrapping bob's mention never is. + expect(spoken).toEqual(["the dock is closed on sunday"]) + + // The duration is the mock's, which is what proves the player is on audio it decoded rather + // than on an element that failed to load. + await expect(alice.getByText("/ 0:03")).toBeVisible() + + // Chromium decides for itself whether the autoplay is allowed, so the control is read for + // which way it is about to flip rather than assumed to start paused. + const playPause = alice.getByRole("button", {name: /^(Play|Pause) message$/}) + const wasPlaying = (await playPause.getAttribute("aria-label")) === "Pause message" + + await playPause.click() + await expect(playPause).toHaveAttribute( + "aria-label", + wasPlaying ? "Play message" : "Pause message", + ) + + const seek = alice.getByRole("slider", {name: "Seek within the message"}) + + await seek.fill("2") + + await expect(alice.getByText("0:02 /")).toBeVisible() + + await alice.getByRole("button", {name: "Stop reading"}).click() + + await expect(alice.getByText("a message from Bob Barnacle")).toHaveCount(0) +}) + test("US-115 connect a wallet without losing the zap you were composing", async ({seed, as}) => { // The lightning address on bob's profile, and the lnurl endpoint it resolves to. Zapping only gets // as far as a dialog once dufflepud answers with a zapper for that endpoint. diff --git a/src/app/call.ts b/src/app/call.ts index 30262728..48df8b5d 100644 --- a/src/app/call.ts +++ b/src/app/call.ts @@ -1,4 +1,4 @@ -import {derived, writable} from "svelte/store" +import {derived, get, writable} from "svelte/store" import {first, uniqBy} from "@welshman/lib" import {sortEventsDesc, tagSpec, tagValues} from "@welshman/util" import {deriveDeduplicated} from "@welshman/store" @@ -262,15 +262,32 @@ export const callState = writable(CallState.Disconnected) export const callTargetRoom = writable(undefined) +export const isCallActive = derived( + callState, + $state => $state === CallState.Joining || $state === CallState.Connected, +) + export const deriveIsCallActiveElsewhere = (url: string | undefined, h: string | undefined) => derived( - [callState, callTargetRoom], - ([$state, $targetRoom]) => - ($state === CallState.Joining || $state === CallState.Connected) && + [isCallActive, callTargetRoom], + ([$isCallActive, $targetRoom]) => + $isCallActive && $targetRoom !== undefined && !($targetRoom.url === url && $targetRoom.h === h), ) +// leaveVoiceRoom no-ops during Joining, since no session exists yet to leave — cancel the +// in-flight join instead, otherwise ending a call that is still connecting does nothing. +export const endCall = async () => { + const engine = await import("@app/callEngine") + + if (get(callState) === CallState.Joining) { + engine.cancelJoinVoiceRoom() + } else { + await engine.leaveVoiceRoom() + } +} + export const speakingParticipants = writable([]) export const participantMediaState = writable(new Map()) diff --git a/src/app/components/AppContainer.svelte b/src/app/components/AppContainer.svelte index b3584f22..3d20af0f 100644 --- a/src/app/components/AppContainer.svelte +++ b/src/app/components/AppContainer.svelte @@ -4,6 +4,7 @@ import Landing from "@app/components/Landing.svelte" import Toast from "@app/components/Toast.svelte" import CallBanner from "@app/components/CallBanner.svelte" + import SpeechBanner from "@app/components/SpeechBanner.svelte" import PrimaryNav from "@app/components/PrimaryNav.svelte" import {app} from "@app/core" import {modal} from "@app/modal" @@ -26,5 +27,6 @@ {/if} + diff --git a/src/app/components/Banner.svelte b/src/app/components/Banner.svelte new file mode 100644 index 00000000..e035fc1f --- /dev/null +++ b/src/app/components/Banner.svelte @@ -0,0 +1,19 @@ + + + +
+ {@render children()} +
diff --git a/src/app/components/CallBanner.svelte b/src/app/components/CallBanner.svelte index ffe76ded..be8b6e8a 100644 --- a/src/app/components/CallBanner.svelte +++ b/src/app/components/CallBanner.svelte @@ -1,5 +1,4 @@ {#if visible} -
+
- + {/if} diff --git a/src/app/components/DictationButton.svelte b/src/app/components/DictationButton.svelte index 13969fb8..7ba28b6d 100644 --- a/src/app/components/DictationButton.svelte +++ b/src/app/components/DictationButton.svelte @@ -8,7 +8,7 @@ import Button from "@lib/components/Button.svelte" import Spinner from "@lib/components/Spinner.svelte" import {errorMessage} from "@lib/util" - import DictationEnable from "@app/components/DictationEnable.svelte" + import OpenRouterEnable from "@app/components/OpenRouterEnable.svelte" import {startDictation, transcribe} from "@app/dictation" import {getSetting} from "@app/settings" import {pushModal} from "@app/modal" @@ -42,7 +42,10 @@ loading = false } } else { - pushModal(DictationEnable) + pushModal(OpenRouterEnable, { + feature: "Voice input", + subtitle: "Dictate your messages instead of typing them.", + }) } } diff --git a/src/app/components/DictationEnable.svelte b/src/app/components/OpenRouterEnable.svelte similarity index 81% rename from src/app/components/DictationEnable.svelte rename to src/app/components/OpenRouterEnable.svelte index 35bc2bef..d3b00e06 100644 --- a/src/app/components/DictationEnable.svelte +++ b/src/app/components/OpenRouterEnable.svelte @@ -19,6 +19,15 @@ import {publishSettings} from "@app/settings" import {pushToast} from "@app/toast" + type Props = { + feature: string + subtitle: string + } + + const {feature, subtitle}: Props = $props() + + const action = `Enable ${feature.toLowerCase()}` + const back = () => history.back() const save = async () => { @@ -27,7 +36,7 @@ try { await publishSettings({openrouter_key: openrouterKey.trim()}) - pushToast({message: "Voice input is ready to use!"}) + pushToast({message: `${feature} is ready to use!`}) back() } catch (error) { @@ -45,16 +54,14 @@ - Enable voice input? - Dictate your messages instead of typing them. + {action}? + {subtitle}

- Your recordings are transcribed by OpenRouter, which charges a fraction of a cent per minute of audio. To turn voice input on, add some - credit to an OpenRouter account, then + {feature} is powered by OpenRouter, which charges a fraction of a cent for each use. To turn it on, add some credit to an + OpenRouter account, then create an API key and paste it below.

@@ -84,7 +91,7 @@ Go back diff --git a/src/app/components/RoomItemMenu.svelte b/src/app/components/RoomItemMenu.svelte index 413d3893..0ea25a76 100644 --- a/src/app/components/RoomItemMenu.svelte +++ b/src/app/components/RoomItemMenu.svelte @@ -9,6 +9,7 @@ import Danger from "@assets/icons/danger.svg?dataurl" import Pin from "@assets/icons/pin.svg?dataurl" import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl" + import VolumeLoud from "@assets/icons/volume-loud.svg?dataurl" import Button from "@lib/components/Button.svelte" import Icon from "@lib/components/Icon.svelte" import Confirm from "@lib/components/Confirm.svelte" @@ -21,6 +22,7 @@ import {deriveUserIsSpaceAdmin} from "@app/management" import {ROOM, deriveUserIsRoomAdmin} from "@app/rooms" import {shareEvent} from "@app/share" + import {readAloud} from "@app/speech" import {pushModal} from "@app/modal" import {pushToast} from "@app/toast" @@ -48,6 +50,11 @@ pushModal(PinboardSelect, {url, event}) } + const read = () => { + onClick() + readAloud(event) + } + const createThread = () => { onClick() pushModal(ThreadCreate, { @@ -129,6 +136,12 @@ Add to Library +
  • + +
    • {#if h}
  • + {#if h} + {:else} + + {/if} +
    + + + {$speech.src ? "Reading" : "Preparing"} + a message from {$speech.title} + + {#if $speech.src} + + + {display(currentTime)} / {display(scrubbable)} + + {/if} +
    + + +{/if} diff --git a/src/app/speech.ts b/src/app/speech.ts new file mode 100644 index 00000000..de10b33f --- /dev/null +++ b/src/app/speech.ts @@ -0,0 +1,107 @@ +import {get, writable} from "svelte/store" +import {parse, renderAsText} from "@welshman/content" +import type {Maybe} from "@welshman/lib" +import type {TrustedEvent} from "@welshman/util" +import {errorMessage} from "@lib/util" +import Confirm from "@lib/components/Confirm.svelte" +import OpenRouterEnable from "@app/components/OpenRouterEnable.svelte" +import {endCall, isCallActive} from "@app/call" +import {profiles} from "@app/core" +import {pushModal} from "@app/modal" +import {getSetting} from "@app/settings" +import {pushToast} from "@app/toast" + +const SPEECH_MODEL = "openai/gpt-audio-mini" + +const SPEECH_VOICE = "alloy" + +export type Speech = { + id: string + title: string + src?: string +} + +export const speech = writable>(undefined) + +export const synthesize = async (text: string) => { + const response = await fetch("https://openrouter.ai/api/v1/audio/speech", { + method: "POST", + headers: { + Authorization: `Bearer ${getSetting("openrouter_key")}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({model: SPEECH_MODEL, voice: SPEECH_VOICE, input: text}), + }) + + // A successful response is audio rather than json, so the error body is only worth reading once + // the status says there is one. + if (!response.ok) { + const {error} = await response.json().catch(() => ({error: undefined})) + + throw new Error(error?.message || `OpenRouter returned a ${response.status}.`) + } + + return response.blob() +} + +export const stopSpeech = () => + speech.update(current => { + if (current?.src) { + URL.revokeObjectURL(current.src) + } + + return undefined + }) + +const play = async (event: TrustedEvent, text: string) => { + const {id, pubkey} = event + const title = profiles.get().display(pubkey).get() + + stopSpeech() + speech.set({id, title}) + + try { + const src = URL.createObjectURL(await synthesize(text)) + + // Reading a second message while this one was in flight hands the player to that one. + if (get(speech)?.id === id) { + speech.set({id, title, src}) + } else { + URL.revokeObjectURL(src) + } + } catch (error) { + console.error(error) + pushToast({theme: "error", message: `Failed to read this message: ${errorMessage(error)}`}) + + if (get(speech)?.id === id) { + stopSpeech() + } + } +} + +export const readAloud = (event: TrustedEvent) => { + const text = renderAsText(parse(event)).toString().trim() + + if (getSetting("openrouter_key")) { + if (get(isCallActive)) { + pushModal(Confirm, { + title: "Leave the call?", + message: "Reading a message out loud would talk over the call you are in.", + confirm: async () => { + await endCall() + + history.back() + + await play(event, text) + }, + }) + } else { + play(event, text) + } + } else { + pushModal(OpenRouterEnable, { + feature: "Read out loud", + subtitle: "Have a message read to you instead of reading it yourself.", + }) + } +} diff --git a/src/routes/settings/content/+page.svelte b/src/routes/settings/content/+page.svelte index 9b1c0064..c1beb9fd 100644 --- a/src/routes/settings/content/+page.svelte +++ b/src/routes/settings/content/+page.svelte @@ -142,7 +142,7 @@ {#snippet label()} -

    Voice Input

    +

    OpenRouter

    {/snippet} {#snippet input()}