Read a message out loud from the message menu

This commit is contained in:
Coracle-Bot 2026-09-09 14:43:55 +00:00 committed by hodlbod
parent 7e82429a1a
commit e66dde1ea0
15 changed files with 416 additions and 37 deletions

View file

@ -438,6 +438,19 @@ Acceptance:
quote of the shared message. quote of the shared message.
- Sending posts the quoted message in the destination, visible to bob there. - Sending posts the quoted message in the destination, visible to bob there.
### US-119 — Have a message read out loud
As alice, I want a message read to me, so that I can take in what was said
without looking at the screen.
Acceptance:
- "Read Out Loud" on a message with no OpenRouter key saved asks for one, the
same prompt dictation uses.
- Once a key is saved, the same menu item puts a player at the bottom of the
app naming whose message is being read.
- The player plays, pauses, scrubs, and closes, and closing it takes it away.
### US-115 — Connect a wallet while sending a zap ### US-115 — Connect a wallet while sending a zap
As alice, I want to connect a wallet from the zap dialog and go on zapping, so As alice, I want to connect a wallet from the zap dialog and go on zapping, so

View file

@ -38,6 +38,7 @@ export {
mockBlossom, mockBlossom,
mockDufflepud, mockDufflepud,
mockLivekit, mockLivekit,
mockOpenRouterSpeech,
} from "./net/http" } from "./net/http"
export type {DufflepudFixtures, HostingFixtures, HostingHandle, HostingRecord} from "./net/http" export type {DufflepudFixtures, HostingFixtures, HostingHandle, HostingRecord} from "./net/http"
export type {WebLnInfo} from "./app/webln" export type {WebLnInfo} from "./app/webln"

View file

@ -17,6 +17,9 @@ const HOSTING_ORIGIN = "https://api.hosting.coracle.social"
// Hard-coded in src/app.html, so every navigation asks for it whatever the scenario is doing. // Hard-coded in src/app.html, so every navigation asks for it whatever the scenario is doing.
const PLAUSIBLE_ORIGIN = "https://plausible.coracle.social" const PLAUSIBLE_ORIGIN = "https://plausible.coracle.social"
// Transcription and speech both go here, against whichever key the user has saved.
const OPENROUTER_ORIGIN = "https://openrouter.ai"
// Where the hosting api sends a browser to pay. `.test` resolves nowhere and the block-all aborts // Where the hosting api sends a browser to pay. `.test` resolves nowhere and the block-all aborts
// the navigation, so a spec sees the redirect without one leaving. // the navigation, so a spec sees the redirect without one leaving.
const CHECKOUT_ORIGIN = "https://checkout.test" const CHECKOUT_ORIGIN = "https://checkout.test"
@ -210,6 +213,45 @@ export const mockDufflepud = (context: BrowserContext, fixtures: DufflepudFixtur
return route.fallback() return route.fallback()
}) })
// Silence as a wav, built rather than inlined so a spec can ask for a length and then assert the
// duration the player reads off it. 16 bit mono pcm is the shortest header a browser will decode.
const silence = (seconds: number) => {
const rate = 8000
const bytes = seconds * rate * 2
const wav = Buffer.alloc(44 + bytes)
wav.write("RIFF", 0)
wav.writeUInt32LE(36 + bytes, 4)
wav.write("WAVEfmt ", 8)
wav.writeUInt32LE(16, 16)
wav.writeUInt16LE(1, 20)
wav.writeUInt16LE(1, 22)
wav.writeUInt32LE(rate, 24)
wav.writeUInt32LE(rate * 2, 28)
wav.writeUInt16LE(2, 32)
wav.writeUInt16LE(16, 34)
wav.write("data", 36)
wav.writeUInt32LE(bytes, 40)
return wav
}
/**
* OpenRouter's text to speech, answering every request with the same silence. The array it returns
* collects what the app asked to have read, in the order it asked.
*/
export const mockOpenRouterSpeech = async (context: BrowserContext, seconds = 3) => {
const spoken: string[] = []
await context.route(`${OPENROUTER_ORIGIN}/api/v1/audio/speech`, route => {
spoken.push(JSON.parse(route.request().postData() ?? "{}").input)
return route.fulfill({contentType: "audio/wav", body: silence(seconds)})
})
return spoken
}
export type BlossomOptions = { export type BlossomOptions = {
// The blossom server the scenario expects an upload to land on, e.g. a space's own url. // The blossom server the scenario expects an upload to land on, e.g. a space's own url.
server: string server: string

View file

@ -2,7 +2,16 @@ import {DAY, HOUR, MINUTE, WEEK, bech32ToHex} from "@welshman/lib"
import {getLnUrl} from "@welshman/util" import {getLnUrl} from "@welshman/util"
import {MessagingRelayList, Profile, RelayList, displayPubkey} from "@welshman/domain" import {MessagingRelayList, Profile, RelayList, displayPubkey} from "@welshman/domain"
import type {Locator, Page} from "@playwright/test" import type {Locator, Page} from "@playwright/test"
import {expect, makeTestUser, mockDufflepud, roomPath, spacePath, test, users} from "../harness" import {
expect,
makeTestUser,
mockDufflepud,
mockOpenRouterSpeech,
roomPath,
spacePath,
test,
users,
} from "../harness"
import type {SeededSpace, TestUser} from "../harness" import type {SeededSpace, TestUser} from "../harness"
// A handle to a seeded event. SeededEvent isn't exported from the harness, and only its id and // A handle to a seeded event. SeededEvent isn't exported from the harness, and only its id and
@ -855,6 +864,70 @@ test("US-028 share a message somewhere else", async ({seed, as}) => {
await expect(message(bob, "heads up")).toContainText("the dock is closed on sunday") await expect(message(bob, "heads up")).toContainText("the dock is closed on sunday")
}) })
test("US-119 have a message read out loud", async ({seed, as}) => {
const scenario = await seed(({relay, user, at}) => {
const space = relay("space")
space.room("general", {name: "General"})
space.join(user.alice, "general")
space.join(user.bob, "general")
space.profile(user.bob, {name: "Bob Barnacle"})
space.message(user.bob, "general", "the dock is closed on sunday", at(2, HOUR))
seedChatter(space, user.alice)
})
const {url} = scenario.space("space")
const alice = await as(users.alice, roomPath(url, "general"))
const spoken = await mockOpenRouterSpeech(alice.context())
await expect(message(alice, "the dock is closed on sunday")).toBeVisible()
// With no key saved, reading a message asks for one the way dictation does.
await openMessageMenu(alice, "the dock is closed on sunday")
await alice.getByRole("button", {name: "Read Out Loud"}).click()
const enable = dialog(alice, "Enable read out loud?")
await enable.locator('input[name="flotilla-openrouter-key"]').fill("sk-or-test")
await enable.getByRole("button", {name: "Enable read out loud"}).click()
await expect(alice.getByRole("alert")).toContainText("Read out loud is ready to use!")
await openMessageMenu(alice, "the dock is closed on sunday")
await alice.getByRole("button", {name: "Read Out Loud"}).click()
await expect(alice.getByText("a message from Bob Barnacle")).toBeVisible()
// Only what the message says is sent, so the nostr uri wrapping bob's mention never is.
expect(spoken).toEqual(["the dock is closed on sunday"])
// The duration is the mock's, which is what proves the player is on audio it decoded rather
// than on an element that failed to load.
await expect(alice.getByText("/ 0:03")).toBeVisible()
// Chromium decides for itself whether the autoplay is allowed, so the control is read for
// which way it is about to flip rather than assumed to start paused.
const playPause = alice.getByRole("button", {name: /^(Play|Pause) message$/})
const wasPlaying = (await playPause.getAttribute("aria-label")) === "Pause message"
await playPause.click()
await expect(playPause).toHaveAttribute(
"aria-label",
wasPlaying ? "Play message" : "Pause message",
)
const seek = alice.getByRole("slider", {name: "Seek within the message"})
await seek.fill("2")
await expect(alice.getByText("0:02 /")).toBeVisible()
await alice.getByRole("button", {name: "Stop reading"}).click()
await expect(alice.getByText("a message from Bob Barnacle")).toHaveCount(0)
})
test("US-115 connect a wallet without losing the zap you were composing", async ({seed, as}) => { test("US-115 connect a wallet without losing the zap you were composing", async ({seed, as}) => {
// The lightning address on bob's profile, and the lnurl endpoint it resolves to. Zapping only gets // The lightning address on bob's profile, and the lnurl endpoint it resolves to. Zapping only gets
// as far as a dialog once dufflepud answers with a zapper for that endpoint. // as far as a dialog once dufflepud answers with a zapper for that endpoint.

View file

@ -1,4 +1,4 @@
import {derived, writable} from "svelte/store" import {derived, get, writable} from "svelte/store"
import {first, uniqBy} from "@welshman/lib" import {first, uniqBy} from "@welshman/lib"
import {sortEventsDesc, tagSpec, tagValues} from "@welshman/util" import {sortEventsDesc, tagSpec, tagValues} from "@welshman/util"
import {deriveDeduplicated} from "@welshman/store" import {deriveDeduplicated} from "@welshman/store"
@ -262,15 +262,32 @@ export const callState = writable<CallState>(CallState.Disconnected)
export const callTargetRoom = writable<Room | undefined>(undefined) export const callTargetRoom = writable<Room | undefined>(undefined)
export const isCallActive = derived(
callState,
$state => $state === CallState.Joining || $state === CallState.Connected,
)
export const deriveIsCallActiveElsewhere = (url: string | undefined, h: string | undefined) => export const deriveIsCallActiveElsewhere = (url: string | undefined, h: string | undefined) =>
derived( derived(
[callState, callTargetRoom], [isCallActive, callTargetRoom],
([$state, $targetRoom]) => ([$isCallActive, $targetRoom]) =>
($state === CallState.Joining || $state === CallState.Connected) && $isCallActive &&
$targetRoom !== undefined && $targetRoom !== undefined &&
!($targetRoom.url === url && $targetRoom.h === h), !($targetRoom.url === url && $targetRoom.h === h),
) )
// leaveVoiceRoom no-ops during Joining, since no session exists yet to leave — cancel the
// in-flight join instead, otherwise ending a call that is still connecting does nothing.
export const endCall = async () => {
const engine = await import("@app/callEngine")
if (get(callState) === CallState.Joining) {
engine.cancelJoinVoiceRoom()
} else {
await engine.leaveVoiceRoom()
}
}
export const speakingParticipants = writable<CallParticipant[]>([]) export const speakingParticipants = writable<CallParticipant[]>([])
export const participantMediaState = writable(new Map<string, ParticipantMediaState>()) export const participantMediaState = writable(new Map<string, ParticipantMediaState>())

View file

@ -4,6 +4,7 @@
import Landing from "@app/components/Landing.svelte" import Landing from "@app/components/Landing.svelte"
import Toast from "@app/components/Toast.svelte" import Toast from "@app/components/Toast.svelte"
import CallBanner from "@app/components/CallBanner.svelte" import CallBanner from "@app/components/CallBanner.svelte"
import SpeechBanner from "@app/components/SpeechBanner.svelte"
import PrimaryNav from "@app/components/PrimaryNav.svelte" import PrimaryNav from "@app/components/PrimaryNav.svelte"
import {app} from "@app/core" import {app} from "@app/core"
import {modal} from "@app/modal" import {modal} from "@app/modal"
@ -26,5 +27,6 @@
{/if} {/if}
</div> </div>
<CallBanner /> <CallBanner />
<SpeechBanner />
</div> </div>
<Toast /> <Toast />

View file

@ -0,0 +1,19 @@
<script lang="ts">
import type {Snippet} from "svelte"
import {fly} from "svelte/transition"
type Props = {
children: Snippet
}
const {children}: Props = $props()
</script>
<!-- Only the bottom-most banner has to clear the mobile nav; one stacked above another sits
flush against it instead of leaving the nav's height between them. -->
<div
in:fly={{y: 60, duration: 250}}
out:fly={{y: 60, duration: 200}}
class="relative flex shrink-0 items-center gap-3 border-t border-line bg-surface py-2 pl-4 pr-2 mb-[calc(var(--saib)+3.5rem)] md:mb-[var(--saib)] md:pl-6 [&:not(:last-child)]:mb-0">
{@render children()}
</div>

View file

@ -1,5 +1,4 @@
<script lang="ts"> <script lang="ts">
import {fly} from "svelte/transition"
import {navigate} from "@app/modal" import {navigate} from "@app/modal"
import {page} from "$app/stores" import {page} from "$app/stores"
import cx from "classnames" import cx from "classnames"
@ -9,6 +8,7 @@
import EndCall from "@assets/icons/end-call-rounded.svg?dataurl" import EndCall from "@assets/icons/end-call-rounded.svg?dataurl"
import Icon from "@lib/components/Icon.svelte" import Icon from "@lib/components/Icon.svelte"
import Button from "@lib/components/Button.svelte" import Button from "@lib/components/Button.svelte"
import Banner from "@app/components/Banner.svelte"
import {decodeRelay} from "@app/relays" import {decodeRelay} from "@app/relays"
import {displayRoom} from "@app/rooms" import {displayRoom} from "@app/rooms"
import {makeRoomPath} from "@app/routes" import {makeRoomPath} from "@app/routes"
@ -18,6 +18,7 @@
callTargetRoom, callTargetRoom,
callMicMuted, callMicMuted,
deriveIsCallActiveElsewhere, deriveIsCallActiveElsewhere,
endCall,
} from "@app/call" } from "@app/call"
const {relay, h} = $derived($page.params) const {relay, h} = $derived($page.params)
@ -45,25 +46,10 @@
await engine.toggleMute() await engine.toggleMute()
} }
// leaveVoiceRoom no-ops during Joining (no session exists yet to leave) — cancel
// the in-flight join instead, otherwise this button silently does nothing.
const endCall = async () => {
const engine = await import("@app/callEngine")
if ($callState === CallState.Joining) {
engine.cancelJoinVoiceRoom()
} else {
await engine.leaveVoiceRoom()
}
}
</script> </script>
{#if visible} {#if visible}
<div <Banner>
in:fly={{y: 60, duration: 250}}
out:fly={{y: 60, duration: 200}}
class="relative flex shrink-0 items-center gap-3 border-t border-line bg-surface py-2 pl-4 pr-2 mb-[calc(var(--saib)+3.5rem)] md:mb-[var(--saib)] md:pl-6">
<Button <Button
class="flex min-w-0 flex-1 items-center gap-2 text-left" class="flex min-w-0 flex-1 items-center gap-2 text-left"
onclick={goToRoom} onclick={goToRoom}
@ -103,5 +89,5 @@
<Icon icon={EndCall} size={4} /> <Icon icon={EndCall} size={4} />
</Button> </Button>
</div> </div>
</div> </Banner>
{/if} {/if}

View file

@ -8,7 +8,7 @@
import Button from "@lib/components/Button.svelte" import Button from "@lib/components/Button.svelte"
import Spinner from "@lib/components/Spinner.svelte" import Spinner from "@lib/components/Spinner.svelte"
import {errorMessage} from "@lib/util" import {errorMessage} from "@lib/util"
import DictationEnable from "@app/components/DictationEnable.svelte" import OpenRouterEnable from "@app/components/OpenRouterEnable.svelte"
import {startDictation, transcribe} from "@app/dictation" import {startDictation, transcribe} from "@app/dictation"
import {getSetting} from "@app/settings" import {getSetting} from "@app/settings"
import {pushModal} from "@app/modal" import {pushModal} from "@app/modal"
@ -42,7 +42,10 @@
loading = false loading = false
} }
} else { } else {
pushModal(DictationEnable) pushModal(OpenRouterEnable, {
feature: "Voice input",
subtitle: "Dictate your messages instead of typing them.",
})
} }
} }

View file

@ -19,6 +19,15 @@
import {publishSettings} from "@app/settings" import {publishSettings} from "@app/settings"
import {pushToast} from "@app/toast" import {pushToast} from "@app/toast"
type Props = {
feature: string
subtitle: string
}
const {feature, subtitle}: Props = $props()
const action = `Enable ${feature.toLowerCase()}`
const back = () => history.back() const back = () => history.back()
const save = async () => { const save = async () => {
@ -27,7 +36,7 @@
try { try {
await publishSettings({openrouter_key: openrouterKey.trim()}) await publishSettings({openrouter_key: openrouterKey.trim()})
pushToast({message: "Voice input is ready to use!"}) pushToast({message: `${feature} is ready to use!`})
back() back()
} catch (error) { } catch (error) {
@ -45,16 +54,14 @@
<Modal tag="form" onsubmit={preventDefault(save)}> <Modal tag="form" onsubmit={preventDefault(save)}>
<ModalBody> <ModalBody>
<ModalHeader> <ModalHeader>
<ModalTitle>Enable voice input?</ModalTitle> <ModalTitle>{action}?</ModalTitle>
<ModalSubtitle>Dictate your messages instead of typing them.</ModalSubtitle> <ModalSubtitle>{subtitle}</ModalSubtitle>
</ModalHeader> </ModalHeader>
<p> <p>
Your recordings are transcribed by <Link {feature} is powered by <Link external href="https://openrouter.ai" class="text-primary"
external >OpenRouter</Link
href="https://openrouter.ai" >, which charges a fraction of a cent for each use. To turn it on, add some credit to an
class="text-primary">OpenRouter</Link OpenRouter account, then
>, which charges a fraction of a cent per minute of audio. To turn voice input on, add some
credit to an OpenRouter account, then
<Link external href="https://openrouter.ai/settings/keys" class="text-primary" <Link external href="https://openrouter.ai/settings/keys" class="text-primary"
>create an API key</Link> and paste it below. >create an API key</Link> and paste it below.
</p> </p>
@ -84,7 +91,7 @@
Go back Go back
</Button> </Button>
<Button type="submit" class="button button-primary" disabled={!openrouterKey || loading}> <Button type="submit" class="button button-primary" disabled={!openrouterKey || loading}>
<Spinner {loading}>Enable voice input</Spinner> <Spinner {loading}>{action}</Spinner>
<Icon icon={AltArrowRight} /> <Icon icon={AltArrowRight} />
</Button> </Button>
</ModalFooter> </ModalFooter>

View file

@ -9,6 +9,7 @@
import Danger from "@assets/icons/danger.svg?dataurl" import Danger from "@assets/icons/danger.svg?dataurl"
import Pin from "@assets/icons/pin.svg?dataurl" import Pin from "@assets/icons/pin.svg?dataurl"
import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl" import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl"
import VolumeLoud from "@assets/icons/volume-loud.svg?dataurl"
import Button from "@lib/components/Button.svelte" import Button from "@lib/components/Button.svelte"
import Icon from "@lib/components/Icon.svelte" import Icon from "@lib/components/Icon.svelte"
import Confirm from "@lib/components/Confirm.svelte" import Confirm from "@lib/components/Confirm.svelte"
@ -21,6 +22,7 @@
import {deriveUserIsSpaceAdmin} from "@app/management" import {deriveUserIsSpaceAdmin} from "@app/management"
import {ROOM, deriveUserIsRoomAdmin} from "@app/rooms" import {ROOM, deriveUserIsRoomAdmin} from "@app/rooms"
import {shareEvent} from "@app/share" import {shareEvent} from "@app/share"
import {readAloud} from "@app/speech"
import {pushModal} from "@app/modal" import {pushModal} from "@app/modal"
import {pushToast} from "@app/toast" import {pushToast} from "@app/toast"
@ -48,6 +50,11 @@
pushModal(PinboardSelect, {url, event}) pushModal(PinboardSelect, {url, event})
} }
const read = () => {
onClick()
readAloud(event)
}
const createThread = () => { const createThread = () => {
onClick() onClick()
pushModal(ThreadCreate, { pushModal(ThreadCreate, {
@ -129,6 +136,12 @@
Add to Library Add to Library
</Button> </Button>
</li> </li>
<li>
<Button onclick={read}>
<Icon size={4} icon={VolumeLoud} />
Read Out Loud
</Button>
</li>
{#if h} {#if h}
<li> <li>
<Button onclick={createThread}> <Button onclick={createThread}>

View file

@ -15,6 +15,7 @@
import AltArrowUp from "@assets/icons/alt-arrow-up.svg?dataurl" import AltArrowUp from "@assets/icons/alt-arrow-up.svg?dataurl"
import Pin from "@assets/icons/pin.svg?dataurl" import Pin from "@assets/icons/pin.svg?dataurl"
import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl" import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl"
import VolumeLoud from "@assets/icons/volume-loud.svg?dataurl"
import {slideAndFade} from "@lib/transition" import {slideAndFade} from "@lib/transition"
import Button from "@lib/components/Button.svelte" import Button from "@lib/components/Button.svelte"
import Link from "@lib/components/Link.svelte" import Link from "@lib/components/Link.svelte"
@ -31,6 +32,7 @@
import {ENABLE_ZAPS} from "@app/env" import {ENABLE_ZAPS} from "@app/env"
import {makeContentPath} from "@app/routes" import {makeContentPath} from "@app/routes"
import {shareEvent} from "@app/share" import {shareEvent} from "@app/share"
import {readAloud} from "@app/speech"
import {pushModal} from "@app/modal" import {pushModal} from "@app/modal"
import {pushToast} from "@app/toast" import {pushToast} from "@app/toast"
@ -88,6 +90,11 @@
shareEvent(url, "Message", event) shareEvent(url, "Message", event)
} }
const read = () => {
history.back()
readAloud(event)
}
const showInfo = () => pushModal(EventInfo, {url, event}, {replaceState: true}) const showInfo = () => pushModal(EventInfo, {url, event}, {replaceState: true})
const showDelete = () => pushModal(EventDeleteConfirm, {url, event}) const showDelete = () => pushModal(EventDeleteConfirm, {url, event})
@ -153,6 +160,10 @@
<Icon size={4} icon={ShareCircle} /> <Icon size={4} icon={ShareCircle} />
Share Share
</Button> </Button>
<Button class="button button-neutral w-full" onclick={read}>
<Icon size={4} icon={VolumeLoud} />
Read Out Loud
</Button>
{#if h} {#if h}
<Button class="button button-neutral w-full" onclick={createThread}> <Button class="button button-neutral w-full" onclick={createThread}>
<Icon size={4} icon={NotesMinimalistic} /> <Icon size={4} icon={NotesMinimalistic} />

View file

@ -0,0 +1,84 @@
<script lang="ts">
import Play from "@assets/icons/play.svg?dataurl"
import Pause from "@assets/icons/pause.svg?dataurl"
import Close from "@assets/icons/close.svg?dataurl"
import VolumeLoud from "@assets/icons/volume-loud.svg?dataurl"
import Icon from "@lib/components/Icon.svelte"
import Button from "@lib/components/Button.svelte"
import Spinner from "@lib/components/Spinner.svelte"
import Banner from "@app/components/Banner.svelte"
import {speech, stopSpeech} from "@app/speech"
let audio: HTMLAudioElement
let playing = $state(false)
let currentTime = $state(0)
let duration = $state(0)
// Duration is NaN until the browser has read the metadata, so the scrubber stays at zero
// length rather than rendering a range with no end.
const scrubbable = $derived(Number.isFinite(duration) ? duration : 0)
const toggle = () => (playing ? audio.pause() : audio.play())
const scrub = (event: Event & {currentTarget: HTMLInputElement}) => {
audio.currentTime = Number(event.currentTarget.value)
currentTime = audio.currentTime
}
const display = (seconds: number) => {
const whole = Math.floor(seconds)
return `${Math.floor(whole / 60)}:${String(whole % 60).padStart(2, "0")}`
}
</script>
{#if $speech}
<Banner>
{#if $speech.src}
<audio
bind:this={audio}
src={$speech.src}
autoplay
onplay={() => (playing = true)}
onpause={() => (playing = false)}
onended={() => (playing = false)}
ondurationchange={() => (duration = audio.duration)}
ontimeupdate={() => (currentTime = audio.currentTime)}></audio>
<Button
aria-label={playing ? "Pause message" : "Play message"}
class="button button-circle button-sm button-primary shrink-0"
onclick={toggle}>
<Icon icon={playing ? Pause : Play} size={4} />
</Button>
{:else}
<Spinner size="xs" />
{/if}
<div class="flex min-w-0 flex-1 flex-col gap-1">
<span class="flex min-w-0 items-center gap-2 truncate text-sm">
<Icon icon={VolumeLoud} size={4} />
<span class="font-semibold">{$speech.src ? "Reading" : "Preparing"}</span>
<span class="truncate text-content-muted">a message from {$speech.title}</span>
</span>
{#if $speech.src}
<span class="flex items-center gap-2 text-xs text-content-muted">
<input
aria-label="Seek within the message"
class="range min-w-0 flex-1"
type="range"
min="0"
max={scrubbable}
step="0.1"
value={currentTime}
oninput={scrub} />
<span class="whitespace-nowrap">{display(currentTime)} / {display(scrubbable)}</span>
</span>
{/if}
</div>
<Button
aria-label="Stop reading"
class="button button-circle button-sm button-neutral shrink-0"
onclick={stopSpeech}>
<Icon icon={Close} size={4} />
</Button>
</Banner>
{/if}

107
src/app/speech.ts Normal file
View file

@ -0,0 +1,107 @@
import {get, writable} from "svelte/store"
import {parse, renderAsText} from "@welshman/content"
import type {Maybe} from "@welshman/lib"
import type {TrustedEvent} from "@welshman/util"
import {errorMessage} from "@lib/util"
import Confirm from "@lib/components/Confirm.svelte"
import OpenRouterEnable from "@app/components/OpenRouterEnable.svelte"
import {endCall, isCallActive} from "@app/call"
import {profiles} from "@app/core"
import {pushModal} from "@app/modal"
import {getSetting} from "@app/settings"
import {pushToast} from "@app/toast"
const SPEECH_MODEL = "openai/gpt-audio-mini"
const SPEECH_VOICE = "alloy"
export type Speech = {
id: string
title: string
src?: string
}
export const speech = writable<Maybe<Speech>>(undefined)
export const synthesize = async (text: string) => {
const response = await fetch("https://openrouter.ai/api/v1/audio/speech", {
method: "POST",
headers: {
Authorization: `Bearer ${getSetting("openrouter_key")}`,
"Content-Type": "application/json",
},
body: JSON.stringify({model: SPEECH_MODEL, voice: SPEECH_VOICE, input: text}),
})
// A successful response is audio rather than json, so the error body is only worth reading once
// the status says there is one.
if (!response.ok) {
const {error} = await response.json().catch(() => ({error: undefined}))
throw new Error(error?.message || `OpenRouter returned a ${response.status}.`)
}
return response.blob()
}
export const stopSpeech = () =>
speech.update(current => {
if (current?.src) {
URL.revokeObjectURL(current.src)
}
return undefined
})
const play = async (event: TrustedEvent, text: string) => {
const {id, pubkey} = event
const title = profiles.get().display(pubkey).get()
stopSpeech()
speech.set({id, title})
try {
const src = URL.createObjectURL(await synthesize(text))
// Reading a second message while this one was in flight hands the player to that one.
if (get(speech)?.id === id) {
speech.set({id, title, src})
} else {
URL.revokeObjectURL(src)
}
} catch (error) {
console.error(error)
pushToast({theme: "error", message: `Failed to read this message: ${errorMessage(error)}`})
if (get(speech)?.id === id) {
stopSpeech()
}
}
}
export const readAloud = (event: TrustedEvent) => {
const text = renderAsText(parse(event)).toString().trim()
if (getSetting("openrouter_key")) {
if (get(isCallActive)) {
pushModal(Confirm, {
title: "Leave the call?",
message: "Reading a message out loud would talk over the call you are in.",
confirm: async () => {
await endCall()
history.back()
await play(event, text)
},
})
} else {
play(event, text)
}
} else {
pushModal(OpenRouterEnable, {
feature: "Read out loud",
subtitle: "Have a message read to you instead of reading it yourself.",
})
}
}

View file

@ -142,7 +142,7 @@
</Field> </Field>
<Field> <Field>
{#snippet label()} {#snippet label()}
<p>Voice Input</p> <p>OpenRouter</p>
{/snippet} {/snippet}
{#snippet input()} {#snippet input()}
<label class="input flex w-full items-center gap-2"> <label class="input flex w-full items-center gap-2">
@ -159,7 +159,8 @@
{#snippet info()} {#snippet info()}
<p> <p>
Add an <Link external href="https://openrouter.ai/settings/keys" class="text-primary" Add an <Link external href="https://openrouter.ai/settings/keys" class="text-primary"
>OpenRouter API key</Link> to dictate messages using the microphone button in your composer. >OpenRouter API key</Link> to dictate messages using the microphone button in your composer,
and to have messages read out loud to you.
</p> </p>
{/snippet} {/snippet}
</Field> </Field>