Read a message out loud from the message menu
This commit is contained in:
parent
7e82429a1a
commit
e66dde1ea0
15 changed files with 416 additions and 37 deletions
|
|
@ -438,6 +438,19 @@ Acceptance:
|
||||||
quote of the shared message.
|
quote of the shared message.
|
||||||
- Sending posts the quoted message in the destination, visible to bob there.
|
- Sending posts the quoted message in the destination, visible to bob there.
|
||||||
|
|
||||||
|
### US-119 — Have a message read out loud
|
||||||
|
|
||||||
|
As alice, I want a message read to me, so that I can take in what was said
|
||||||
|
without looking at the screen.
|
||||||
|
|
||||||
|
Acceptance:
|
||||||
|
|
||||||
|
- "Read Out Loud" on a message with no OpenRouter key saved asks for one, the
|
||||||
|
same prompt dictation uses.
|
||||||
|
- Once a key is saved, the same menu item puts a player at the bottom of the
|
||||||
|
app naming whose message is being read.
|
||||||
|
- The player plays, pauses, scrubs, and closes, and closing it takes it away.
|
||||||
|
|
||||||
### US-115 — Connect a wallet while sending a zap
|
### US-115 — Connect a wallet while sending a zap
|
||||||
|
|
||||||
As alice, I want to connect a wallet from the zap dialog and go on zapping, so
|
As alice, I want to connect a wallet from the zap dialog and go on zapping, so
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,7 @@ export {
|
||||||
mockBlossom,
|
mockBlossom,
|
||||||
mockDufflepud,
|
mockDufflepud,
|
||||||
mockLivekit,
|
mockLivekit,
|
||||||
|
mockOpenRouterSpeech,
|
||||||
} from "./net/http"
|
} from "./net/http"
|
||||||
export type {DufflepudFixtures, HostingFixtures, HostingHandle, HostingRecord} from "./net/http"
|
export type {DufflepudFixtures, HostingFixtures, HostingHandle, HostingRecord} from "./net/http"
|
||||||
export type {WebLnInfo} from "./app/webln"
|
export type {WebLnInfo} from "./app/webln"
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,9 @@ const HOSTING_ORIGIN = "https://api.hosting.coracle.social"
|
||||||
// Hard-coded in src/app.html, so every navigation asks for it whatever the scenario is doing.
|
// Hard-coded in src/app.html, so every navigation asks for it whatever the scenario is doing.
|
||||||
const PLAUSIBLE_ORIGIN = "https://plausible.coracle.social"
|
const PLAUSIBLE_ORIGIN = "https://plausible.coracle.social"
|
||||||
|
|
||||||
|
// Transcription and speech both go here, against whichever key the user has saved.
|
||||||
|
const OPENROUTER_ORIGIN = "https://openrouter.ai"
|
||||||
|
|
||||||
// Where the hosting api sends a browser to pay. `.test` resolves nowhere and the block-all aborts
|
// Where the hosting api sends a browser to pay. `.test` resolves nowhere and the block-all aborts
|
||||||
// the navigation, so a spec sees the redirect without one leaving.
|
// the navigation, so a spec sees the redirect without one leaving.
|
||||||
const CHECKOUT_ORIGIN = "https://checkout.test"
|
const CHECKOUT_ORIGIN = "https://checkout.test"
|
||||||
|
|
@ -210,6 +213,45 @@ export const mockDufflepud = (context: BrowserContext, fixtures: DufflepudFixtur
|
||||||
return route.fallback()
|
return route.fallback()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Silence as a wav, built rather than inlined so a spec can ask for a length and then assert the
|
||||||
|
// duration the player reads off it. 16 bit mono pcm is the shortest header a browser will decode.
|
||||||
|
const silence = (seconds: number) => {
|
||||||
|
const rate = 8000
|
||||||
|
const bytes = seconds * rate * 2
|
||||||
|
const wav = Buffer.alloc(44 + bytes)
|
||||||
|
|
||||||
|
wav.write("RIFF", 0)
|
||||||
|
wav.writeUInt32LE(36 + bytes, 4)
|
||||||
|
wav.write("WAVEfmt ", 8)
|
||||||
|
wav.writeUInt32LE(16, 16)
|
||||||
|
wav.writeUInt16LE(1, 20)
|
||||||
|
wav.writeUInt16LE(1, 22)
|
||||||
|
wav.writeUInt32LE(rate, 24)
|
||||||
|
wav.writeUInt32LE(rate * 2, 28)
|
||||||
|
wav.writeUInt16LE(2, 32)
|
||||||
|
wav.writeUInt16LE(16, 34)
|
||||||
|
wav.write("data", 36)
|
||||||
|
wav.writeUInt32LE(bytes, 40)
|
||||||
|
|
||||||
|
return wav
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OpenRouter's text to speech, answering every request with the same silence. The array it returns
|
||||||
|
* collects what the app asked to have read, in the order it asked.
|
||||||
|
*/
|
||||||
|
export const mockOpenRouterSpeech = async (context: BrowserContext, seconds = 3) => {
|
||||||
|
const spoken: string[] = []
|
||||||
|
|
||||||
|
await context.route(`${OPENROUTER_ORIGIN}/api/v1/audio/speech`, route => {
|
||||||
|
spoken.push(JSON.parse(route.request().postData() ?? "{}").input)
|
||||||
|
|
||||||
|
return route.fulfill({contentType: "audio/wav", body: silence(seconds)})
|
||||||
|
})
|
||||||
|
|
||||||
|
return spoken
|
||||||
|
}
|
||||||
|
|
||||||
export type BlossomOptions = {
|
export type BlossomOptions = {
|
||||||
// The blossom server the scenario expects an upload to land on, e.g. a space's own url.
|
// The blossom server the scenario expects an upload to land on, e.g. a space's own url.
|
||||||
server: string
|
server: string
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,16 @@ import {DAY, HOUR, MINUTE, WEEK, bech32ToHex} from "@welshman/lib"
|
||||||
import {getLnUrl} from "@welshman/util"
|
import {getLnUrl} from "@welshman/util"
|
||||||
import {MessagingRelayList, Profile, RelayList, displayPubkey} from "@welshman/domain"
|
import {MessagingRelayList, Profile, RelayList, displayPubkey} from "@welshman/domain"
|
||||||
import type {Locator, Page} from "@playwright/test"
|
import type {Locator, Page} from "@playwright/test"
|
||||||
import {expect, makeTestUser, mockDufflepud, roomPath, spacePath, test, users} from "../harness"
|
import {
|
||||||
|
expect,
|
||||||
|
makeTestUser,
|
||||||
|
mockDufflepud,
|
||||||
|
mockOpenRouterSpeech,
|
||||||
|
roomPath,
|
||||||
|
spacePath,
|
||||||
|
test,
|
||||||
|
users,
|
||||||
|
} from "../harness"
|
||||||
import type {SeededSpace, TestUser} from "../harness"
|
import type {SeededSpace, TestUser} from "../harness"
|
||||||
|
|
||||||
// A handle to a seeded event. SeededEvent isn't exported from the harness, and only its id and
|
// A handle to a seeded event. SeededEvent isn't exported from the harness, and only its id and
|
||||||
|
|
@ -855,6 +864,70 @@ test("US-028 share a message somewhere else", async ({seed, as}) => {
|
||||||
await expect(message(bob, "heads up")).toContainText("the dock is closed on sunday")
|
await expect(message(bob, "heads up")).toContainText("the dock is closed on sunday")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("US-119 have a message read out loud", async ({seed, as}) => {
|
||||||
|
const scenario = await seed(({relay, user, at}) => {
|
||||||
|
const space = relay("space")
|
||||||
|
|
||||||
|
space.room("general", {name: "General"})
|
||||||
|
space.join(user.alice, "general")
|
||||||
|
space.join(user.bob, "general")
|
||||||
|
space.profile(user.bob, {name: "Bob Barnacle"})
|
||||||
|
space.message(user.bob, "general", "the dock is closed on sunday", at(2, HOUR))
|
||||||
|
|
||||||
|
seedChatter(space, user.alice)
|
||||||
|
})
|
||||||
|
|
||||||
|
const {url} = scenario.space("space")
|
||||||
|
const alice = await as(users.alice, roomPath(url, "general"))
|
||||||
|
const spoken = await mockOpenRouterSpeech(alice.context())
|
||||||
|
|
||||||
|
await expect(message(alice, "the dock is closed on sunday")).toBeVisible()
|
||||||
|
|
||||||
|
// With no key saved, reading a message asks for one the way dictation does.
|
||||||
|
await openMessageMenu(alice, "the dock is closed on sunday")
|
||||||
|
await alice.getByRole("button", {name: "Read Out Loud"}).click()
|
||||||
|
|
||||||
|
const enable = dialog(alice, "Enable read out loud?")
|
||||||
|
|
||||||
|
await enable.locator('input[name="flotilla-openrouter-key"]').fill("sk-or-test")
|
||||||
|
await enable.getByRole("button", {name: "Enable read out loud"}).click()
|
||||||
|
|
||||||
|
await expect(alice.getByRole("alert")).toContainText("Read out loud is ready to use!")
|
||||||
|
|
||||||
|
await openMessageMenu(alice, "the dock is closed on sunday")
|
||||||
|
await alice.getByRole("button", {name: "Read Out Loud"}).click()
|
||||||
|
|
||||||
|
await expect(alice.getByText("a message from Bob Barnacle")).toBeVisible()
|
||||||
|
|
||||||
|
// Only what the message says is sent, so the nostr uri wrapping bob's mention never is.
|
||||||
|
expect(spoken).toEqual(["the dock is closed on sunday"])
|
||||||
|
|
||||||
|
// The duration is the mock's, which is what proves the player is on audio it decoded rather
|
||||||
|
// than on an element that failed to load.
|
||||||
|
await expect(alice.getByText("/ 0:03")).toBeVisible()
|
||||||
|
|
||||||
|
// Chromium decides for itself whether the autoplay is allowed, so the control is read for
|
||||||
|
// which way it is about to flip rather than assumed to start paused.
|
||||||
|
const playPause = alice.getByRole("button", {name: /^(Play|Pause) message$/})
|
||||||
|
const wasPlaying = (await playPause.getAttribute("aria-label")) === "Pause message"
|
||||||
|
|
||||||
|
await playPause.click()
|
||||||
|
await expect(playPause).toHaveAttribute(
|
||||||
|
"aria-label",
|
||||||
|
wasPlaying ? "Play message" : "Pause message",
|
||||||
|
)
|
||||||
|
|
||||||
|
const seek = alice.getByRole("slider", {name: "Seek within the message"})
|
||||||
|
|
||||||
|
await seek.fill("2")
|
||||||
|
|
||||||
|
await expect(alice.getByText("0:02 /")).toBeVisible()
|
||||||
|
|
||||||
|
await alice.getByRole("button", {name: "Stop reading"}).click()
|
||||||
|
|
||||||
|
await expect(alice.getByText("a message from Bob Barnacle")).toHaveCount(0)
|
||||||
|
})
|
||||||
|
|
||||||
test("US-115 connect a wallet without losing the zap you were composing", async ({seed, as}) => {
|
test("US-115 connect a wallet without losing the zap you were composing", async ({seed, as}) => {
|
||||||
// The lightning address on bob's profile, and the lnurl endpoint it resolves to. Zapping only gets
|
// The lightning address on bob's profile, and the lnurl endpoint it resolves to. Zapping only gets
|
||||||
// as far as a dialog once dufflepud answers with a zapper for that endpoint.
|
// as far as a dialog once dufflepud answers with a zapper for that endpoint.
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import {derived, writable} from "svelte/store"
|
import {derived, get, writable} from "svelte/store"
|
||||||
import {first, uniqBy} from "@welshman/lib"
|
import {first, uniqBy} from "@welshman/lib"
|
||||||
import {sortEventsDesc, tagSpec, tagValues} from "@welshman/util"
|
import {sortEventsDesc, tagSpec, tagValues} from "@welshman/util"
|
||||||
import {deriveDeduplicated} from "@welshman/store"
|
import {deriveDeduplicated} from "@welshman/store"
|
||||||
|
|
@ -262,15 +262,32 @@ export const callState = writable<CallState>(CallState.Disconnected)
|
||||||
|
|
||||||
export const callTargetRoom = writable<Room | undefined>(undefined)
|
export const callTargetRoom = writable<Room | undefined>(undefined)
|
||||||
|
|
||||||
|
export const isCallActive = derived(
|
||||||
|
callState,
|
||||||
|
$state => $state === CallState.Joining || $state === CallState.Connected,
|
||||||
|
)
|
||||||
|
|
||||||
export const deriveIsCallActiveElsewhere = (url: string | undefined, h: string | undefined) =>
|
export const deriveIsCallActiveElsewhere = (url: string | undefined, h: string | undefined) =>
|
||||||
derived(
|
derived(
|
||||||
[callState, callTargetRoom],
|
[isCallActive, callTargetRoom],
|
||||||
([$state, $targetRoom]) =>
|
([$isCallActive, $targetRoom]) =>
|
||||||
($state === CallState.Joining || $state === CallState.Connected) &&
|
$isCallActive &&
|
||||||
$targetRoom !== undefined &&
|
$targetRoom !== undefined &&
|
||||||
!($targetRoom.url === url && $targetRoom.h === h),
|
!($targetRoom.url === url && $targetRoom.h === h),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// leaveVoiceRoom no-ops during Joining, since no session exists yet to leave — cancel the
|
||||||
|
// in-flight join instead, otherwise ending a call that is still connecting does nothing.
|
||||||
|
export const endCall = async () => {
|
||||||
|
const engine = await import("@app/callEngine")
|
||||||
|
|
||||||
|
if (get(callState) === CallState.Joining) {
|
||||||
|
engine.cancelJoinVoiceRoom()
|
||||||
|
} else {
|
||||||
|
await engine.leaveVoiceRoom()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export const speakingParticipants = writable<CallParticipant[]>([])
|
export const speakingParticipants = writable<CallParticipant[]>([])
|
||||||
|
|
||||||
export const participantMediaState = writable(new Map<string, ParticipantMediaState>())
|
export const participantMediaState = writable(new Map<string, ParticipantMediaState>())
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
import Landing from "@app/components/Landing.svelte"
|
import Landing from "@app/components/Landing.svelte"
|
||||||
import Toast from "@app/components/Toast.svelte"
|
import Toast from "@app/components/Toast.svelte"
|
||||||
import CallBanner from "@app/components/CallBanner.svelte"
|
import CallBanner from "@app/components/CallBanner.svelte"
|
||||||
|
import SpeechBanner from "@app/components/SpeechBanner.svelte"
|
||||||
import PrimaryNav from "@app/components/PrimaryNav.svelte"
|
import PrimaryNav from "@app/components/PrimaryNav.svelte"
|
||||||
import {app} from "@app/core"
|
import {app} from "@app/core"
|
||||||
import {modal} from "@app/modal"
|
import {modal} from "@app/modal"
|
||||||
|
|
@ -26,5 +27,6 @@
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
<CallBanner />
|
<CallBanner />
|
||||||
|
<SpeechBanner />
|
||||||
</div>
|
</div>
|
||||||
<Toast />
|
<Toast />
|
||||||
|
|
|
||||||
19
src/app/components/Banner.svelte
Normal file
19
src/app/components/Banner.svelte
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
<script lang="ts">
|
||||||
|
import type {Snippet} from "svelte"
|
||||||
|
import {fly} from "svelte/transition"
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
children: Snippet
|
||||||
|
}
|
||||||
|
|
||||||
|
const {children}: Props = $props()
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<!-- Only the bottom-most banner has to clear the mobile nav; one stacked above another sits
|
||||||
|
flush against it instead of leaving the nav's height between them. -->
|
||||||
|
<div
|
||||||
|
in:fly={{y: 60, duration: 250}}
|
||||||
|
out:fly={{y: 60, duration: 200}}
|
||||||
|
class="relative flex shrink-0 items-center gap-3 border-t border-line bg-surface py-2 pl-4 pr-2 mb-[calc(var(--saib)+3.5rem)] md:mb-[var(--saib)] md:pl-6 [&:not(:last-child)]:mb-0">
|
||||||
|
{@render children()}
|
||||||
|
</div>
|
||||||
|
|
@ -1,5 +1,4 @@
|
||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import {fly} from "svelte/transition"
|
|
||||||
import {navigate} from "@app/modal"
|
import {navigate} from "@app/modal"
|
||||||
import {page} from "$app/stores"
|
import {page} from "$app/stores"
|
||||||
import cx from "classnames"
|
import cx from "classnames"
|
||||||
|
|
@ -9,6 +8,7 @@
|
||||||
import EndCall from "@assets/icons/end-call-rounded.svg?dataurl"
|
import EndCall from "@assets/icons/end-call-rounded.svg?dataurl"
|
||||||
import Icon from "@lib/components/Icon.svelte"
|
import Icon from "@lib/components/Icon.svelte"
|
||||||
import Button from "@lib/components/Button.svelte"
|
import Button from "@lib/components/Button.svelte"
|
||||||
|
import Banner from "@app/components/Banner.svelte"
|
||||||
import {decodeRelay} from "@app/relays"
|
import {decodeRelay} from "@app/relays"
|
||||||
import {displayRoom} from "@app/rooms"
|
import {displayRoom} from "@app/rooms"
|
||||||
import {makeRoomPath} from "@app/routes"
|
import {makeRoomPath} from "@app/routes"
|
||||||
|
|
@ -18,6 +18,7 @@
|
||||||
callTargetRoom,
|
callTargetRoom,
|
||||||
callMicMuted,
|
callMicMuted,
|
||||||
deriveIsCallActiveElsewhere,
|
deriveIsCallActiveElsewhere,
|
||||||
|
endCall,
|
||||||
} from "@app/call"
|
} from "@app/call"
|
||||||
|
|
||||||
const {relay, h} = $derived($page.params)
|
const {relay, h} = $derived($page.params)
|
||||||
|
|
@ -45,25 +46,10 @@
|
||||||
|
|
||||||
await engine.toggleMute()
|
await engine.toggleMute()
|
||||||
}
|
}
|
||||||
|
|
||||||
// leaveVoiceRoom no-ops during Joining (no session exists yet to leave) — cancel
|
|
||||||
// the in-flight join instead, otherwise this button silently does nothing.
|
|
||||||
const endCall = async () => {
|
|
||||||
const engine = await import("@app/callEngine")
|
|
||||||
|
|
||||||
if ($callState === CallState.Joining) {
|
|
||||||
engine.cancelJoinVoiceRoom()
|
|
||||||
} else {
|
|
||||||
await engine.leaveVoiceRoom()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
{#if visible}
|
{#if visible}
|
||||||
<div
|
<Banner>
|
||||||
in:fly={{y: 60, duration: 250}}
|
|
||||||
out:fly={{y: 60, duration: 200}}
|
|
||||||
class="relative flex shrink-0 items-center gap-3 border-t border-line bg-surface py-2 pl-4 pr-2 mb-[calc(var(--saib)+3.5rem)] md:mb-[var(--saib)] md:pl-6">
|
|
||||||
<Button
|
<Button
|
||||||
class="flex min-w-0 flex-1 items-center gap-2 text-left"
|
class="flex min-w-0 flex-1 items-center gap-2 text-left"
|
||||||
onclick={goToRoom}
|
onclick={goToRoom}
|
||||||
|
|
@ -103,5 +89,5 @@
|
||||||
<Icon icon={EndCall} size={4} />
|
<Icon icon={EndCall} size={4} />
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</Banner>
|
||||||
{/if}
|
{/if}
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@
|
||||||
import Button from "@lib/components/Button.svelte"
|
import Button from "@lib/components/Button.svelte"
|
||||||
import Spinner from "@lib/components/Spinner.svelte"
|
import Spinner from "@lib/components/Spinner.svelte"
|
||||||
import {errorMessage} from "@lib/util"
|
import {errorMessage} from "@lib/util"
|
||||||
import DictationEnable from "@app/components/DictationEnable.svelte"
|
import OpenRouterEnable from "@app/components/OpenRouterEnable.svelte"
|
||||||
import {startDictation, transcribe} from "@app/dictation"
|
import {startDictation, transcribe} from "@app/dictation"
|
||||||
import {getSetting} from "@app/settings"
|
import {getSetting} from "@app/settings"
|
||||||
import {pushModal} from "@app/modal"
|
import {pushModal} from "@app/modal"
|
||||||
|
|
@ -42,7 +42,10 @@
|
||||||
loading = false
|
loading = false
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
pushModal(DictationEnable)
|
pushModal(OpenRouterEnable, {
|
||||||
|
feature: "Voice input",
|
||||||
|
subtitle: "Dictate your messages instead of typing them.",
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,15 @@
|
||||||
import {publishSettings} from "@app/settings"
|
import {publishSettings} from "@app/settings"
|
||||||
import {pushToast} from "@app/toast"
|
import {pushToast} from "@app/toast"
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
feature: string
|
||||||
|
subtitle: string
|
||||||
|
}
|
||||||
|
|
||||||
|
const {feature, subtitle}: Props = $props()
|
||||||
|
|
||||||
|
const action = `Enable ${feature.toLowerCase()}`
|
||||||
|
|
||||||
const back = () => history.back()
|
const back = () => history.back()
|
||||||
|
|
||||||
const save = async () => {
|
const save = async () => {
|
||||||
|
|
@ -27,7 +36,7 @@
|
||||||
try {
|
try {
|
||||||
await publishSettings({openrouter_key: openrouterKey.trim()})
|
await publishSettings({openrouter_key: openrouterKey.trim()})
|
||||||
|
|
||||||
pushToast({message: "Voice input is ready to use!"})
|
pushToast({message: `${feature} is ready to use!`})
|
||||||
|
|
||||||
back()
|
back()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -45,16 +54,14 @@
|
||||||
<Modal tag="form" onsubmit={preventDefault(save)}>
|
<Modal tag="form" onsubmit={preventDefault(save)}>
|
||||||
<ModalBody>
|
<ModalBody>
|
||||||
<ModalHeader>
|
<ModalHeader>
|
||||||
<ModalTitle>Enable voice input?</ModalTitle>
|
<ModalTitle>{action}?</ModalTitle>
|
||||||
<ModalSubtitle>Dictate your messages instead of typing them.</ModalSubtitle>
|
<ModalSubtitle>{subtitle}</ModalSubtitle>
|
||||||
</ModalHeader>
|
</ModalHeader>
|
||||||
<p>
|
<p>
|
||||||
Your recordings are transcribed by <Link
|
{feature} is powered by <Link external href="https://openrouter.ai" class="text-primary"
|
||||||
external
|
>OpenRouter</Link
|
||||||
href="https://openrouter.ai"
|
>, which charges a fraction of a cent for each use. To turn it on, add some credit to an
|
||||||
class="text-primary">OpenRouter</Link
|
OpenRouter account, then
|
||||||
>, which charges a fraction of a cent per minute of audio. To turn voice input on, add some
|
|
||||||
credit to an OpenRouter account, then
|
|
||||||
<Link external href="https://openrouter.ai/settings/keys" class="text-primary"
|
<Link external href="https://openrouter.ai/settings/keys" class="text-primary"
|
||||||
>create an API key</Link> and paste it below.
|
>create an API key</Link> and paste it below.
|
||||||
</p>
|
</p>
|
||||||
|
|
@ -84,7 +91,7 @@
|
||||||
Go back
|
Go back
|
||||||
</Button>
|
</Button>
|
||||||
<Button type="submit" class="button button-primary" disabled={!openrouterKey || loading}>
|
<Button type="submit" class="button button-primary" disabled={!openrouterKey || loading}>
|
||||||
<Spinner {loading}>Enable voice input</Spinner>
|
<Spinner {loading}>{action}</Spinner>
|
||||||
<Icon icon={AltArrowRight} />
|
<Icon icon={AltArrowRight} />
|
||||||
</Button>
|
</Button>
|
||||||
</ModalFooter>
|
</ModalFooter>
|
||||||
|
|
@ -9,6 +9,7 @@
|
||||||
import Danger from "@assets/icons/danger.svg?dataurl"
|
import Danger from "@assets/icons/danger.svg?dataurl"
|
||||||
import Pin from "@assets/icons/pin.svg?dataurl"
|
import Pin from "@assets/icons/pin.svg?dataurl"
|
||||||
import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl"
|
import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl"
|
||||||
|
import VolumeLoud from "@assets/icons/volume-loud.svg?dataurl"
|
||||||
import Button from "@lib/components/Button.svelte"
|
import Button from "@lib/components/Button.svelte"
|
||||||
import Icon from "@lib/components/Icon.svelte"
|
import Icon from "@lib/components/Icon.svelte"
|
||||||
import Confirm from "@lib/components/Confirm.svelte"
|
import Confirm from "@lib/components/Confirm.svelte"
|
||||||
|
|
@ -21,6 +22,7 @@
|
||||||
import {deriveUserIsSpaceAdmin} from "@app/management"
|
import {deriveUserIsSpaceAdmin} from "@app/management"
|
||||||
import {ROOM, deriveUserIsRoomAdmin} from "@app/rooms"
|
import {ROOM, deriveUserIsRoomAdmin} from "@app/rooms"
|
||||||
import {shareEvent} from "@app/share"
|
import {shareEvent} from "@app/share"
|
||||||
|
import {readAloud} from "@app/speech"
|
||||||
import {pushModal} from "@app/modal"
|
import {pushModal} from "@app/modal"
|
||||||
import {pushToast} from "@app/toast"
|
import {pushToast} from "@app/toast"
|
||||||
|
|
||||||
|
|
@ -48,6 +50,11 @@
|
||||||
pushModal(PinboardSelect, {url, event})
|
pushModal(PinboardSelect, {url, event})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const read = () => {
|
||||||
|
onClick()
|
||||||
|
readAloud(event)
|
||||||
|
}
|
||||||
|
|
||||||
const createThread = () => {
|
const createThread = () => {
|
||||||
onClick()
|
onClick()
|
||||||
pushModal(ThreadCreate, {
|
pushModal(ThreadCreate, {
|
||||||
|
|
@ -129,6 +136,12 @@
|
||||||
Add to Library
|
Add to Library
|
||||||
</Button>
|
</Button>
|
||||||
</li>
|
</li>
|
||||||
|
<li>
|
||||||
|
<Button onclick={read}>
|
||||||
|
<Icon size={4} icon={VolumeLoud} />
|
||||||
|
Read Out Loud
|
||||||
|
</Button>
|
||||||
|
</li>
|
||||||
{#if h}
|
{#if h}
|
||||||
<li>
|
<li>
|
||||||
<Button onclick={createThread}>
|
<Button onclick={createThread}>
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@
|
||||||
import AltArrowUp from "@assets/icons/alt-arrow-up.svg?dataurl"
|
import AltArrowUp from "@assets/icons/alt-arrow-up.svg?dataurl"
|
||||||
import Pin from "@assets/icons/pin.svg?dataurl"
|
import Pin from "@assets/icons/pin.svg?dataurl"
|
||||||
import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl"
|
import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl"
|
||||||
|
import VolumeLoud from "@assets/icons/volume-loud.svg?dataurl"
|
||||||
import {slideAndFade} from "@lib/transition"
|
import {slideAndFade} from "@lib/transition"
|
||||||
import Button from "@lib/components/Button.svelte"
|
import Button from "@lib/components/Button.svelte"
|
||||||
import Link from "@lib/components/Link.svelte"
|
import Link from "@lib/components/Link.svelte"
|
||||||
|
|
@ -31,6 +32,7 @@
|
||||||
import {ENABLE_ZAPS} from "@app/env"
|
import {ENABLE_ZAPS} from "@app/env"
|
||||||
import {makeContentPath} from "@app/routes"
|
import {makeContentPath} from "@app/routes"
|
||||||
import {shareEvent} from "@app/share"
|
import {shareEvent} from "@app/share"
|
||||||
|
import {readAloud} from "@app/speech"
|
||||||
import {pushModal} from "@app/modal"
|
import {pushModal} from "@app/modal"
|
||||||
import {pushToast} from "@app/toast"
|
import {pushToast} from "@app/toast"
|
||||||
|
|
||||||
|
|
@ -88,6 +90,11 @@
|
||||||
shareEvent(url, "Message", event)
|
shareEvent(url, "Message", event)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const read = () => {
|
||||||
|
history.back()
|
||||||
|
readAloud(event)
|
||||||
|
}
|
||||||
|
|
||||||
const showInfo = () => pushModal(EventInfo, {url, event}, {replaceState: true})
|
const showInfo = () => pushModal(EventInfo, {url, event}, {replaceState: true})
|
||||||
|
|
||||||
const showDelete = () => pushModal(EventDeleteConfirm, {url, event})
|
const showDelete = () => pushModal(EventDeleteConfirm, {url, event})
|
||||||
|
|
@ -153,6 +160,10 @@
|
||||||
<Icon size={4} icon={ShareCircle} />
|
<Icon size={4} icon={ShareCircle} />
|
||||||
Share
|
Share
|
||||||
</Button>
|
</Button>
|
||||||
|
<Button class="button button-neutral w-full" onclick={read}>
|
||||||
|
<Icon size={4} icon={VolumeLoud} />
|
||||||
|
Read Out Loud
|
||||||
|
</Button>
|
||||||
{#if h}
|
{#if h}
|
||||||
<Button class="button button-neutral w-full" onclick={createThread}>
|
<Button class="button button-neutral w-full" onclick={createThread}>
|
||||||
<Icon size={4} icon={NotesMinimalistic} />
|
<Icon size={4} icon={NotesMinimalistic} />
|
||||||
|
|
|
||||||
84
src/app/components/SpeechBanner.svelte
Normal file
84
src/app/components/SpeechBanner.svelte
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
<script lang="ts">
|
||||||
|
import Play from "@assets/icons/play.svg?dataurl"
|
||||||
|
import Pause from "@assets/icons/pause.svg?dataurl"
|
||||||
|
import Close from "@assets/icons/close.svg?dataurl"
|
||||||
|
import VolumeLoud from "@assets/icons/volume-loud.svg?dataurl"
|
||||||
|
import Icon from "@lib/components/Icon.svelte"
|
||||||
|
import Button from "@lib/components/Button.svelte"
|
||||||
|
import Spinner from "@lib/components/Spinner.svelte"
|
||||||
|
import Banner from "@app/components/Banner.svelte"
|
||||||
|
import {speech, stopSpeech} from "@app/speech"
|
||||||
|
|
||||||
|
let audio: HTMLAudioElement
|
||||||
|
let playing = $state(false)
|
||||||
|
let currentTime = $state(0)
|
||||||
|
let duration = $state(0)
|
||||||
|
|
||||||
|
// Duration is NaN until the browser has read the metadata, so the scrubber stays at zero
|
||||||
|
// length rather than rendering a range with no end.
|
||||||
|
const scrubbable = $derived(Number.isFinite(duration) ? duration : 0)
|
||||||
|
|
||||||
|
const toggle = () => (playing ? audio.pause() : audio.play())
|
||||||
|
|
||||||
|
const scrub = (event: Event & {currentTarget: HTMLInputElement}) => {
|
||||||
|
audio.currentTime = Number(event.currentTarget.value)
|
||||||
|
currentTime = audio.currentTime
|
||||||
|
}
|
||||||
|
|
||||||
|
const display = (seconds: number) => {
|
||||||
|
const whole = Math.floor(seconds)
|
||||||
|
|
||||||
|
return `${Math.floor(whole / 60)}:${String(whole % 60).padStart(2, "0")}`
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
{#if $speech}
|
||||||
|
<Banner>
|
||||||
|
{#if $speech.src}
|
||||||
|
<audio
|
||||||
|
bind:this={audio}
|
||||||
|
src={$speech.src}
|
||||||
|
autoplay
|
||||||
|
onplay={() => (playing = true)}
|
||||||
|
onpause={() => (playing = false)}
|
||||||
|
onended={() => (playing = false)}
|
||||||
|
ondurationchange={() => (duration = audio.duration)}
|
||||||
|
ontimeupdate={() => (currentTime = audio.currentTime)}></audio>
|
||||||
|
<Button
|
||||||
|
aria-label={playing ? "Pause message" : "Play message"}
|
||||||
|
class="button button-circle button-sm button-primary shrink-0"
|
||||||
|
onclick={toggle}>
|
||||||
|
<Icon icon={playing ? Pause : Play} size={4} />
|
||||||
|
</Button>
|
||||||
|
{:else}
|
||||||
|
<Spinner size="xs" />
|
||||||
|
{/if}
|
||||||
|
<div class="flex min-w-0 flex-1 flex-col gap-1">
|
||||||
|
<span class="flex min-w-0 items-center gap-2 truncate text-sm">
|
||||||
|
<Icon icon={VolumeLoud} size={4} />
|
||||||
|
<span class="font-semibold">{$speech.src ? "Reading" : "Preparing"}</span>
|
||||||
|
<span class="truncate text-content-muted">a message from {$speech.title}</span>
|
||||||
|
</span>
|
||||||
|
{#if $speech.src}
|
||||||
|
<span class="flex items-center gap-2 text-xs text-content-muted">
|
||||||
|
<input
|
||||||
|
aria-label="Seek within the message"
|
||||||
|
class="range min-w-0 flex-1"
|
||||||
|
type="range"
|
||||||
|
min="0"
|
||||||
|
max={scrubbable}
|
||||||
|
step="0.1"
|
||||||
|
value={currentTime}
|
||||||
|
oninput={scrub} />
|
||||||
|
<span class="whitespace-nowrap">{display(currentTime)} / {display(scrubbable)}</span>
|
||||||
|
</span>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
<Button
|
||||||
|
aria-label="Stop reading"
|
||||||
|
class="button button-circle button-sm button-neutral shrink-0"
|
||||||
|
onclick={stopSpeech}>
|
||||||
|
<Icon icon={Close} size={4} />
|
||||||
|
</Button>
|
||||||
|
</Banner>
|
||||||
|
{/if}
|
||||||
107
src/app/speech.ts
Normal file
107
src/app/speech.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
||||||
|
import {get, writable} from "svelte/store"
|
||||||
|
import {parse, renderAsText} from "@welshman/content"
|
||||||
|
import type {Maybe} from "@welshman/lib"
|
||||||
|
import type {TrustedEvent} from "@welshman/util"
|
||||||
|
import {errorMessage} from "@lib/util"
|
||||||
|
import Confirm from "@lib/components/Confirm.svelte"
|
||||||
|
import OpenRouterEnable from "@app/components/OpenRouterEnable.svelte"
|
||||||
|
import {endCall, isCallActive} from "@app/call"
|
||||||
|
import {profiles} from "@app/core"
|
||||||
|
import {pushModal} from "@app/modal"
|
||||||
|
import {getSetting} from "@app/settings"
|
||||||
|
import {pushToast} from "@app/toast"
|
||||||
|
|
||||||
|
const SPEECH_MODEL = "openai/gpt-audio-mini"
|
||||||
|
|
||||||
|
const SPEECH_VOICE = "alloy"
|
||||||
|
|
||||||
|
export type Speech = {
|
||||||
|
id: string
|
||||||
|
title: string
|
||||||
|
src?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export const speech = writable<Maybe<Speech>>(undefined)
|
||||||
|
|
||||||
|
export const synthesize = async (text: string) => {
|
||||||
|
const response = await fetch("https://openrouter.ai/api/v1/audio/speech", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${getSetting("openrouter_key")}`,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({model: SPEECH_MODEL, voice: SPEECH_VOICE, input: text}),
|
||||||
|
})
|
||||||
|
|
||||||
|
// A successful response is audio rather than json, so the error body is only worth reading once
|
||||||
|
// the status says there is one.
|
||||||
|
if (!response.ok) {
|
||||||
|
const {error} = await response.json().catch(() => ({error: undefined}))
|
||||||
|
|
||||||
|
throw new Error(error?.message || `OpenRouter returned a ${response.status}.`)
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.blob()
|
||||||
|
}
|
||||||
|
|
||||||
|
export const stopSpeech = () =>
|
||||||
|
speech.update(current => {
|
||||||
|
if (current?.src) {
|
||||||
|
URL.revokeObjectURL(current.src)
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined
|
||||||
|
})
|
||||||
|
|
||||||
|
const play = async (event: TrustedEvent, text: string) => {
|
||||||
|
const {id, pubkey} = event
|
||||||
|
const title = profiles.get().display(pubkey).get()
|
||||||
|
|
||||||
|
stopSpeech()
|
||||||
|
speech.set({id, title})
|
||||||
|
|
||||||
|
try {
|
||||||
|
const src = URL.createObjectURL(await synthesize(text))
|
||||||
|
|
||||||
|
// Reading a second message while this one was in flight hands the player to that one.
|
||||||
|
if (get(speech)?.id === id) {
|
||||||
|
speech.set({id, title, src})
|
||||||
|
} else {
|
||||||
|
URL.revokeObjectURL(src)
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error)
|
||||||
|
pushToast({theme: "error", message: `Failed to read this message: ${errorMessage(error)}`})
|
||||||
|
|
||||||
|
if (get(speech)?.id === id) {
|
||||||
|
stopSpeech()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const readAloud = (event: TrustedEvent) => {
|
||||||
|
const text = renderAsText(parse(event)).toString().trim()
|
||||||
|
|
||||||
|
if (getSetting("openrouter_key")) {
|
||||||
|
if (get(isCallActive)) {
|
||||||
|
pushModal(Confirm, {
|
||||||
|
title: "Leave the call?",
|
||||||
|
message: "Reading a message out loud would talk over the call you are in.",
|
||||||
|
confirm: async () => {
|
||||||
|
await endCall()
|
||||||
|
|
||||||
|
history.back()
|
||||||
|
|
||||||
|
await play(event, text)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
play(event, text)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
pushModal(OpenRouterEnable, {
|
||||||
|
feature: "Read out loud",
|
||||||
|
subtitle: "Have a message read to you instead of reading it yourself.",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -142,7 +142,7 @@
|
||||||
</Field>
|
</Field>
|
||||||
<Field>
|
<Field>
|
||||||
{#snippet label()}
|
{#snippet label()}
|
||||||
<p>Voice Input</p>
|
<p>OpenRouter</p>
|
||||||
{/snippet}
|
{/snippet}
|
||||||
{#snippet input()}
|
{#snippet input()}
|
||||||
<label class="input flex w-full items-center gap-2">
|
<label class="input flex w-full items-center gap-2">
|
||||||
|
|
@ -159,7 +159,8 @@
|
||||||
{#snippet info()}
|
{#snippet info()}
|
||||||
<p>
|
<p>
|
||||||
Add an <Link external href="https://openrouter.ai/settings/keys" class="text-primary"
|
Add an <Link external href="https://openrouter.ai/settings/keys" class="text-primary"
|
||||||
>OpenRouter API key</Link> to dictate messages using the microphone button in your composer.
|
>OpenRouter API key</Link> to dictate messages using the microphone button in your composer,
|
||||||
|
and to have messages read out loud to you.
|
||||||
</p>
|
</p>
|
||||||
{/snippet}
|
{/snippet}
|
||||||
</Field>
|
</Field>
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue