matt/flotilla
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Add voice dictation

Browse source
This commit is contained in:
Jon Staab 2026-08-31 21:56:37 -07:00
parent 19f6516a4b
commit 0ce9084e1d
11 changed files with 422 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
src/app/components/ChatCompose.svelte
View file

@ -4,12 +4,13 @@
import cx from "classnames"
import type {MaybeAsync} from "@welshman/lib"
import type {EventContent} from "@welshman/util"
import {isMobile, preventDefault} from "@lib/html"
import {escapeHtml, isMobile, preventDefault} from "@lib/html"
import GallerySend from "@assets/icons/gallery-send.svg?dataurl"
import Plane from "@assets/icons/plane-2.svg?dataurl"
import Icon from "@lib/components/Icon.svelte"
import Button from "@lib/components/Button.svelte"
import Spinner from "@lib/components/Spinner.svelte"
import DictationButton from "@app/components/DictationButton.svelte"
import EditorContent from "@app/editor/EditorContent.svelte"
import {makeEditor} from "@app/editor"
import {type DraftKey, type Draft} from "@app/drafts"
@ -61,6 +62,14 @@
const uploadFiles = () => editor.then(ed => ed.chain().selectFiles().run())
// Tiptap parses a string handed to insertContent as html, so an angle bracket in the
// transcript would eat the rest of the sentence.
const insertTranscript = async (text: string) => {
const ed = await editor
ed.chain().focus().insertContent(escapeHtml(text)).run()
}
const submit = async () => {
if ($uploading || disabled) return
@ -79,19 +88,23 @@
}
draftKey?.clear()
ed.chain().clearContent().run()
ed.chain().clearContent(true).run()
}
let content = $state(
initialValues?.type === "text" ? initialValues.value : (draftKey?.get()?.content ?? ""),
)
let recording = $state(false)
const onChange = (json: object) => {
content = json
}
const empty = writable(true)
const editor = makeEditor({
content,
empty,
submit,
uploading,
onChange,
@ -100,7 +113,11 @@
})
$effect(() => {
draftKey?.set({content})
if ($empty) {
draftKey?.clear()
} else {
draftKey?.set({content})
}
})
onMount(async () => {
@ -135,11 +152,15 @@
<div class={editorClass} aria-disabled={disabled}>
<EditorContent {autofocus} {editor} />
</div>
<Button
data-tip="{window.navigator.platform.includes('Mac') ? 'cmd' : 'ctrl'}+enter to send"
class="button button-primary button-circle tip tip-left h-10 w-10 min-w-10"
disabled={$uploading || disabled}
onclick={submit}>
<Icon icon={Plane} />
</Button>
{#if recording || ($empty && !disabled)}
<DictationButton bind:recording onTranscript={insertTranscript} />
{:else}
<Button
data-tip="{window.navigator.platform.includes('Mac') ? 'cmd' : 'ctrl'}+enter to send"
class="button button-primary button-circle tip tip-left h-10 w-10 min-w-10"
disabled={$uploading || disabled}
onclick={submit}>
<Icon icon={Plane} />
</Button>
{/if}
</form>

9
src/app/components/CommentCompose.svelte
View file

@ -75,17 +75,24 @@
content = json
}
const empty = writable(true)
const editor = makeEditor({
url,
submit,
uploading,
onChange,
content,
empty,
placeholder: parent ? "Write a reply..." : "Write a comment...",
})
$effect(() => {
draftKey.set({content})
if ($empty) {
draftKey.clear()
} else {
draftKey.set({content})
}
})
</script>

105
src/app/components/DictationButton.svelte Normal file
View file

@ -0,0 +1,105 @@
<script lang="ts">
import {onDestroy} from "svelte"
import cx from "classnames"
import type {Maybe, MaybeAsync} from "@welshman/lib"
import Microphone from "@assets/icons/microphone.svg?dataurl"
import Stop from "@assets/icons/record.svg?dataurl"
import Icon from "@lib/components/Icon.svelte"
import Button from "@lib/components/Button.svelte"
import Spinner from "@lib/components/Spinner.svelte"
import {errorMessage} from "@lib/util"
import DictationEnable from "@app/components/DictationEnable.svelte"
import {startDictation, transcribe} from "@app/dictation"
import {getSetting} from "@app/settings"
import {pushModal} from "@app/modal"
import {pushToast} from "@app/toast"
type Props = {
recording?: boolean
onTranscript: (text: string) => MaybeAsync<void>
}
let {recording = $bindable(false), onTranscript}: Props = $props()
// Room noise idles just below this, so the pulse follows quiet speech too.
const onLevel = (level: number) => {
loud = level > 0.03
}
const start = async () => {
if (getSetting("openrouter_key")) {
// Granting microphone access can sit on a permission prompt for a while, so hold the button
// until it resolves — a second click would open a stream nothing is left holding on to.
loading = true
try {
finish = await startDictation(onLevel)
recording = true
} catch (error) {
console.error(error)
pushToast({theme: "error", message: "Failed to access your microphone."})
} finally {
loading = false
}
} else {
pushModal(DictationEnable)
}
}
const stop = async () => {
if (finish) {
const audio = finish()
finish = undefined
recording = false
loading = true
loud = false
try {
const text = await transcribe(await audio)
await onTranscript(text)
} catch (error) {
console.error(error)
pushToast({theme: "error", message: `Failed to transcribe: ${errorMessage(error)}`})
} finally {
loading = false
}
}
}
const toggle = () => (recording ? stop() : start())
// Held outside of state because only the recording flag drives the markup, and clearing this
// before the recorder has finished flushing is what keeps a second stop from re-entering.
let finish: Maybe<() => Promise<Blob>>
let loading = $state(false)
let loud = $state(false)
const buttonClass = $derived(
cx(
"button button-circle tip tip-left h-10 w-10 min-w-10",
recording ? "button-error" : "button-primary",
{"pulse-scale": loud},
),
)
onDestroy(() => {
finish?.()
})
</script>
<Button
class={buttonClass}
data-tip={recording ? "Stop and transcribe" : "Record a message"}
aria-label={recording ? "Stop recording" : "Start dictation"}
disabled={loading}
onclick={toggle}>
{#if loading}
<Spinner size="xs" />
{:else if recording}
<Icon icon={Stop} />
{:else}
<Icon icon={Microphone} />
{/if}
</Button>

91
src/app/components/DictationEnable.svelte Normal file
View file

@ -0,0 +1,91 @@
<script lang="ts">
import {preventDefault} from "@lib/html"
import AltArrowLeft from "@assets/icons/alt-arrow-left.svg?dataurl"
import AltArrowRight from "@assets/icons/alt-arrow-right.svg?dataurl"
import Lock from "@assets/icons/lock-keyhole.svg?dataurl"
import Icon from "@lib/components/Icon.svelte"
import Link from "@lib/components/Link.svelte"
import Button from "@lib/components/Button.svelte"
import Spinner from "@lib/components/Spinner.svelte"
import {errorMessage} from "@lib/util"
import Field from "@lib/components/Field.svelte"
import Modal from "@lib/components/Modal.svelte"
import ModalBody from "@lib/components/ModalBody.svelte"
import ModalHeader from "@lib/components/ModalHeader.svelte"
import ModalTitle from "@lib/components/ModalTitle.svelte"
import ModalSubtitle from "@lib/components/ModalSubtitle.svelte"
import ModalFooter from "@lib/components/ModalFooter.svelte"
import {PLATFORM_NAME} from "@app/env"
import {publishSettings} from "@app/settings"
import {pushToast} from "@app/toast"
const back = () => history.back()
const save = async () => {
loading = true
try {
await publishSettings({openrouter_key: openrouterKey.trim()})
pushToast({message: "Voice input is ready to use!"})
back()
} catch (error) {
console.error(error)
pushToast({theme: "error", message: `Failed to save your key: ${errorMessage(error)}`})
} finally {
loading = false
}
}
let openrouterKey = $state("")
let loading = $state(false)
</script>
<Modal tag="form" onsubmit={preventDefault(save)}>
<ModalBody>
<ModalHeader>
<ModalTitle>Enable voice input?</ModalTitle>
<ModalSubtitle>Dictate your messages instead of typing them.</ModalSubtitle>
</ModalHeader>
<p>
Your recordings are transcribed by <Link
external
href="https://openrouter.ai"
class="text-primary">OpenRouter</Link
>, which charges a fraction of a cent per minute of audio. To turn voice input on, add some
credit to an OpenRouter account, then
<Link external href="https://openrouter.ai/settings/keys" class="text-primary"
>create an API key</Link> and paste it below.
</p>
<Field>
{#snippet label()}
OpenRouter API Key
{/snippet}
{#snippet input()}
<label class="input flex w-full items-center gap-2">
<Icon icon={Lock} />
<input
bind:value={openrouterKey}
autocomplete="off"
name="flotilla-openrouter-key"
class="grow"
type="password" />
</label>
{/snippet}
{#snippet info()}
Your key is stored in your {PLATFORM_NAME} settings, encrypted to your nostr identity.
{/snippet}
</Field>
</ModalBody>
<ModalFooter>
<Button class="button button-link" onclick={back}>
<Icon icon={AltArrowLeft} />
Go back
</Button>
<Button type="submit" class="button button-primary" disabled={!openrouterKey || loading}>
<Spinner {loading}>Enable voice input</Spinner>
<Icon icon={AltArrowRight} />
</Button>
</ModalFooter>
</Modal>

10
src/app/components/EventReply.svelte
View file

@ -78,10 +78,16 @@
content = json
}
const editor = makeEditor({url, submit, uploading, content, onChange})
const empty = writable(true)
const editor = makeEditor({url, submit, uploading, content, onChange, empty})
$effect(() => {
draftKey.set({content})
if ($empty) {
draftKey.clear()
} else {
draftKey.set({content})
}
})
onMount(() => {

41
src/app/components/RoomCompose.svelte
View file

@ -2,7 +2,7 @@
import {writable} from "svelte/store"
import type {Maybe} from "@welshman/lib"
import type {EventContent} from "@welshman/util"
import {isMobile, preventDefault} from "@lib/html"
import {escapeHtml, isMobile, preventDefault} from "@lib/html"
import GallerySend from "@assets/icons/gallery-send.svg?dataurl"
import WidgetAdd from "@assets/icons/widget-add.svg?dataurl"
import Plane from "@assets/icons/plane-2.svg?dataurl"
@ -12,6 +12,7 @@
import type {TippyController} from "@lib/components/Tippy.svelte"
import Spinner from "@lib/components/Spinner.svelte"
import ComposeMenu from "@app/components/ComposeMenu.svelte"
import DictationButton from "@app/components/DictationButton.svelte"
import EditorContent from "@app/editor/EditorContent.svelte"
import {makeEditor} from "@app/editor"
import {DraftKey, type Draft} from "@app/drafts"
@ -53,6 +54,14 @@
const uploadFiles = () => editor.then(ed => ed.chain().selectFiles().run())
// Tiptap parses a string handed to insertContent as html, so an angle bracket in the
// transcript would eat the rest of the sentence.
const insertTranscript = async (text: string) => {
const ed = await editor
ed.chain().focus().insertContent(escapeHtml(text)).run()
}
const showPopover = () => tippy?.show()
const hidePopover = () => tippy?.hide()
@ -67,21 +76,25 @@
onSubmit({content, tags})
draftKey?.clear()
ed.chain().clearContent().run()
ed.chain().clearContent(true).run()
}
let tippy: Maybe<TippyController> = $state()
let content = $state(
initialValues?.type === "text" ? initialValues.value : (draftKey?.get()?.content ?? ""),
)
let recording = $state(false)
const onChange = (json: object) => {
content = json
}
const empty = writable(true)
const editor = makeEditor({
url,
content,
empty,
submit,
uploading,
onChange,
@ -89,7 +102,11 @@
})
$effect(() => {
draftKey?.set({content})
if ($empty) {
draftKey?.clear()
} else {
draftKey?.set({content})
}
})
onMount(async () => {
@ -137,11 +154,15 @@
<div class="chat-editor grow overflow-hidden">
<EditorContent {autofocus} {editor} />
</div>
<Button
data-tip="{window.navigator.platform.includes('Mac') ? 'cmd' : 'ctrl'}+enter to send"
class="button button-primary button-circle flex justify-center items-center tip tip-left h-10 w-10 min-w-10"
disabled={$uploading}
onclick={submit}>
<Icon icon={Plane} />
</Button>
{#if recording || $empty}
<DictationButton bind:recording onTranscript={insertTranscript} />
{:else}
<Button
data-tip="{window.navigator.platform.includes('Mac') ? 'cmd' : 'ctrl'}+enter to send"
class="button button-primary button-circle flex justify-center items-center tip tip-left h-10 w-10 min-w-10"
disabled={$uploading}
onclick={submit}>
<Icon icon={Plane} />
</Button>
{/if}
</form>

92
src/app/dictation.ts Normal file
View file

@ -0,0 +1,92 @@
import {getSetting} from "@app/settings"
const TRANSCRIPTION_MODEL = "openai/whisper-large-v3-turbo"
// OpenRouter chooses a decoder using the uploaded file's extension, and MediaRecorder's output
// format varies by engine — webm on chromium, mp4 on webkit.
const EXTENSIONS_BY_MIME_TYPE: Record<string, string> = {
"audio/webm": "webm",
"audio/ogg": "ogg",
"audio/mp4": "m4a",
"audio/mpeg": "mp3",
"audio/wav": "wav",
}
// Reports how loud the microphone is once per frame, so the caller can show the speaker that we're
// hearing them. Levels follow the waveform's envelope — jumping to each peak, then decaying — since
// the raw root mean square drops to nothing in the gaps between words.
export const startDictation = async (onLevel: (level: number) => void) => {
const stream = await navigator.mediaDevices.getUserMedia({audio: true})
const recorder = new MediaRecorder(stream)
const chunks: Blob[] = []
recorder.addEventListener("dataavailable", event => chunks.push(event.data))
recorder.start()
const context = new AudioContext()
const analyser = context.createAnalyser()
analyser.fftSize = 512
context.createMediaStreamSource(stream).connect(analyser)
const samples = new Uint8Array(analyser.frequencyBinCount)
let level = 0
const measure = () => {
analyser.getByteTimeDomainData(samples)
let sumSquares = 0
for (const sample of samples) {
sumSquares += ((sample - 128) / 128) ** 2
}
level = Math.max(Math.sqrt(sumSquares / samples.length), level * 0.92)
onLevel(level)
frame = requestAnimationFrame(measure)
}
let frame = requestAnimationFrame(measure)
return () =>
new Promise<Blob>(resolve => {
recorder.addEventListener("stop", () => {
cancelAnimationFrame(frame)
context.close()
for (const track of stream.getTracks()) {
track.stop()
}
resolve(new Blob(chunks, {type: recorder.mimeType}))
})
recorder.stop()
})
}
export const transcribe = async (audio: Blob) => {
const [mimeType] = audio.type.split(";")
const body = new FormData()
body.append("model", TRANSCRIPTION_MODEL)
body.append("file", audio, `dictation.${EXTENSIONS_BY_MIME_TYPE[mimeType] || "webm"}`)
const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", {
method: "POST",
headers: {Authorization: `Bearer ${getSetting("openrouter_key")}`},
body,
})
const {text, error}: {text?: string; error?: {message?: string}} = await response.json()
if (!response.ok) {
throw new Error(error?.message || `OpenRouter returned a ${response.status}.`)
}
return text?.trim() ?? ""
}

11
src/app/editor/index.ts
View file

@ -25,11 +25,14 @@ import {userSpaceUrls} from "@app/rooms"
import {PLATFORM_RELAYS} from "@app/env"
import {pushToast} from "@app/toast"
const isEmpty = (editor: Editor) => editor.getText({blockSeparator: "\n"}).trim() === ""
export const makeEditor = async ({
encryptFiles = false,
aggressive = false,
charCount,
content = "",
empty,
onChange,
placeholder = "",
url,
@ -41,6 +44,7 @@ export const makeEditor = async ({
aggressive?: boolean
charCount?: Writable<number>
content?: string | object
empty?: Writable<boolean>
onChange?: (json: object) => void
placeholder?: string
url?: string
@ -192,9 +196,16 @@ export const makeEditor = async ({
onUpdate({editor}) {
wordCount?.set(editor.storage.wordCount.words)
charCount?.set(editor.storage.wordCount.chars)
empty?.set(isEmpty(editor))
onChange?.(editor.getJSON())
},
})
// Seed the caller's store from the document tiptap actually parsed — a restored draft is a
// document even when it holds no text, so the caller can't tell from `content` alone. Callers
// clear their draft when this reads true, so it has to be set before they render: keep every
// await in this function inside a callback, below the constructor.
empty?.set(isEmpty(ed))
return ed
}

2
src/app/settings.ts
View file

@ -32,6 +32,7 @@ export type SettingsValues = {
font_size: number
alerts: SpaceNotificationSettings[]
zap_amounts: number[]
openrouter_key: string
}
export const defaultSettings: SettingsValues = {
@ -44,6 +45,7 @@ export const defaultSettings: SettingsValues = {
font_size: 1.1,
alerts: [],
zap_amounts: [21, 210, 2100, 21000],
openrouter_key: "",
}
export class Settings extends DerivedPlugin<AppDataReader> {

18
src/lib/components/base.css
View file

@ -120,6 +120,24 @@
animation: none;
}
}
/* Breathes a control in and out while it's listening to the user's voice. */
@utility pulse-scale {
animation: pulse-scale 200ms ease-in-out infinite;
}
@keyframes pulse-scale {
0%,
100% {
transform: scale(1);
}
50% {
transform: scale(1.1);
}
}
@media (prefers-reduced-motion: reduce) {
.pulse-scale {
animation: none;
}
}
@utility pt-sai {
padding-top: var(--sait);
}

25
src/routes/settings/content/+page.svelte
View file

@ -2,11 +2,13 @@
import {BlossomServerLists, MuteLists, publish} from "@welshman/app"
import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl"
import AddCircle from "@assets/icons/add-circle.svg?dataurl"
import Microphone from "@assets/icons/microphone.svg?dataurl"
import {preventDefault} from "@lib/html"
import Field from "@lib/components/Field.svelte"
import FieldInline from "@lib/components/FieldInline.svelte"
import ToggleInput from "@lib/components/ToggleInput.svelte"
import InputList from "@lib/components/InputList.svelte"
import Link from "@lib/components/Link.svelte"
import Icon from "@lib/components/Icon.svelte"
import Card from "@lib/components/Card.svelte"
import Button from "@lib/components/Button.svelte"
@ -119,6 +121,29 @@
</p>
{/snippet}
</Field>
<Field>
{#snippet label()}
<p>Voice Input</p>
{/snippet}
{#snippet input()}
<label class="input flex w-full items-center gap-2">
<Icon icon={Microphone} />
<input
bind:value={$settings.openrouter_key}
autocomplete="off"
name="flotilla-openrouter-key"
placeholder="OpenRouter API key"
class="grow"
type="password" />
</label>
{/snippet}
{#snippet info()}
<p>
Add an <Link external href="https://openrouter.ai/settings/keys" class="text-primary"
>OpenRouter API key</Link> to dictate messages using the microphone button in your composer.
</p>
{/snippet}
</Field>
<Field>
{#snippet label()}
<p>Media Server</p>