From 0ce9084e1dcc3dddeedf89e2d1d0fb3922161b05 Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Mon, 31 Aug 2026 21:56:37 -0700 Subject: [PATCH] Add voice dictation --- src/app/components/ChatCompose.svelte | 41 ++++++--- src/app/components/CommentCompose.svelte | 9 +- src/app/components/DictationButton.svelte | 105 ++++++++++++++++++++++ src/app/components/DictationEnable.svelte | 91 +++++++++++++++++++ src/app/components/EventReply.svelte | 10 ++- src/app/components/RoomCompose.svelte | 41 ++++++--- src/app/dictation.ts | 92 +++++++++++++++++++ src/app/editor/index.ts | 11 +++ src/app/settings.ts | 2 + src/lib/components/base.css | 18 ++++ src/routes/settings/content/+page.svelte | 25 ++++++ 11 files changed, 422 insertions(+), 23 deletions(-) create mode 100644 src/app/components/DictationButton.svelte create mode 100644 src/app/components/DictationEnable.svelte create mode 100644 src/app/dictation.ts diff --git a/src/app/components/ChatCompose.svelte b/src/app/components/ChatCompose.svelte index be576a28..3f377f9e 100644 --- a/src/app/components/ChatCompose.svelte +++ b/src/app/components/ChatCompose.svelte @@ -4,12 +4,13 @@ import cx from "classnames" import type {MaybeAsync} from "@welshman/lib" import type {EventContent} from "@welshman/util" - import {isMobile, preventDefault} from "@lib/html" + import {escapeHtml, isMobile, preventDefault} from "@lib/html" import GallerySend from "@assets/icons/gallery-send.svg?dataurl" import Plane from "@assets/icons/plane-2.svg?dataurl" import Icon from "@lib/components/Icon.svelte" import Button from "@lib/components/Button.svelte" import Spinner from "@lib/components/Spinner.svelte" + import DictationButton from "@app/components/DictationButton.svelte" import EditorContent from "@app/editor/EditorContent.svelte" import {makeEditor} from "@app/editor" import {type DraftKey, type Draft} from "@app/drafts" @@ -61,6 +62,14 @@ const uploadFiles = () => editor.then(ed => ed.chain().selectFiles().run()) + // Tiptap parses a string handed to insertContent as html, so an angle bracket in the + // transcript would eat the rest of the sentence. + const insertTranscript = async (text: string) => { + const ed = await editor + + ed.chain().focus().insertContent(escapeHtml(text)).run() + } + const submit = async () => { if ($uploading || disabled) return @@ -79,19 +88,23 @@ } draftKey?.clear() - ed.chain().clearContent().run() + ed.chain().clearContent(true).run() } let content = $state( initialValues?.type === "text" ? initialValues.value : (draftKey?.get()?.content ?? ""), ) + let recording = $state(false) const onChange = (json: object) => { content = json } + const empty = writable(true) + const editor = makeEditor({ content, + empty, submit, uploading, onChange, @@ -100,7 +113,11 @@ }) $effect(() => { - draftKey?.set({content}) + if ($empty) { + draftKey?.clear() + } else { + draftKey?.set({content}) + } }) onMount(async () => { @@ -135,11 +152,15 @@
- + {#if recording || ($empty && !disabled)} + + {:else} + + {/if} diff --git a/src/app/components/CommentCompose.svelte b/src/app/components/CommentCompose.svelte index bd208218..2d68f2cc 100644 --- a/src/app/components/CommentCompose.svelte +++ b/src/app/components/CommentCompose.svelte @@ -75,17 +75,24 @@ content = json } + const empty = writable(true) + const editor = makeEditor({ url, submit, uploading, onChange, content, + empty, placeholder: parent ? "Write a reply..." : "Write a comment...", }) $effect(() => { - draftKey.set({content}) + if ($empty) { + draftKey.clear() + } else { + draftKey.set({content}) + } }) diff --git a/src/app/components/DictationButton.svelte b/src/app/components/DictationButton.svelte new file mode 100644 index 00000000..13969fb8 --- /dev/null +++ b/src/app/components/DictationButton.svelte @@ -0,0 +1,105 @@ + + + diff --git a/src/app/components/DictationEnable.svelte b/src/app/components/DictationEnable.svelte new file mode 100644 index 00000000..35bc2bef --- /dev/null +++ b/src/app/components/DictationEnable.svelte @@ -0,0 +1,91 @@ + + + + + + Enable voice input? + Dictate your messages instead of typing them. + +

+ Your recordings are transcribed by OpenRouter, which charges a fraction of a cent per minute of audio. To turn voice input on, add some + credit to an OpenRouter account, then + create an API key and paste it below. +

+ + {#snippet label()} + OpenRouter API Key + {/snippet} + {#snippet input()} + + {/snippet} + {#snippet info()} + Your key is stored in your {PLATFORM_NAME} settings, encrypted to your nostr identity. + {/snippet} + + + + + + + diff --git a/src/app/components/EventReply.svelte b/src/app/components/EventReply.svelte index bb8df47e..3631070c 100644 --- a/src/app/components/EventReply.svelte +++ b/src/app/components/EventReply.svelte @@ -78,10 +78,16 @@ content = json } - const editor = makeEditor({url, submit, uploading, content, onChange}) + const empty = writable(true) + + const editor = makeEditor({url, submit, uploading, content, onChange, empty}) $effect(() => { - draftKey.set({content}) + if ($empty) { + draftKey.clear() + } else { + draftKey.set({content}) + } }) onMount(() => { diff --git a/src/app/components/RoomCompose.svelte b/src/app/components/RoomCompose.svelte index 3769a9dc..6dae07a5 100644 --- a/src/app/components/RoomCompose.svelte +++ b/src/app/components/RoomCompose.svelte @@ -2,7 +2,7 @@ import {writable} from "svelte/store" import type {Maybe} from "@welshman/lib" import type {EventContent} from "@welshman/util" - import {isMobile, preventDefault} from "@lib/html" + import {escapeHtml, isMobile, preventDefault} from "@lib/html" import GallerySend from "@assets/icons/gallery-send.svg?dataurl" import WidgetAdd from "@assets/icons/widget-add.svg?dataurl" import Plane from "@assets/icons/plane-2.svg?dataurl" @@ -12,6 +12,7 @@ import type {TippyController} from "@lib/components/Tippy.svelte" import Spinner from "@lib/components/Spinner.svelte" import ComposeMenu from "@app/components/ComposeMenu.svelte" + import DictationButton from "@app/components/DictationButton.svelte" import EditorContent from "@app/editor/EditorContent.svelte" import {makeEditor} from "@app/editor" import {DraftKey, type Draft} from "@app/drafts" @@ -53,6 +54,14 @@ const uploadFiles = () => editor.then(ed => ed.chain().selectFiles().run()) + // Tiptap parses a string handed to insertContent as html, so an angle bracket in the + // transcript would eat the rest of the sentence. + const insertTranscript = async (text: string) => { + const ed = await editor + + ed.chain().focus().insertContent(escapeHtml(text)).run() + } + const showPopover = () => tippy?.show() const hidePopover = () => tippy?.hide() @@ -67,21 +76,25 @@ onSubmit({content, tags}) draftKey?.clear() - ed.chain().clearContent().run() + ed.chain().clearContent(true).run() } let tippy: Maybe = $state() let content = $state( initialValues?.type === "text" ? initialValues.value : (draftKey?.get()?.content ?? ""), ) + let recording = $state(false) const onChange = (json: object) => { content = json } + const empty = writable(true) + const editor = makeEditor({ url, content, + empty, submit, uploading, onChange, @@ -89,7 +102,11 @@ }) $effect(() => { - draftKey?.set({content}) + if ($empty) { + draftKey?.clear() + } else { + draftKey?.set({content}) + } }) onMount(async () => { @@ -137,11 +154,15 @@
- + {#if recording || $empty} + + {:else} + + {/if} diff --git a/src/app/dictation.ts b/src/app/dictation.ts new file mode 100644 index 00000000..2e607031 --- /dev/null +++ b/src/app/dictation.ts @@ -0,0 +1,92 @@ +import {getSetting} from "@app/settings" + +const TRANSCRIPTION_MODEL = "openai/whisper-large-v3-turbo" + +// OpenRouter chooses a decoder using the uploaded file's extension, and MediaRecorder's output +// format varies by engine — webm on chromium, mp4 on webkit. +const EXTENSIONS_BY_MIME_TYPE: Record = { + "audio/webm": "webm", + "audio/ogg": "ogg", + "audio/mp4": "m4a", + "audio/mpeg": "mp3", + "audio/wav": "wav", +} + +// Reports how loud the microphone is once per frame, so the caller can show the speaker that we're +// hearing them. Levels follow the waveform's envelope — jumping to each peak, then decaying — since +// the raw root mean square drops to nothing in the gaps between words. +export const startDictation = async (onLevel: (level: number) => void) => { + const stream = await navigator.mediaDevices.getUserMedia({audio: true}) + const recorder = new MediaRecorder(stream) + const chunks: Blob[] = [] + + recorder.addEventListener("dataavailable", event => chunks.push(event.data)) + recorder.start() + + const context = new AudioContext() + const analyser = context.createAnalyser() + + analyser.fftSize = 512 + + context.createMediaStreamSource(stream).connect(analyser) + + const samples = new Uint8Array(analyser.frequencyBinCount) + + let level = 0 + + const measure = () => { + analyser.getByteTimeDomainData(samples) + + let sumSquares = 0 + + for (const sample of samples) { + sumSquares += ((sample - 128) / 128) ** 2 + } + + level = Math.max(Math.sqrt(sumSquares / samples.length), level * 0.92) + + onLevel(level) + + frame = requestAnimationFrame(measure) + } + + let frame = requestAnimationFrame(measure) + + return () => + new Promise(resolve => { + recorder.addEventListener("stop", () => { + cancelAnimationFrame(frame) + context.close() + + for (const track of stream.getTracks()) { + track.stop() + } + + resolve(new Blob(chunks, {type: recorder.mimeType})) + }) + + recorder.stop() + }) +} + +export const transcribe = async (audio: Blob) => { + const [mimeType] = audio.type.split(";") + const body = new FormData() + + body.append("model", TRANSCRIPTION_MODEL) + body.append("file", audio, `dictation.${EXTENSIONS_BY_MIME_TYPE[mimeType] || "webm"}`) + + const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", { + method: "POST", + headers: {Authorization: `Bearer ${getSetting("openrouter_key")}`}, + body, + }) + + const {text, error}: {text?: string; error?: {message?: string}} = await response.json() + + if (!response.ok) { + throw new Error(error?.message || `OpenRouter returned a ${response.status}.`) + } + + return text?.trim() ?? "" +} diff --git a/src/app/editor/index.ts b/src/app/editor/index.ts index 85031c09..93e8faed 100644 --- a/src/app/editor/index.ts +++ b/src/app/editor/index.ts @@ -25,11 +25,14 @@ import {userSpaceUrls} from "@app/rooms" import {PLATFORM_RELAYS} from "@app/env" import {pushToast} from "@app/toast" +const isEmpty = (editor: Editor) => editor.getText({blockSeparator: "\n"}).trim() === "" + export const makeEditor = async ({ encryptFiles = false, aggressive = false, charCount, content = "", + empty, onChange, placeholder = "", url, @@ -41,6 +44,7 @@ export const makeEditor = async ({ aggressive?: boolean charCount?: Writable content?: string | object + empty?: Writable onChange?: (json: object) => void placeholder?: string url?: string @@ -192,9 +196,16 @@ export const makeEditor = async ({ onUpdate({editor}) { wordCount?.set(editor.storage.wordCount.words) charCount?.set(editor.storage.wordCount.chars) + empty?.set(isEmpty(editor)) onChange?.(editor.getJSON()) }, }) + // Seed the caller's store from the document tiptap actually parsed — a restored draft is a + // document even when it holds no text, so the caller can't tell from `content` alone. Callers + // clear their draft when this reads true, so it has to be set before they render: keep every + // await in this function inside a callback, below the constructor. + empty?.set(isEmpty(ed)) + return ed } diff --git a/src/app/settings.ts b/src/app/settings.ts index dee35519..224eb3f1 100644 --- a/src/app/settings.ts +++ b/src/app/settings.ts @@ -32,6 +32,7 @@ export type SettingsValues = { font_size: number alerts: SpaceNotificationSettings[] zap_amounts: number[] + openrouter_key: string } export const defaultSettings: SettingsValues = { @@ -44,6 +45,7 @@ export const defaultSettings: SettingsValues = { font_size: 1.1, alerts: [], zap_amounts: [21, 210, 2100, 21000], + openrouter_key: "", } export class Settings extends DerivedPlugin { diff --git a/src/lib/components/base.css b/src/lib/components/base.css index b46cb330..6aed662c 100644 --- a/src/lib/components/base.css +++ b/src/lib/components/base.css @@ -120,6 +120,24 @@ animation: none; } } +/* Breathes a control in and out while it's listening to the user's voice. */ +@utility pulse-scale { + animation: pulse-scale 200ms ease-in-out infinite; +} +@keyframes pulse-scale { + 0%, + 100% { + transform: scale(1); + } + 50% { + transform: scale(1.1); + } +} +@media (prefers-reduced-motion: reduce) { + .pulse-scale { + animation: none; + } +} @utility pt-sai { padding-top: var(--sait); } diff --git a/src/routes/settings/content/+page.svelte b/src/routes/settings/content/+page.svelte index ef15bc33..f7bd6510 100644 --- a/src/routes/settings/content/+page.svelte +++ b/src/routes/settings/content/+page.svelte @@ -2,11 +2,13 @@ import {BlossomServerLists, MuteLists, publish} from "@welshman/app" import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl" import AddCircle from "@assets/icons/add-circle.svg?dataurl" + import Microphone from "@assets/icons/microphone.svg?dataurl" import {preventDefault} from "@lib/html" import Field from "@lib/components/Field.svelte" import FieldInline from "@lib/components/FieldInline.svelte" import ToggleInput from "@lib/components/ToggleInput.svelte" import InputList from "@lib/components/InputList.svelte" + import Link from "@lib/components/Link.svelte" import Icon from "@lib/components/Icon.svelte" import Card from "@lib/components/Card.svelte" import Button from "@lib/components/Button.svelte" @@ -119,6 +121,29 @@

{/snippet} + + {#snippet label()} +

Voice Input

+ {/snippet} + {#snippet input()} + + {/snippet} + {#snippet info()} +

+ Add an OpenRouter API key to dictate messages using the microphone button in your composer. +

+ {/snippet} + {#snippet label()}

Media Server