Add voice dictation

src/app/components/ChatCompose.svelte

@@ -4,12 +4,13 @@
   import cx from "classnames"
   import type {MaybeAsync} from "@welshman/lib"
   import type {EventContent} from "@welshman/util"
-  import {isMobile, preventDefault} from "@lib/html"
+  import {escapeHtml, isMobile, preventDefault} from "@lib/html"
   import GallerySend from "@assets/icons/gallery-send.svg?dataurl"
   import Plane from "@assets/icons/plane-2.svg?dataurl"
   import Icon from "@lib/components/Icon.svelte"
   import Button from "@lib/components/Button.svelte"
   import Spinner from "@lib/components/Spinner.svelte"
+  import DictationButton from "@app/components/DictationButton.svelte"
   import EditorContent from "@app/editor/EditorContent.svelte"
   import {makeEditor} from "@app/editor"
   import {type DraftKey, type Draft} from "@app/drafts"
@@ -61,6 +62,14 @@
 
   const uploadFiles = () => editor.then(ed => ed.chain().selectFiles().run())
 
+  // Tiptap parses a string handed to insertContent as html, so an angle bracket in the
+  // transcript would eat the rest of the sentence.
+  const insertTranscript = async (text: string) => {
+    const ed = await editor
+
+    ed.chain().focus().insertContent(escapeHtml(text)).run()
+  }
+
   const submit = async () => {
     if ($uploading || disabled) return
 
@@ -79,19 +88,23 @@
     }
 
     draftKey?.clear()
-    ed.chain().clearContent().run()
+    ed.chain().clearContent(true).run()
   }
 
   let content = $state(
     initialValues?.type === "text" ? initialValues.value : (draftKey?.get()?.content ?? ""),
   )
+  let recording = $state(false)
 
   const onChange = (json: object) => {
     content = json
   }
 
+  const empty = writable(true)
+
   const editor = makeEditor({
     content,
+    empty,
     submit,
     uploading,
     onChange,
@@ -100,7 +113,11 @@
   })
 
   $effect(() => {
+    if ($empty) {
+      draftKey?.clear()
+    } else {
       draftKey?.set({content})
+    }
   })
 
   onMount(async () => {
@@ -135,6 +152,9 @@
   <div class={editorClass} aria-disabled={disabled}>
     <EditorContent {autofocus} {editor} />
   </div>
+  {#if recording || ($empty && !disabled)}
+    <DictationButton bind:recording onTranscript={insertTranscript} />
+  {:else}
     <Button
       data-tip="{window.navigator.platform.includes('Mac') ? 'cmd' : 'ctrl'}+enter to send"
       class="button button-primary button-circle tip tip-left h-10 w-10 min-w-10"
@@ -142,4 +162,5 @@
       onclick={submit}>
       <Icon icon={Plane} />
     </Button>
+  {/if}
 </form>

src/app/components/CommentCompose.svelte

@@ -75,17 +75,24 @@
     content = json
   }
 
+  const empty = writable(true)
+
   const editor = makeEditor({
     url,
     submit,
     uploading,
     onChange,
     content,
+    empty,
     placeholder: parent ? "Write a reply..." : "Write a comment...",
   })
 
   $effect(() => {
+    if ($empty) {
+      draftKey.clear()
+    } else {
       draftKey.set({content})
+    }
   })
 </script>
 

src/app/components/DictationButton.svelte

@@ -0,0 +1,105 @@
+<script lang="ts">
+  import {onDestroy} from "svelte"
+  import cx from "classnames"
+  import type {Maybe, MaybeAsync} from "@welshman/lib"
+  import Microphone from "@assets/icons/microphone.svg?dataurl"
+  import Stop from "@assets/icons/record.svg?dataurl"
+  import Icon from "@lib/components/Icon.svelte"
+  import Button from "@lib/components/Button.svelte"
+  import Spinner from "@lib/components/Spinner.svelte"
+  import {errorMessage} from "@lib/util"
+  import DictationEnable from "@app/components/DictationEnable.svelte"
+  import {startDictation, transcribe} from "@app/dictation"
+  import {getSetting} from "@app/settings"
+  import {pushModal} from "@app/modal"
+  import {pushToast} from "@app/toast"
+
+  type Props = {
+    recording?: boolean
+    onTranscript: (text: string) => MaybeAsync<void>
+  }
+
+  let {recording = $bindable(false), onTranscript}: Props = $props()
+
+  // Room noise idles just below this, so the pulse follows quiet speech too.
+  const onLevel = (level: number) => {
+    loud = level > 0.03
+  }
+
+  const start = async () => {
+    if (getSetting("openrouter_key")) {
+      // Granting microphone access can sit on a permission prompt for a while, so hold the button
+      // until it resolves — a second click would open a stream nothing is left holding on to.
+      loading = true
+
+      try {
+        finish = await startDictation(onLevel)
+        recording = true
+      } catch (error) {
+        console.error(error)
+        pushToast({theme: "error", message: "Failed to access your microphone."})
+      } finally {
+        loading = false
+      }
+    } else {
+      pushModal(DictationEnable)
+    }
+  }
+
+  const stop = async () => {
+    if (finish) {
+      const audio = finish()
+
+      finish = undefined
+      recording = false
+      loading = true
+      loud = false
+
+      try {
+        const text = await transcribe(await audio)
+
+        await onTranscript(text)
+      } catch (error) {
+        console.error(error)
+        pushToast({theme: "error", message: `Failed to transcribe: ${errorMessage(error)}`})
+      } finally {
+        loading = false
+      }
+    }
+  }
+
+  const toggle = () => (recording ? stop() : start())
+
+  // Held outside of state because only the recording flag drives the markup, and clearing this
+  // before the recorder has finished flushing is what keeps a second stop from re-entering.
+  let finish: Maybe<() => Promise<Blob>>
+  let loading = $state(false)
+  let loud = $state(false)
+
+  const buttonClass = $derived(
+    cx(
+      "button button-circle tip tip-left h-10 w-10 min-w-10",
+      recording ? "button-error" : "button-primary",
+      {"pulse-scale": loud},
+    ),
+  )
+
+  onDestroy(() => {
+    finish?.()
+  })
+</script>
+
+<Button
+  class={buttonClass}
+  data-tip={recording ? "Stop and transcribe" : "Record a message"}
+  aria-label={recording ? "Stop recording" : "Start dictation"}
+  disabled={loading}
+  onclick={toggle}>
+  {#if loading}
+    <Spinner size="xs" />
+  {:else if recording}
+    <Icon icon={Stop} />
+  {:else}
+    <Icon icon={Microphone} />
+  {/if}
+</Button>

src/app/components/DictationEnable.svelte

@@ -0,0 +1,91 @@
+<script lang="ts">
+  import {preventDefault} from "@lib/html"
+  import AltArrowLeft from "@assets/icons/alt-arrow-left.svg?dataurl"
+  import AltArrowRight from "@assets/icons/alt-arrow-right.svg?dataurl"
+  import Lock from "@assets/icons/lock-keyhole.svg?dataurl"
+  import Icon from "@lib/components/Icon.svelte"
+  import Link from "@lib/components/Link.svelte"
+  import Button from "@lib/components/Button.svelte"
+  import Spinner from "@lib/components/Spinner.svelte"
+  import {errorMessage} from "@lib/util"
+  import Field from "@lib/components/Field.svelte"
+  import Modal from "@lib/components/Modal.svelte"
+  import ModalBody from "@lib/components/ModalBody.svelte"
+  import ModalHeader from "@lib/components/ModalHeader.svelte"
+  import ModalTitle from "@lib/components/ModalTitle.svelte"
+  import ModalSubtitle from "@lib/components/ModalSubtitle.svelte"
+  import ModalFooter from "@lib/components/ModalFooter.svelte"
+  import {PLATFORM_NAME} from "@app/env"
+  import {publishSettings} from "@app/settings"
+  import {pushToast} from "@app/toast"
+
+  const back = () => history.back()
+
+  const save = async () => {
+    loading = true
+
+    try {
+      await publishSettings({openrouter_key: openrouterKey.trim()})
+
+      pushToast({message: "Voice input is ready to use!"})
+
+      back()
+    } catch (error) {
+      console.error(error)
+      pushToast({theme: "error", message: `Failed to save your key: ${errorMessage(error)}`})
+    } finally {
+      loading = false
+    }
+  }
+
+  let openrouterKey = $state("")
+  let loading = $state(false)
+</script>
+
+<Modal tag="form" onsubmit={preventDefault(save)}>
+  <ModalBody>
+    <ModalHeader>
+      <ModalTitle>Enable voice input?</ModalTitle>
+      <ModalSubtitle>Dictate your messages instead of typing them.</ModalSubtitle>
+    </ModalHeader>
+    <p>
+      Your recordings are transcribed by <Link
+        external
+        href="https://openrouter.ai"
+        class="text-primary">OpenRouter</Link
+      >, which charges a fraction of a cent per minute of audio. To turn voice input on, add some
+      credit to an OpenRouter account, then
+      <Link external href="https://openrouter.ai/settings/keys" class="text-primary"
+        >create an API key</Link> and paste it below.
+    </p>
+    <Field>
+      {#snippet label()}
+        OpenRouter API Key
+      {/snippet}
+      {#snippet input()}
+        <label class="input flex w-full items-center gap-2">
+          <Icon icon={Lock} />
+          <input
+            bind:value={openrouterKey}
+            autocomplete="off"
+            name="flotilla-openrouter-key"
+            class="grow"
+            type="password" />
+        </label>
+      {/snippet}
+      {#snippet info()}
+        Your key is stored in your {PLATFORM_NAME} settings, encrypted to your nostr identity.
+      {/snippet}
+    </Field>
+  </ModalBody>
+  <ModalFooter>
+    <Button class="button button-link" onclick={back}>
+      <Icon icon={AltArrowLeft} />
+      Go back
+    </Button>
+    <Button type="submit" class="button button-primary" disabled={!openrouterKey || loading}>
+      <Spinner {loading}>Enable voice input</Spinner>
+      <Icon icon={AltArrowRight} />
+    </Button>
+  </ModalFooter>
+</Modal>

src/app/components/EventReply.svelte

@@ -78,10 +78,16 @@
     content = json
   }
 
-  const editor = makeEditor({url, submit, uploading, content, onChange})
+  const empty = writable(true)
+
+  const editor = makeEditor({url, submit, uploading, content, onChange, empty})
 
   $effect(() => {
+    if ($empty) {
+      draftKey.clear()
+    } else {
       draftKey.set({content})
+    }
   })
 
   onMount(() => {

src/app/components/RoomCompose.svelte

@@ -2,7 +2,7 @@
   import {writable} from "svelte/store"
   import type {Maybe} from "@welshman/lib"
   import type {EventContent} from "@welshman/util"
-  import {isMobile, preventDefault} from "@lib/html"
+  import {escapeHtml, isMobile, preventDefault} from "@lib/html"
   import GallerySend from "@assets/icons/gallery-send.svg?dataurl"
   import WidgetAdd from "@assets/icons/widget-add.svg?dataurl"
   import Plane from "@assets/icons/plane-2.svg?dataurl"
@@ -12,6 +12,7 @@
   import type {TippyController} from "@lib/components/Tippy.svelte"
   import Spinner from "@lib/components/Spinner.svelte"
   import ComposeMenu from "@app/components/ComposeMenu.svelte"
+  import DictationButton from "@app/components/DictationButton.svelte"
   import EditorContent from "@app/editor/EditorContent.svelte"
   import {makeEditor} from "@app/editor"
   import {DraftKey, type Draft} from "@app/drafts"
@@ -53,6 +54,14 @@
 
   const uploadFiles = () => editor.then(ed => ed.chain().selectFiles().run())
 
+  // Tiptap parses a string handed to insertContent as html, so an angle bracket in the
+  // transcript would eat the rest of the sentence.
+  const insertTranscript = async (text: string) => {
+    const ed = await editor
+
+    ed.chain().focus().insertContent(escapeHtml(text)).run()
+  }
+
   const showPopover = () => tippy?.show()
 
   const hidePopover = () => tippy?.hide()
@@ -67,21 +76,25 @@
     onSubmit({content, tags})
 
     draftKey?.clear()
-    ed.chain().clearContent().run()
+    ed.chain().clearContent(true).run()
   }
 
   let tippy: Maybe<TippyController> = $state()
   let content = $state(
     initialValues?.type === "text" ? initialValues.value : (draftKey?.get()?.content ?? ""),
   )
+  let recording = $state(false)
 
   const onChange = (json: object) => {
     content = json
   }
 
+  const empty = writable(true)
+
   const editor = makeEditor({
     url,
     content,
+    empty,
     submit,
     uploading,
     onChange,
@@ -89,7 +102,11 @@
   })
 
   $effect(() => {
+    if ($empty) {
+      draftKey?.clear()
+    } else {
       draftKey?.set({content})
+    }
   })
 
   onMount(async () => {
@@ -137,6 +154,9 @@
   <div class="chat-editor grow overflow-hidden">
     <EditorContent {autofocus} {editor} />
   </div>
+  {#if recording || $empty}
+    <DictationButton bind:recording onTranscript={insertTranscript} />
+  {:else}
     <Button
       data-tip="{window.navigator.platform.includes('Mac') ? 'cmd' : 'ctrl'}+enter to send"
       class="button button-primary button-circle flex justify-center items-center tip tip-left h-10 w-10 min-w-10"
@@ -144,4 +164,5 @@
       onclick={submit}>
       <Icon icon={Plane} />
     </Button>
+  {/if}
 </form>

src/app/dictation.ts

@@ -0,0 +1,92 @@
+import {getSetting} from "@app/settings"
+
+const TRANSCRIPTION_MODEL = "openai/whisper-large-v3-turbo"
+
+// OpenRouter chooses a decoder using the uploaded file's extension, and MediaRecorder's output
+// format varies by engine — webm on chromium, mp4 on webkit.
+const EXTENSIONS_BY_MIME_TYPE: Record<string, string> = {
+  "audio/webm": "webm",
+  "audio/ogg": "ogg",
+  "audio/mp4": "m4a",
+  "audio/mpeg": "mp3",
+  "audio/wav": "wav",
+}
+
+// Reports how loud the microphone is once per frame, so the caller can show the speaker that we're
+// hearing them. Levels follow the waveform's envelope — jumping to each peak, then decaying — since
+// the raw root mean square drops to nothing in the gaps between words.
+export const startDictation = async (onLevel: (level: number) => void) => {
+  const stream = await navigator.mediaDevices.getUserMedia({audio: true})
+  const recorder = new MediaRecorder(stream)
+  const chunks: Blob[] = []
+
+  recorder.addEventListener("dataavailable", event => chunks.push(event.data))
+  recorder.start()
+
+  const context = new AudioContext()
+  const analyser = context.createAnalyser()
+
+  analyser.fftSize = 512
+
+  context.createMediaStreamSource(stream).connect(analyser)
+
+  const samples = new Uint8Array(analyser.frequencyBinCount)
+
+  let level = 0
+
+  const measure = () => {
+    analyser.getByteTimeDomainData(samples)
+
+    let sumSquares = 0
+
+    for (const sample of samples) {
+      sumSquares += ((sample - 128) / 128) ** 2
+    }
+
+    level = Math.max(Math.sqrt(sumSquares / samples.length), level * 0.92)
+
+    onLevel(level)
+
+    frame = requestAnimationFrame(measure)
+  }
+
+  let frame = requestAnimationFrame(measure)
+
+  return () =>
+    new Promise<Blob>(resolve => {
+      recorder.addEventListener("stop", () => {
+        cancelAnimationFrame(frame)
+        context.close()
+
+        for (const track of stream.getTracks()) {
+          track.stop()
+        }
+
+        resolve(new Blob(chunks, {type: recorder.mimeType}))
+      })
+
+      recorder.stop()
+    })
+}
+
+export const transcribe = async (audio: Blob) => {
+  const [mimeType] = audio.type.split(";")
+  const body = new FormData()
+
+  body.append("model", TRANSCRIPTION_MODEL)
+  body.append("file", audio, `dictation.${EXTENSIONS_BY_MIME_TYPE[mimeType] || "webm"}`)
+
+  const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", {
+    method: "POST",
+    headers: {Authorization: `Bearer ${getSetting("openrouter_key")}`},
+    body,
+  })
+
+  const {text, error}: {text?: string; error?: {message?: string}} = await response.json()
+
+  if (!response.ok) {
+    throw new Error(error?.message || `OpenRouter returned a ${response.status}.`)
+  }
+
+  return text?.trim() ?? ""
+}

src/app/editor/index.ts

@@ -25,11 +25,14 @@ import {userSpaceUrls} from "@app/rooms"
 import {PLATFORM_RELAYS} from "@app/env"
 import {pushToast} from "@app/toast"
 
+const isEmpty = (editor: Editor) => editor.getText({blockSeparator: "\n"}).trim() === ""
+
 export const makeEditor = async ({
   encryptFiles = false,
   aggressive = false,
   charCount,
   content = "",
+  empty,
   onChange,
   placeholder = "",
   url,
@@ -41,6 +44,7 @@ export const makeEditor = async ({
   aggressive?: boolean
   charCount?: Writable<number>
   content?: string | object
+  empty?: Writable<boolean>
   onChange?: (json: object) => void
   placeholder?: string
   url?: string
@@ -192,9 +196,16 @@ export const makeEditor = async ({
     onUpdate({editor}) {
       wordCount?.set(editor.storage.wordCount.words)
       charCount?.set(editor.storage.wordCount.chars)
+      empty?.set(isEmpty(editor))
       onChange?.(editor.getJSON())
     },
   })
 
+  // Seed the caller's store from the document tiptap actually parsed — a restored draft is a
+  // document even when it holds no text, so the caller can't tell from `content` alone. Callers
+  // clear their draft when this reads true, so it has to be set before they render: keep every
+  // await in this function inside a callback, below the constructor.
+  empty?.set(isEmpty(ed))
+
   return ed
 }

src/app/settings.ts

@@ -32,6 +32,7 @@ export type SettingsValues = {
   font_size: number
   alerts: SpaceNotificationSettings[]
   zap_amounts: number[]
+  openrouter_key: string
 }
 
 export const defaultSettings: SettingsValues = {
@@ -44,6 +45,7 @@ export const defaultSettings: SettingsValues = {
   font_size: 1.1,
   alerts: [],
   zap_amounts: [21, 210, 2100, 21000],
+  openrouter_key: "",
 }
 
 export class Settings extends DerivedPlugin<AppDataReader> {

src/lib/components/base.css

@@ -120,6 +120,24 @@
     animation: none;
   }
 }
+/* Breathes a control in and out while it's listening to the user's voice. */
+@utility pulse-scale {
+  animation: pulse-scale 200ms ease-in-out infinite;
+}
+@keyframes pulse-scale {
+  0%,
+  100% {
+    transform: scale(1);
+  }
+  50% {
+    transform: scale(1.1);
+  }
+}
+@media (prefers-reduced-motion: reduce) {
+  .pulse-scale {
+    animation: none;
+  }
+}
 @utility pt-sai {
   padding-top: var(--sait);
 }

src/routes/settings/content/+page.svelte

@@ -2,11 +2,13 @@
   import {BlossomServerLists, MuteLists, publish} from "@welshman/app"
   import NotesMinimalistic from "@assets/icons/notes-minimalistic.svg?dataurl"
   import AddCircle from "@assets/icons/add-circle.svg?dataurl"
+  import Microphone from "@assets/icons/microphone.svg?dataurl"
   import {preventDefault} from "@lib/html"
   import Field from "@lib/components/Field.svelte"
   import FieldInline from "@lib/components/FieldInline.svelte"
   import ToggleInput from "@lib/components/ToggleInput.svelte"
   import InputList from "@lib/components/InputList.svelte"
+  import Link from "@lib/components/Link.svelte"
   import Icon from "@lib/components/Icon.svelte"
   import Card from "@lib/components/Card.svelte"
   import Button from "@lib/components/Button.svelte"
@@ -119,6 +121,29 @@
           </p>
         {/snippet}
       </Field>
+      <Field>
+        {#snippet label()}
+          <p>Voice Input</p>
+        {/snippet}
+        {#snippet input()}
+          <label class="input flex w-full items-center gap-2">
+            <Icon icon={Microphone} />
+            <input
+              bind:value={$settings.openrouter_key}
+              autocomplete="off"
+              name="flotilla-openrouter-key"
+              placeholder="OpenRouter API key"
+              class="grow"
+              type="password" />
+          </label>
+        {/snippet}
+        {#snippet info()}
+          <p>
+            Add an <Link external href="https://openrouter.ai/settings/keys" class="text-primary"
+              >OpenRouter API key</Link> to dictate messages using the microphone button in your composer.
+          </p>
+        {/snippet}
+      </Field>
       <Field>
         {#snippet label()}
           <p>Media Server</p>