Refactors audio handling by introducing a custom useAudio hook to manage the state and lifecycle of audio recording and playback, improving component modularity and reusability.

src/App.jsx

@@ -22,10 +22,8 @@ import {
   readFileAsDataUrl,
 } from './attachments'
 import {
-  AUDIO_RECORDING_TICK_MS,
   BOTTOM_EPSILON,
   DEFAULT_BACKEND_API_URL,
-  MAX_AUDIO_RECORDING_MS,
   MAX_IMAGE_ATTACHMENT_BYTES,
   MAX_IMAGE_ATTACHMENTS,
   TOP_ALIGN_OFFSET,
@@ -46,36 +44,9 @@ import {
   loadStoredWebsearchEngines,
   normalizeWebsearchEngines,
 } from './websearchEngines'
-import {
+import { formatRecordingDuration, useAudioInput } from './useAudioInput'
-  getAudioInputConstraints,
-  getPreferredAudioRecorderMimeType,
-  stopMediaStream,
-  supportsAudioInputCapture,
-} from './audioInput'
 import { useChatLibrarySelection } from './useChatLibrarySelection'
 
-function appendTranscriptToComposer(currentInput, transcript) {
-  const nextTranscript = String(transcript || '').trim()
-  if (!nextTranscript) {
-    return currentInput || ''
-  }
-
-  const existing = String(currentInput || '')
-  if (!existing.trim()) {
-    return nextTranscript
-  }
-
-  const separator = /[\s\n]$/.test(existing) ? '' : '\n'
-  return `${existing}${separator}${nextTranscript}`
-}
-
-function formatRecordingDuration(milliseconds) {
-  const totalSeconds = Math.max(0, Math.floor(Number(milliseconds || 0) / 1000))
-  const minutes = Math.floor(totalSeconds / 60)
-  const seconds = totalSeconds % 60
-  return `${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`
-}
-
 export default function App() {
   const [chatSessions, setChatSessions] = useState([])
   const [activeSessionId, setActiveSessionId] = useState(null)

src/useAudioInput.js

@@ -0,0 +1,327 @@
+import { useEffect, useRef, useState } from 'react'
+import { readFileAsDataUrl } from './attachments'
+import {
+  AUDIO_RECORDING_TICK_MS,
+  MAX_AUDIO_RECORDING_MS,
+} from './appConfig'
+import { expectBackendJson, getErrorText, isAbortError } from './backendApi'
+import {
+  getAudioInputConstraints,
+  getPreferredAudioRecorderMimeType,
+  stopMediaStream,
+  supportsAudioInputCapture,
+} from './audioInput'
+
+function appendTranscriptToComposer(currentInput, transcript) {
+  const nextTranscript = String(transcript || '').trim()
+  if (!nextTranscript) {
+    return currentInput || ''
+  }
+
+  const existing = String(currentInput || '')
+  if (!existing.trim()) {
+    return nextTranscript
+  }
+
+  const separator = /[\s\n]$/.test(existing) ? '' : '\n'
+  return `${existing}${separator}${nextTranscript}`
+}
+
+export function formatRecordingDuration(milliseconds) {
+  const totalSeconds = Math.max(0, Math.floor(Number(milliseconds || 0) / 1000))
+  const minutes = Math.floor(totalSeconds / 60)
+  const seconds = totalSeconds % 60
+  return `${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`
+}
+
+export function useAudioInput({
+  activeSidebarMode,
+  audioInputDeviceId,
+  audioInputEnabled,
+  audioInputLanguage,
+  backendApiUrl,
+  isSending,
+  setInput,
+  textareaRef,
+  transcriptionModel,
+}) {
+  const [audioInputRuntimeReady, setAudioInputRuntimeReady] = useState(true)
+  const [audioInputRuntimeMessage, setAudioInputRuntimeMessage] = useState('')
+  const [isRecordingAudio, setIsRecordingAudio] = useState(false)
+  const [isTranscribingAudio, setIsTranscribingAudio] = useState(false)
+  const [audioRecordingMs, setAudioRecordingMs] = useState(0)
+  const audioRecorderRef = useRef(null)
+  const audioStreamRef = useRef(null)
+  const audioChunksRef = useRef([])
+  const audioStopPromiseRef = useRef(null)
+  const audioRecorderMimeTypeRef = useRef('')
+  const audioTickTimerRef = useRef(null)
+  const audioAutoStopTimerRef = useRef(null)
+  const audioStartedAtRef = useRef(0)
+  const audioTranscriptionAbortRef = useRef(null)
+
+  function syncAudioInputRuntimeFromStartupStatus(status) {
+    const whisperReady = Boolean(status?.whisper_model_available)
+    if (whisperReady) {
+      setAudioInputRuntimeReady(true)
+      setAudioInputRuntimeMessage('')
+      return
+    }
+
+    const modelName = status?.whisper_model || 'base'
+    const reason = String(status?.whisper_error || '').trim()
+    setAudioInputRuntimeReady(false)
+    setAudioInputRuntimeMessage(reason || `Whisper ${modelName} is not available.`)
+  }
+
+  function clearAudioTimers() {
+    if (audioTickTimerRef.current) {
+      window.clearInterval(audioTickTimerRef.current)
+      audioTickTimerRef.current = null
+    }
+    if (audioAutoStopTimerRef.current) {
+      window.clearTimeout(audioAutoStopTimerRef.current)
+      audioAutoStopTimerRef.current = null
+    }
+  }
+
+  function cleanupAudioRecorder(stream = audioStreamRef.current) {
+    clearAudioTimers()
+    stopMediaStream(stream)
+    if (audioStreamRef.current === stream) {
+      audioStreamRef.current = null
+    }
+    audioRecorderRef.current = null
+  }
+
+  async function transcribeRecordedAudio(blob, mimeType) {
+    if (!backendApiUrl) {
+      throw new Error('The Heimgeist backend is not configured.')
+    }
+
+    const controller = new AbortController()
+    audioTranscriptionAbortRef.current = controller
+    setIsTranscribingAudio(true)
+
+    try {
+      const dataUrl = await readFileAsDataUrl(blob)
+      const commaIndex = dataUrl.indexOf(',')
+      if (commaIndex <= 5) {
+        throw new Error('Recorded audio could not be encoded for upload.')
+      }
+      const header = dataUrl.slice(0, commaIndex)
+      const payload = dataUrl.slice(commaIndex + 1)
+      if (!/^data:[^,]+;base64$/i.test(header) || !payload) {
+        throw new Error('Recorded audio could not be encoded for upload.')
+      }
+      const detectedMimeType = header
+        .slice(5)
+        .replace(/;base64$/i, '')
+        .trim()
+
+      const response = await fetch(`${backendApiUrl}/audio/transcribe`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        signal: controller.signal,
+        body: JSON.stringify({
+          mime_type: mimeType || detectedMimeType || 'audio/webm',
+          audio_base64: payload,
+          model: transcriptionModel || null,
+          language: audioInputLanguage || null,
+        }),
+      })
+      const data = await expectBackendJson(response)
+      const transcript = String(data?.text || '').trim()
+
+      if (!transcript) {
+        window.alert('No speech was detected. Try again and speak closer to the selected microphone.')
+        return
+      }
+
+      setInput(prev => appendTranscriptToComposer(prev, transcript))
+      requestAnimationFrame(() => textareaRef.current?.focus())
+    } catch (error) {
+      if (isAbortError(error)) {
+        return
+      }
+      throw error
+    } finally {
+      if (audioTranscriptionAbortRef.current === controller) {
+        audioTranscriptionAbortRef.current = null
+      }
+      setIsTranscribingAudio(false)
+    }
+  }
+
+  async function stopAudioRecording(options = {}) {
+    const { shouldTranscribe = true } = options
+    const recorder = audioRecorderRef.current
+    const stopPromise = audioStopPromiseRef.current
+    const mimeType = audioRecorderMimeTypeRef.current || recorder?.mimeType || 'audio/webm'
+
+    if (!recorder || !stopPromise) {
+      return
+    }
+
+    setIsRecordingAudio(false)
+    clearAudioTimers()
+
+    try {
+      if (recorder.state !== 'inactive') {
+        recorder.stop()
+      }
+
+      const blob = await stopPromise
+      if (!shouldTranscribe) {
+        return
+      }
+      if (!blob || blob.size <= 0) {
+        throw new Error('Recorded audio was empty.')
+      }
+      await transcribeRecordedAudio(blob, mimeType)
+    } catch (error) {
+      if (shouldTranscribe) {
+        console.error('Failed to stop microphone recording', error)
+        window.alert(`Microphone transcription failed: ${getErrorText(error)}`)
+      }
+    } finally {
+      audioStopPromiseRef.current = null
+      audioChunksRef.current = []
+      audioRecorderMimeTypeRef.current = ''
+      audioStartedAtRef.current = 0
+      setAudioRecordingMs(0)
+    }
+  }
+
+  async function startAudioRecording() {
+    if (!audioInputEnabled || !audioInputRuntimeReady || isRecordingAudio || isTranscribingAudio || isSending) {
+      return
+    }
+    if (!supportsAudioInputCapture()) {
+      window.alert('Microphone capture is not available in this environment.')
+      return
+    }
+
+    let stream = null
+    try {
+      stream = await navigator.mediaDevices.getUserMedia(getAudioInputConstraints(audioInputDeviceId))
+    } catch (error) {
+      console.error('Failed to access microphone', error)
+      window.alert(`Microphone access failed: ${getErrorText(error)}`)
+      return
+    }
+
+    const preferredMimeType = getPreferredAudioRecorderMimeType()
+    let recorder = null
+    try {
+      recorder = preferredMimeType
+        ? new MediaRecorder(stream, { mimeType: preferredMimeType })
+        : new MediaRecorder(stream)
+    } catch (error) {
+      cleanupAudioRecorder(stream)
+      console.error('Failed to create audio recorder', error)
+      window.alert(`Microphone recording is not available: ${getErrorText(error)}`)
+      return
+    }
+
+    audioStreamRef.current = stream
+    audioRecorderRef.current = recorder
+    audioChunksRef.current = []
+    audioRecorderMimeTypeRef.current = recorder.mimeType || preferredMimeType || 'audio/webm'
+    audioStartedAtRef.current = Date.now()
+    setAudioRecordingMs(0)
+    setIsRecordingAudio(true)
+
+    recorder.ondataavailable = (event) => {
+      if (event.data && event.data.size > 0) {
+        audioChunksRef.current.push(event.data)
+      }
+    }
+
+    audioStopPromiseRef.current = new Promise((resolve, reject) => {
+      recorder.onstop = () => {
+        const blob = new Blob(audioChunksRef.current, {
+          type: recorder.mimeType || audioRecorderMimeTypeRef.current || 'audio/webm',
+        })
+        cleanupAudioRecorder(stream)
+        resolve(blob)
+      }
+      recorder.onerror = (event) => {
+        cleanupAudioRecorder(stream)
+        reject(event?.error || new Error('Audio recording failed.'))
+      }
+    })
+
+    audioTickTimerRef.current = window.setInterval(() => {
+      setAudioRecordingMs(Date.now() - audioStartedAtRef.current)
+    }, AUDIO_RECORDING_TICK_MS)
+    audioAutoStopTimerRef.current = window.setTimeout(() => {
+      stopAudioRecording()
+    }, MAX_AUDIO_RECORDING_MS)
+
+    recorder.start()
+  }
+
+  async function toggleAudioRecording() {
+    if (isTranscribingAudio) {
+      return
+    }
+    if (isRecordingAudio) {
+      await stopAudioRecording()
+      return
+    }
+    await startAudioRecording()
+  }
+
+  useEffect(() => {
+    if (audioInputEnabled || !isRecordingAudio) {
+      return
+    }
+    stopAudioRecording({ shouldTranscribe: false })
+  }, [audioInputEnabled, isRecordingAudio])
+
+  useEffect(() => {
+    if (audioInputRuntimeReady || !isRecordingAudio) {
+      return
+    }
+    stopAudioRecording({ shouldTranscribe: false })
+  }, [audioInputRuntimeReady, isRecordingAudio])
+
+  useEffect(() => {
+    if (activeSidebarMode === 'chats' || !isRecordingAudio) {
+      return
+    }
+    stopAudioRecording()
+  }, [activeSidebarMode, isRecordingAudio])
+
+  useEffect(() => {
+    return () => {
+      audioTranscriptionAbortRef.current?.abort()
+      clearAudioTimers()
+      const recorder = audioRecorderRef.current
+      if (recorder) {
+        recorder.ondataavailable = null
+        recorder.onstop = null
+        recorder.onerror = null
+        try {
+          if (recorder.state !== 'inactive') {
+            recorder.stop()
+          }
+        } catch {}
+      }
+      stopMediaStream(audioStreamRef.current)
+    }
+  }, [])
+
+  return {
+    audioInputRuntimeMessage,
+    audioInputRuntimeReady,
+    audioRecordingMs,
+    isRecordingAudio,
+    isTranscribingAudio,
+    setAudioInputRuntimeMessage,
+    setAudioInputRuntimeReady,
+    syncAudioInputRuntimeFromStartupStatus,
+    toggleAudioRecording,
+  }
+}