Voice conversations as a feature preview, through Unmute

deploy/workflow-unmute.yaml

@@ -0,0 +1,114 @@
+# Unmute (Kyutai) as a voice front end for an ACTIVATE Studio: speech
+# recognition, semantic turn-taking, and speech synthesis around the
+# Studio's own assistant, reached as the studio-voice model on the
+# Studio's /v1 endpoint. Runs the upstream docker compose on a GPU node
+# with the bundled LLM service disabled, and exposes Unmute's web
+# interface as a platform session. Feature preview.
+permissions:
+  - '*'
+'on':
+  execute:
+    inputs:
+      resource:
+        label: GPU system
+        type: compute-clusters
+        autoselect: true
+        optional: false
+        tooltip: A Linux x86_64 node with one GPU of 16 GB or more (STT about 2.5 GB, TTS about 5.3 GB) and Docker with the NVIDIA runtime.
+      studio:
+        type: group
+        label: Studio
+        items:
+          v1_url:
+            label: Studio /v1 base URL
+            type: string
+            optional: false
+            tooltip: The Studio's OpenAI-compatible endpoint, e.g. https://my-studio-rag.example/v1 (the registered RAG endpoint) or https://my-studio.example/v1.
+          api_key:
+            label: Studio API key
+            type: string
+            optional: false
+            tooltip: A platform token or API key the Studio accepts on /v1.
+          model:
+            label: Model
+            type: string
+            default: studio-voice
+            tooltip: studio-voice is the assistant with its tools and grounding, answering in spoken sentences.
+      unmute:
+        type: group
+        label: Unmute
+        items:
+          repo:
+            label: Repository
+            type: string
+            default: https://github.com/kyutai-labs/unmute.git
+          ref:
+            label: Git ref
+            type: string
+            default: main
+          hf_token:
+            label: Hugging Face token
+            type: string
+            optional: true
+            tooltip: Needed once to download the Kyutai STT and TTS weights and the voice files, which are gated behind accepting the model terms.
+          session_name:
+            label: Session name
+            type: string
+            default: unmute
+          workdir:
+            label: Working directory
+            type: string
+            default: ~/unmute-work
+jobs:
+  deploy:
+    ssh:
+      remoteHost: ${{ inputs.resource.ip }}
+    steps:
+      - name: Fetch Unmute
+        run: |
+          set -e
+          mkdir -p ${{ inputs.unmute.workdir }} && cd ${{ inputs.unmute.workdir }}
+          if [ ! -d unmute ]; then git clone --depth 1 --branch "${{ inputs.unmute.ref }}" "${{ inputs.unmute.repo }}" unmute; fi
+          cd unmute && git fetch --depth 1 origin "${{ inputs.unmute.ref }}" && git checkout -q FETCH_HEAD
+          command -v docker >/dev/null || { echo "docker is required on the node"; exit 1; }
+          docker info 2>/dev/null | grep -qi nvidia || echo "warning: no NVIDIA runtime reported by docker; STT and TTS will not start without it"
+      - name: Point the backend at the Studio and drop the bundled LLM
+        run: |
+          set -e
+          cd ${{ inputs.unmute.workdir }}/unmute
+          cat > docker-compose.studio.yml << 'EOF2'
+          services:
+            backend:
+              environment:
+                KYUTAI_LLM_URL: ${STUDIO_V1_URL}
+                KYUTAI_LLM_MODEL: ${STUDIO_MODEL}
+                KYUTAI_LLM_API_KEY: ${STUDIO_API_KEY}
+              depends_on: !override
+                - stt
+                - tts
+            llm:
+              profiles: ["disabled"]
+            traefik:
+              ports: !override
+                - "${UNMUTE_PORT:-80}:80"
+          EOF2
+          {
+            echo "STUDIO_V1_URL=${{ inputs.studio.v1_url }}"
+            echo "STUDIO_MODEL=${{ inputs.studio.model }}"
+            echo "STUDIO_API_KEY=${{ inputs.studio.api_key }}"
+            echo "HUGGING_FACE_HUB_TOKEN=${{ inputs.unmute.hf_token }}"
+          } > .env.studio
+          chmod 600 .env.studio
+      - name: Build the images
+        run: |
+          set -e
+          cd ${{ inputs.unmute.workdir }}/unmute
+          docker compose --env-file .env.studio -f docker-compose.yml -f docker-compose.studio.yml build
+      - name: Serve Unmute as a platform session
+        run: |
+          set -e
+          cd ${{ inputs.unmute.workdir }}/unmute
+          # The endpoint agent hands this command a free port in PORT; traefik
+          # publishes on it, and the agent tunnels it to the session URL.
+          pw endpoints run --name "${{ inputs.unmute.session_name }}" --subdomain "${{ inputs.unmute.session_name }}" -- \
+            bash -c 'UNMUTE_PORT="$PORT" docker compose --env-file .env.studio -f docker-compose.yml -f docker-compose.studio.yml up --abort-on-container-exit'

docs/HELP.md

@@ -67,6 +67,10 @@ Everything added becomes searchable in about a second.
 - Add by URL: web pages are reduced to text with the source recorded; PDFs saved as-is.
 - Files that arrive outside the interface are picked up by the background sync within minutes, or immediately with **sync now**.
 
+## Feature previews
+
+Capabilities a deployment can switch on under Settings, "Feature previews". **Voice conversations** put a Voice button above the chat that opens a live back-and-forth with the assistant: it listens, decides when you have finished a thought, answers in spoken sentences, and can be interrupted. It runs on a separate Unmute deployment (the `unmute` workflow) whose model is this Studio's own assistant, with its tools and knowledge base, so it can look things up and launch work while you talk.
+
 ## Stats
 
 Corpus health at a glance, and every element is a shortcut: storage rows and label pills open a prefilled query listing the matching files, largest and recently-changed rows open in the viewer, and the activity card tracks conversations, exported transcripts, and attachments.

server/src/ragProxy.ts

@@ -142,9 +142,14 @@ async function buildContext(query: string, topK: number, tags?: string[]): Promi
 }
 
 /** Resolve a virtual model id to { mode, underlying }. */
-function resolveModel(requested: string): { mode: 'agent' | 'rag'; underlying: string } {
+export function resolveModel(requested: string): { mode: 'agent' | 'rag'; underlying: string; voice?: boolean } {
   const eff = effectiveSettings()
   const fallback = eff.ragDefaultModel
+  // studio-voice is the agent tuned for a live voice conversation: the
+  // same tools and grounding, answers shaped to be spoken.
+  if (requested === 'studio-voice' || requested.startsWith('studio-voice/')) {
+    return { mode: 'agent', underlying: requested.slice('studio-voice/'.length) || fallback, voice: true }
+  }
   if (requested === 'studio-agent' || requested.startsWith('studio-agent/')) {
     return { mode: 'agent', underlying: requested.slice('studio-agent/'.length) || fallback }
   }
@@ -154,6 +159,8 @@ function resolveModel(requested: string): { mode: 'agent' | 'rag'; underlying: s
   return { mode: 'rag', underlying: requested }
 }
 
+export const VOICE_STYLE = 'You are speaking aloud in a live voice conversation. Answer in one or two short sentences of plain spoken language. No markdown, no lists, no headings, no links: say a file name instead of writing a path. Before a tool call that will take time, say in a few words what you are checking. If the user is silent or vague, ask one short question.'
+
 /* ---- OpenAI wire helpers ---- */
 
 function chunkOf(id: string, model: string, delta: Record<string, unknown>, finish: string | null): string {
@@ -170,11 +177,12 @@ async function agenticCompletion(
   key: string,
   onDelta: (text: string) => void,
   signal?: AbortSignal,
+  voice = false,
 ): Promise<{ content: string; finish: string }> {
   const sys = await systemPrompt()
   const today = new Date().toISOString().slice(0, 10)
   const messages: WireMessage[] = [
-    { role: 'system', content: `${sys}\n\nToday's date is ${today}.` },
+    { role: 'system', content: `${sys}\n\nToday's date is ${today}.${voice ? `\n\n${VOICE_STYLE}` : ''}` },
     ...(clientMessages as WireMessage[]).filter(m => m.role !== 'system').map(m => ({
       role: m.role,
       content: typeof m.content === 'string' ? m.content : m.content == null ? '' : JSON.stringify(m.content),
@@ -258,7 +266,7 @@ export async function ragProxyRoutes(app: FastifyInstance): Promise<void> {
     const entry = (id: string) => ({ id, object: 'model', created, owned_by: 'studio' })
     const eff0 = effectiveSettings()
     const data: { id: string; object: string; created: number; owned_by: string }[] = []
-    if (eff0.ragAdvertiseAgentModel) data.push(entry('studio-agent'))
+    if (eff0.ragAdvertiseAgentModel) { data.push(entry('studio-agent')); data.push(entry('studio-voice')) }
     if (eff0.ragAdvertiseRagModel) data.push(entry('studio-rag'))
     if (String((req.query as { all?: string }).all ?? '') === '1') {
       for (const id of ['studio-agent', 'studio-rag']) {
@@ -289,8 +297,9 @@ export async function ragProxyRoutes(app: FastifyInstance): Promise<void> {
     const eff = effectiveSettings()
     const body = { ...(req.body as Record<string, unknown>) }
     const requested = String(body.model ?? 'studio-rag')
-    let { mode, underlying } = resolveModel(requested)
-    const bare = requested === 'studio-agent' || requested === 'studio-rag'
+    const resolved = resolveModel(requested)
+    let { mode, underlying } = resolved
+    const bare = requested === 'studio-agent' || requested === 'studio-rag' || requested === 'studio-voice'
     if (bare) underlying = await defaultModelFor(key)
     if (!underlying) {
       return reply.status(400).send({ error: { message: `No underlying model available to this key: pass ${requested}/<gateway-model-id> or set a default model on the Settings RAG endpoint section.` } })
@@ -327,7 +336,7 @@ export async function ragProxyRoutes(app: FastifyInstance): Promise<void> {
         try {
           const r = await agenticCompletion(underlying, messages, key, t => {
             reply.raw.write(chunkOf(id, requested, { content: t }, null))
-          }, abort.signal)
+          }, abort.signal, !!resolved.voice)
           reply.raw.write(chunkOf(id, requested, {}, r.finish))
           rec.status = 'ok'
         } catch (err) {

server/src/routes.ts

@@ -165,6 +165,9 @@ export async function kbRoutes(app: FastifyInstance): Promise<void> {
       bannerText: eff.bannerText,
       bannerColor: eff.bannerColor,
       bannerWhenEmbedded: eff.bannerWhenEmbedded,
+      // Feature previews a deployment has switched on; the client shows
+      // their controls only when the flag and its configuration are both present.
+      features: { voice: { enabled: !!eff.voiceEnabled && !!eff.voiceUrl, url: eff.voiceUrl || '' } },
       user,
     }
   })

server/src/settings.ts

@@ -54,6 +54,9 @@ export interface StudioSettings {
   delegationEnabled?: boolean
   delegationMaxAgents?: number
   delegationMaxDepth?: number
+  /** Feature preview: voice conversations through an Unmute deployment. */
+  voiceEnabled?: boolean
+  voiceUrl?: string
 }
 
 let cache: StudioSettings | null = null
@@ -108,6 +111,8 @@ export function effectiveSettings(): Required<StudioSettings> {
     delegationEnabled: s.delegationEnabled ?? process.env.DELEGATION_ENABLED !== '0',
     delegationMaxAgents: s.delegationMaxAgents ?? Number(process.env.DELEGATION_MAX_AGENTS ?? 6),
     delegationMaxDepth: s.delegationMaxDepth ?? Number(process.env.DELEGATION_MAX_DEPTH ?? 1),
+    voiceEnabled: s.voiceEnabled ?? process.env.VOICE_ENABLED === '1',
+    voiceUrl: s.voiceUrl ?? process.env.VOICE_URL ?? '',
     customTools: s.customTools ?? [],
     ragDefaultModel: s.ragDefaultModel ?? process.env.RAG_DEFAULT_MODEL ?? '',
     ragTopK: s.ragTopK ?? Math.min(Math.max(Number(process.env.RAG_TOP_K) || 6, 1), 20),
@@ -215,6 +220,12 @@ export async function settingsRoutes(app: FastifyInstance): Promise<void> {
       }
     }
     if (body.delegationEnabled !== undefined) next.delegationEnabled = !!body.delegationEnabled
+    if (body.voiceEnabled !== undefined) next.voiceEnabled = !!body.voiceEnabled
+    if (body.voiceUrl !== undefined) {
+      const u = String(body.voiceUrl).trim().replace(/\/+$/, '')
+      if (u && !/^https?:\/\//.test(u)) throw new KbError(400, 'voice URL must be http(s)')
+      next.voiceUrl = u || undefined
+    }
     if (body.delegationMaxAgents !== undefined) {
       const n = Number(body.delegationMaxAgents)
       if (!Number.isFinite(n) || n < 1 || n > 24) throw new KbError(400, 'agents must be 1 to 24')

server/test/voicePreview.test.mjs

@@ -0,0 +1,28 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+process.env.KB_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-'))
+process.env.INDEX_BASE = fs.mkdtempSync(path.join(os.tmpdir(), 'ix-'))
+delete process.env.VOICE_ENABLED; delete process.env.VOICE_URL
+const { effectiveSettings } = await import('../dist/settings.js')
+const { resolveModel, VOICE_STYLE } = await import('../dist/ragProxy.js')
+
+test('the voice preview is off until a deployment turns it on, and the env flag can', () => {
+  assert.equal(effectiveSettings().voiceEnabled, false)
+  assert.equal(effectiveSettings().voiceUrl, '')
+  process.env.VOICE_ENABLED = '1'; process.env.VOICE_URL = 'https://unmute.example'
+  assert.equal(effectiveSettings().voiceEnabled, true)
+  assert.equal(effectiveSettings().voiceUrl, 'https://unmute.example')
+  delete process.env.VOICE_ENABLED; delete process.env.VOICE_URL
+})
+test('studio-voice is the agent with a spoken-answer style, and keeps an explicit underlying model', () => {
+  const v = resolveModel('studio-voice')
+  assert.equal(v.mode, 'agent'); assert.equal(v.voice, true)
+  const w = resolveModel('studio-voice/me:provider/model-x')
+  assert.equal(w.underlying, 'me:provider/model-x'); assert.equal(w.voice, true)
+  assert.equal(resolveModel('studio-agent').voice, undefined)
+  assert.match(VOICE_STYLE, /one or two short sentences/)
+  assert.match(VOICE_STYLE, /No markdown/)
+})

web/src/components/VoiceOverlay.tsx

@@ -0,0 +1,21 @@
+/**
+ * Voice conversations, as a feature preview: the Unmute deployment's own
+ * interface, in an overlay over the chat. Unmute is a separate service
+ * (speech recognition, turn-taking, synthesis, and the model behind them,
+ * which is this Studio's own assistant on the /v1 endpoint), so the
+ * overlay is an iframe with microphone permission delegated to it. The
+ * transcript stays with Unmute for now; recording it as a Studio
+ * conversation is the next step once the voice turns flow through here.
+ */
+export function VoiceOverlay({ url, onClose }: { url: string; onClose: () => void }) {
+  return (
+    <div className="voice-overlay" role="dialog" aria-label="Voice conversation">
+      <div className="voice-overlay-bar">
+        <span className="voice-overlay-title">Voice · feature preview</span>
+        <a className="link-button" href={url} target="_blank" rel="noopener noreferrer">open in a new tab</a>
+        <button className="btn-secondary" onClick={onClose}>Close</button>
+      </div>
+      <iframe className="voice-overlay-frame" src={url} title="Voice conversation" allow="microphone; autoplay; camera" />
+    </div>
+  )
+}

web/src/config.ts

@@ -16,6 +16,7 @@ export interface AppConfig {
   bannerColor: string
   /** Draw it even inside the platform frame, which shows its own. */
   bannerWhenEmbedded: boolean
+  features?: { voice?: { enabled: boolean; url: string } }
   user: { id: string; username: string; name?: string }
   loaded: boolean
   /** True when the name and icon came from the previous visit, so the header

web/src/styles.css

@@ -1270,6 +1270,13 @@ code, pre, kbd, .text-body, .viewer-path, .sql-box, [data-streamdown='code-block
   background: var(--empty-brand-icon) center / contain no-repeat;
 }
 
+/* Voice preview: Unmute's interface over the chat. */
+.voice-overlay { position: absolute; inset: 0; z-index: 40; display: flex; flex-direction: column; background: #000; }
+.voice-overlay-bar { display: flex; align-items: center; gap: 12px; padding: 8px 12px; background: var(--pw-panel); border-bottom: 1px solid var(--pw-border); }
+.voice-overlay-title { flex: 1; font-size: 13px; font-weight: 600; }
+.voice-overlay-frame { flex: 1; border: 0; width: 100%; background: #000; }
+.voice-btn { margin-right: 6px; }
+
 /* What to type next: chips above the composer while it is empty. */
 .next-up { position: absolute; z-index: 20; display: flex; flex-wrap: wrap; gap: 6px; pointer-events: none; }
 .next-up-chip {

web/src/views/ChatView.tsx

@@ -2,6 +2,7 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { useEffectiveTheme } from '../theme'
 import { SlashPalette } from '../components/SlashPalette'
 import { NextUp } from '../components/NextUp'
+import { VoiceOverlay } from '../components/VoiceOverlay'
 import { rememberHash } from '../lastLocation'
 import {
   ChatProvider, ChatLayout, ChatThread, ChatEmptyState, AttachmentManager, useChat,
@@ -94,6 +95,7 @@ export function ChatView() {
   const adapter = useMemo(() => createStudioAdapter(), [])
   const [credNote, setCredNote] = useState<string | null>(null)
   const effectiveTheme = useEffectiveTheme()
+  const [voiceOpen, setVoiceOpen] = useState(false)
   const [vocab, setVocab] = useState<{ tag: string; count: number }[]>([])
   const [scope, setScope] = useState<Set<string>>(new Set())
   const [chatFilter, setChatFilterState] = useState<'all' | 'mine'>(getChatListFilter())
@@ -379,6 +381,7 @@ export function ChatView() {
           {activeId && !showAttachments && <ConversationScrubber />}
           {!showAttachments && <SlashPalette canvas={canvasRef} />}
           {!showAttachments && <NextUp canvas={canvasRef} />}
+          {voiceOpen && cfg.features?.voice?.url && <VoiceOverlay url={cfg.features.voice.url} onClose={() => setVoiceOpen(false)} />}
           {railOpen && <div className="chat-rail-backdrop" onClick={() => setRailOpen(false)} />}
           <div className="chat-think-handle" onMouseDown={onThinkDrag} title="Drag to resize the activity panel" />
           {multiUser && sharedHistory && !showAttachments && (
@@ -391,6 +394,12 @@ export function ChatView() {
             </button>
           )}
           <div className="chat-controls">
+            {cfg.features?.voice?.enabled && (
+              <button className="scope-btn voice-btn" title="Talk with the assistant (feature preview)" onClick={() => setVoiceOpen(true)}>
+                <svg viewBox="0 0 16 16" width="13" height="13" fill="none" stroke="currentColor" strokeWidth="1.4"><rect x="5.5" y="1.5" width="5" height="8" rx="2.5"/><path d="M3 7.5a5 5 0 0 0 10 0M8 12.5v2M5.5 14.5h5"/></svg>
+                <span className="persona-label">Voice</span>
+              </button>
+            )}
             {(
               <div className={`chat-persona-anchor ${personaOpen ? 'open' : ''}`}>
                 <button