diff --git a/deploy/workflow-unmute.yaml b/deploy/workflow-unmute.yaml new file mode 100644 index 0000000..f8278cd --- /dev/null +++ b/deploy/workflow-unmute.yaml @@ -0,0 +1,114 @@ +# Unmute (Kyutai) as a voice front end for an ACTIVATE Studio: speech +# recognition, semantic turn-taking, and speech synthesis around the +# Studio's own assistant, reached as the studio-voice model on the +# Studio's /v1 endpoint. Runs the upstream docker compose on a GPU node +# with the bundled LLM service disabled, and exposes Unmute's web +# interface as a platform session. Feature preview. +permissions: + - '*' +'on': + execute: + inputs: + resource: + label: GPU system + type: compute-clusters + autoselect: true + optional: false + tooltip: A Linux x86_64 node with one GPU of 16 GB or more (STT about 2.5 GB, TTS about 5.3 GB) and Docker with the NVIDIA runtime. + studio: + type: group + label: Studio + items: + v1_url: + label: Studio /v1 base URL + type: string + optional: false + tooltip: The Studio's OpenAI-compatible endpoint, e.g. https://my-studio-rag.example/v1 (the registered RAG endpoint) or https://my-studio.example/v1. + api_key: + label: Studio API key + type: string + optional: false + tooltip: A platform token or API key the Studio accepts on /v1. + model: + label: Model + type: string + default: studio-voice + tooltip: studio-voice is the assistant with its tools and grounding, answering in spoken sentences. + unmute: + type: group + label: Unmute + items: + repo: + label: Repository + type: string + default: https://github.com/kyutai-labs/unmute.git + ref: + label: Git ref + type: string + default: main + hf_token: + label: Hugging Face token + type: string + optional: true + tooltip: Needed once to download the Kyutai STT and TTS weights and the voice files, which are gated behind accepting the model terms. + session_name: + label: Session name + type: string + default: unmute + workdir: + label: Working directory + type: string + default: ~/unmute-work +jobs: + deploy: + ssh: + remoteHost: ${{ inputs.resource.ip }} + steps: + - name: Fetch Unmute + run: | + set -e + mkdir -p ${{ inputs.unmute.workdir }} && cd ${{ inputs.unmute.workdir }} + if [ ! -d unmute ]; then git clone --depth 1 --branch "${{ inputs.unmute.ref }}" "${{ inputs.unmute.repo }}" unmute; fi + cd unmute && git fetch --depth 1 origin "${{ inputs.unmute.ref }}" && git checkout -q FETCH_HEAD + command -v docker >/dev/null || { echo "docker is required on the node"; exit 1; } + docker info 2>/dev/null | grep -qi nvidia || echo "warning: no NVIDIA runtime reported by docker; STT and TTS will not start without it" + - name: Point the backend at the Studio and drop the bundled LLM + run: | + set -e + cd ${{ inputs.unmute.workdir }}/unmute + cat > docker-compose.studio.yml << 'EOF2' + services: + backend: + environment: + KYUTAI_LLM_URL: ${STUDIO_V1_URL} + KYUTAI_LLM_MODEL: ${STUDIO_MODEL} + KYUTAI_LLM_API_KEY: ${STUDIO_API_KEY} + depends_on: !override + - stt + - tts + llm: + profiles: ["disabled"] + traefik: + ports: !override + - "${UNMUTE_PORT:-80}:80" + EOF2 + { + echo "STUDIO_V1_URL=${{ inputs.studio.v1_url }}" + echo "STUDIO_MODEL=${{ inputs.studio.model }}" + echo "STUDIO_API_KEY=${{ inputs.studio.api_key }}" + echo "HUGGING_FACE_HUB_TOKEN=${{ inputs.unmute.hf_token }}" + } > .env.studio + chmod 600 .env.studio + - name: Build the images + run: | + set -e + cd ${{ inputs.unmute.workdir }}/unmute + docker compose --env-file .env.studio -f docker-compose.yml -f docker-compose.studio.yml build + - name: Serve Unmute as a platform session + run: | + set -e + cd ${{ inputs.unmute.workdir }}/unmute + # The endpoint agent hands this command a free port in PORT; traefik + # publishes on it, and the agent tunnels it to the session URL. + pw endpoints run --name "${{ inputs.unmute.session_name }}" --subdomain "${{ inputs.unmute.session_name }}" -- \ + bash -c 'UNMUTE_PORT="$PORT" docker compose --env-file .env.studio -f docker-compose.yml -f docker-compose.studio.yml up --abort-on-container-exit' diff --git a/docs/HELP.md b/docs/HELP.md index 68e5ce8..211f19e 100644 --- a/docs/HELP.md +++ b/docs/HELP.md @@ -67,6 +67,10 @@ Everything added becomes searchable in about a second. - Add by URL: web pages are reduced to text with the source recorded; PDFs saved as-is. - Files that arrive outside the interface are picked up by the background sync within minutes, or immediately with **sync now**. +## Feature previews + +Capabilities a deployment can switch on under Settings, "Feature previews". **Voice conversations** put a Voice button above the chat that opens a live back-and-forth with the assistant: it listens, decides when you have finished a thought, answers in spoken sentences, and can be interrupted. It runs on a separate Unmute deployment (the `unmute` workflow) whose model is this Studio's own assistant, with its tools and knowledge base, so it can look things up and launch work while you talk. + ## Stats Corpus health at a glance, and every element is a shortcut: storage rows and label pills open a prefilled query listing the matching files, largest and recently-changed rows open in the viewer, and the activity card tracks conversations, exported transcripts, and attachments. diff --git a/server/src/ragProxy.ts b/server/src/ragProxy.ts index fc65b7b..baa5562 100644 --- a/server/src/ragProxy.ts +++ b/server/src/ragProxy.ts @@ -142,9 +142,14 @@ async function buildContext(query: string, topK: number, tags?: string[]): Promi } /** Resolve a virtual model id to { mode, underlying }. */ -function resolveModel(requested: string): { mode: 'agent' | 'rag'; underlying: string } { +export function resolveModel(requested: string): { mode: 'agent' | 'rag'; underlying: string; voice?: boolean } { const eff = effectiveSettings() const fallback = eff.ragDefaultModel + // studio-voice is the agent tuned for a live voice conversation: the + // same tools and grounding, answers shaped to be spoken. + if (requested === 'studio-voice' || requested.startsWith('studio-voice/')) { + return { mode: 'agent', underlying: requested.slice('studio-voice/'.length) || fallback, voice: true } + } if (requested === 'studio-agent' || requested.startsWith('studio-agent/')) { return { mode: 'agent', underlying: requested.slice('studio-agent/'.length) || fallback } } @@ -154,6 +159,8 @@ function resolveModel(requested: string): { mode: 'agent' | 'rag'; underlying: s return { mode: 'rag', underlying: requested } } +export const VOICE_STYLE = 'You are speaking aloud in a live voice conversation. Answer in one or two short sentences of plain spoken language. No markdown, no lists, no headings, no links: say a file name instead of writing a path. Before a tool call that will take time, say in a few words what you are checking. If the user is silent or vague, ask one short question.' + /* ---- OpenAI wire helpers ---- */ function chunkOf(id: string, model: string, delta: Record, finish: string | null): string { @@ -170,11 +177,12 @@ async function agenticCompletion( key: string, onDelta: (text: string) => void, signal?: AbortSignal, + voice = false, ): Promise<{ content: string; finish: string }> { const sys = await systemPrompt() const today = new Date().toISOString().slice(0, 10) const messages: WireMessage[] = [ - { role: 'system', content: `${sys}\n\nToday's date is ${today}.` }, + { role: 'system', content: `${sys}\n\nToday's date is ${today}.${voice ? `\n\n${VOICE_STYLE}` : ''}` }, ...(clientMessages as WireMessage[]).filter(m => m.role !== 'system').map(m => ({ role: m.role, content: typeof m.content === 'string' ? m.content : m.content == null ? '' : JSON.stringify(m.content), @@ -258,7 +266,7 @@ export async function ragProxyRoutes(app: FastifyInstance): Promise { const entry = (id: string) => ({ id, object: 'model', created, owned_by: 'studio' }) const eff0 = effectiveSettings() const data: { id: string; object: string; created: number; owned_by: string }[] = [] - if (eff0.ragAdvertiseAgentModel) data.push(entry('studio-agent')) + if (eff0.ragAdvertiseAgentModel) { data.push(entry('studio-agent')); data.push(entry('studio-voice')) } if (eff0.ragAdvertiseRagModel) data.push(entry('studio-rag')) if (String((req.query as { all?: string }).all ?? '') === '1') { for (const id of ['studio-agent', 'studio-rag']) { @@ -289,8 +297,9 @@ export async function ragProxyRoutes(app: FastifyInstance): Promise { const eff = effectiveSettings() const body = { ...(req.body as Record) } const requested = String(body.model ?? 'studio-rag') - let { mode, underlying } = resolveModel(requested) - const bare = requested === 'studio-agent' || requested === 'studio-rag' + const resolved = resolveModel(requested) + let { mode, underlying } = resolved + const bare = requested === 'studio-agent' || requested === 'studio-rag' || requested === 'studio-voice' if (bare) underlying = await defaultModelFor(key) if (!underlying) { return reply.status(400).send({ error: { message: `No underlying model available to this key: pass ${requested}/ or set a default model on the Settings RAG endpoint section.` } }) @@ -327,7 +336,7 @@ export async function ragProxyRoutes(app: FastifyInstance): Promise { try { const r = await agenticCompletion(underlying, messages, key, t => { reply.raw.write(chunkOf(id, requested, { content: t }, null)) - }, abort.signal) + }, abort.signal, !!resolved.voice) reply.raw.write(chunkOf(id, requested, {}, r.finish)) rec.status = 'ok' } catch (err) { diff --git a/server/src/routes.ts b/server/src/routes.ts index 2db254c..e0d6c96 100644 --- a/server/src/routes.ts +++ b/server/src/routes.ts @@ -165,6 +165,9 @@ export async function kbRoutes(app: FastifyInstance): Promise { bannerText: eff.bannerText, bannerColor: eff.bannerColor, bannerWhenEmbedded: eff.bannerWhenEmbedded, + // Feature previews a deployment has switched on; the client shows + // their controls only when the flag and its configuration are both present. + features: { voice: { enabled: !!eff.voiceEnabled && !!eff.voiceUrl, url: eff.voiceUrl || '' } }, user, } }) diff --git a/server/src/settings.ts b/server/src/settings.ts index 8d55f44..e7effaf 100644 --- a/server/src/settings.ts +++ b/server/src/settings.ts @@ -54,6 +54,9 @@ export interface StudioSettings { delegationEnabled?: boolean delegationMaxAgents?: number delegationMaxDepth?: number + /** Feature preview: voice conversations through an Unmute deployment. */ + voiceEnabled?: boolean + voiceUrl?: string } let cache: StudioSettings | null = null @@ -108,6 +111,8 @@ export function effectiveSettings(): Required { delegationEnabled: s.delegationEnabled ?? process.env.DELEGATION_ENABLED !== '0', delegationMaxAgents: s.delegationMaxAgents ?? Number(process.env.DELEGATION_MAX_AGENTS ?? 6), delegationMaxDepth: s.delegationMaxDepth ?? Number(process.env.DELEGATION_MAX_DEPTH ?? 1), + voiceEnabled: s.voiceEnabled ?? process.env.VOICE_ENABLED === '1', + voiceUrl: s.voiceUrl ?? process.env.VOICE_URL ?? '', customTools: s.customTools ?? [], ragDefaultModel: s.ragDefaultModel ?? process.env.RAG_DEFAULT_MODEL ?? '', ragTopK: s.ragTopK ?? Math.min(Math.max(Number(process.env.RAG_TOP_K) || 6, 1), 20), @@ -215,6 +220,12 @@ export async function settingsRoutes(app: FastifyInstance): Promise { } } if (body.delegationEnabled !== undefined) next.delegationEnabled = !!body.delegationEnabled + if (body.voiceEnabled !== undefined) next.voiceEnabled = !!body.voiceEnabled + if (body.voiceUrl !== undefined) { + const u = String(body.voiceUrl).trim().replace(/\/+$/, '') + if (u && !/^https?:\/\//.test(u)) throw new KbError(400, 'voice URL must be http(s)') + next.voiceUrl = u || undefined + } if (body.delegationMaxAgents !== undefined) { const n = Number(body.delegationMaxAgents) if (!Number.isFinite(n) || n < 1 || n > 24) throw new KbError(400, 'agents must be 1 to 24') diff --git a/server/test/voicePreview.test.mjs b/server/test/voicePreview.test.mjs new file mode 100644 index 0000000..cbf4199 --- /dev/null +++ b/server/test/voicePreview.test.mjs @@ -0,0 +1,28 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +process.env.KB_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-')) +process.env.INDEX_BASE = fs.mkdtempSync(path.join(os.tmpdir(), 'ix-')) +delete process.env.VOICE_ENABLED; delete process.env.VOICE_URL +const { effectiveSettings } = await import('../dist/settings.js') +const { resolveModel, VOICE_STYLE } = await import('../dist/ragProxy.js') + +test('the voice preview is off until a deployment turns it on, and the env flag can', () => { + assert.equal(effectiveSettings().voiceEnabled, false) + assert.equal(effectiveSettings().voiceUrl, '') + process.env.VOICE_ENABLED = '1'; process.env.VOICE_URL = 'https://unmute.example' + assert.equal(effectiveSettings().voiceEnabled, true) + assert.equal(effectiveSettings().voiceUrl, 'https://unmute.example') + delete process.env.VOICE_ENABLED; delete process.env.VOICE_URL +}) +test('studio-voice is the agent with a spoken-answer style, and keeps an explicit underlying model', () => { + const v = resolveModel('studio-voice') + assert.equal(v.mode, 'agent'); assert.equal(v.voice, true) + const w = resolveModel('studio-voice/me:provider/model-x') + assert.equal(w.underlying, 'me:provider/model-x'); assert.equal(w.voice, true) + assert.equal(resolveModel('studio-agent').voice, undefined) + assert.match(VOICE_STYLE, /one or two short sentences/) + assert.match(VOICE_STYLE, /No markdown/) +}) diff --git a/web/src/components/VoiceOverlay.tsx b/web/src/components/VoiceOverlay.tsx new file mode 100644 index 0000000..3e963c8 --- /dev/null +++ b/web/src/components/VoiceOverlay.tsx @@ -0,0 +1,21 @@ +/** + * Voice conversations, as a feature preview: the Unmute deployment's own + * interface, in an overlay over the chat. Unmute is a separate service + * (speech recognition, turn-taking, synthesis, and the model behind them, + * which is this Studio's own assistant on the /v1 endpoint), so the + * overlay is an iframe with microphone permission delegated to it. The + * transcript stays with Unmute for now; recording it as a Studio + * conversation is the next step once the voice turns flow through here. + */ +export function VoiceOverlay({ url, onClose }: { url: string; onClose: () => void }) { + return ( +
+
+ Voice ยท feature preview + open in a new tab + +
+