From 8d72defb8256c0cf35fc1c36a62333a0aafcd1c0 Mon Sep 17 00:00:00 2001 From: Anais Saraiva Date: Thu, 24 Sep 2026 11:31:43 +0200 Subject: [PATCH] natural voice --- .env.example | 15 ++ README.md | 2 + docs/FRONTEND.md | 4 +- docs/OPERATIONS.md | 32 ++- frontend/src/components/AdminCenter.tsx | 23 +- frontend/src/components/Chat.tsx | 27 ++- frontend/src/components/dialogs.tsx | 66 +++++- frontend/src/lib/api.ts | 24 ++ frontend/src/lib/i18n.ts | 96 ++++++++ frontend/src/lib/voice.ts | 198 +++++++++++++---- prompt_v2 | 10 + src/config.ts | 8 + src/db.ts | 5 + src/index.ts | 57 ++++- src/tools/registry.ts | 5 +- src/tools/tts.ts | 279 ++++++++++++++++++++++++ 16 files changed, 778 insertions(+), 73 deletions(-) create mode 100644 src/tools/tts.ts diff --git a/.env.example b/.env.example index 0ca80ec..dafba8b 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,21 @@ CDN_DIR=./cdn OCR_LANG=eng OCR_MAX_CHARS=100000 +# --- platform tools: neural TTS (first configured wins, TTS_PROVIDER forces one) --- +# ElevenLabs (most natural): https://elevenlabs.io/app/settings/api-keys +#ELEVENLABS_API_KEY= +#ELEVENLABS_VOICE= +# OpenAI (reuses the LLM key when set, tts-1 + alloy/echo/fable/onyx/nova/shimmer) +#OPENAI_API_KEY= +#TTS_VOICE=alloy +# Local piper (offline, like tesseract): PIPER_BIN + onnx model +#PIPER_BIN=piper +#PIPER_MODEL=/var/lib/kisassistant/piper/en_US-lessac-medium.onnx +# Test/development overrides (base URLs, default = api.elevenlabs.io / api.openai.com) +#ELEVENLABS_BASE_URL= +#OPENAI_BASE_URL= +#TTS_PROVIDER=auto + # Per-driver debug logging to stdout ([driver:name] lines). DRIVER_DEBUG=true diff --git a/README.md b/README.md index 30023b0..30cb62e 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ from the **Accounts** panel (each gets a one-time access key to hand over). | GET | `/api/admin/ollama/nodes/:id/status`, `…/stats?window=` | admin | live probe (version, running, VRAM) / load charts 1h–30d | | GET | `/api/admin/ollama/probe?host=` | admin | test a host before registering it | | GET | `/api/admin/platform` | admin | this host facts: hostname, kernel, uptime, load, memory, IPs | +| GET | `/api/tools/tts/info` | Bearer | neural TTS provider + voices (null = browser fallback) | +| POST | `/api/tools/tts` | Bearer | synthesize `{text, voice?}` → audio bytes (cached, 30/hour) | | GET | `/api/admin/ollama/status` | admin | Ollama connectivity probe: reachability, version, on-disk models | | POST | `/api/access/request` | public | reserve username+email with a motivation → ticket (`/review/:id`) | | GET/POST | `/api/access/ticket/:id` (+`/message`) | ticket bearer | follow + reply on an access request | diff --git a/docs/FRONTEND.md b/docs/FRONTEND.md index b6be4a3..a31b79f 100644 --- a/docs/FRONTEND.md +++ b/docs/FRONTEND.md @@ -92,7 +92,9 @@ No router library — a tiny history wrapper (`lib/route.ts`: `navigate()`, also repaints the SVG favicon (data URL) so the tab icon matches the dress. - Voice (`lib/voice.ts`, no dependency): `Chat.tsx` composer mic with live waveform + speech-recognition transcript (editable before send), per-bubble - TTS playback with glow, `Settings → Voice` prefs (voice/lang/speed/pitch, + playback with glow through a unified player (`playText`: neural server + audio via `POST /api/tools/tts` with browser-synthesis fallback), + `Settings → Voice` prefs (engine/lang/voice/speed/pitch/neural voice, localStorage), admin-editable preview lines. ## Theming diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 5df0f6d..0ec580f 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -193,15 +193,31 @@ Disk meters are intentionally absent (models live on the Ollama hosts). it away. Unsupported browsers get a disabled mic with an explanatory tooltip. No audio leaves the browser (no server STT endpoint yet — whisper.cpp via `WHISPER_BIN` is the documented next step). -- **Message playback**: every assistant bubble has a Listen button - (speechSynthesis, Siri-style breathing glow while speaking, click again to - stop; playback stops on send / new message / conversation switch). -- **Settings → Voice** (per browser, localStorage): language (auto-detect - per message via `detectLang`, or forced), voice picker grouped by locale, - speed, pitch, and one-tap previews of the admin-set lines. +- **Message playback**: every assistant bubble has a Listen button with a + Siri-style breathing glow while speaking (click again to stop; playback + stops on send / new message / conversation switch). Engine per user + (Settings → Voice): **auto** (neural server voice when available, browser + fallback), **browser**, or **server**. +- **Neural server TTS** (`POST /api/tools/tts` `{text, voice?}` → audio + bytes, `GET /api/tools/tts/info` → provider + voices): ElevenLabs + (any voice_id, most natural) → OpenAI `tts-1` (6 stock voices, reuses the + LLM key) → local piper (`PIPER_BIN` + `PIPER_MODEL`, offline WAV). + `TTS_PROVIDER` forces one; `TTS_VOICE` / `ELEVENLABS_VOICE` set defaults. + Output is cached on disk (sha of provider|voice|text, 200 files cap), + rate-limited to 30 jobs/hour/account (429), kill-switched via + `tools_tts_enabled` (Admin Center → Features). Test overrides: + `ELEVENLABS_BASE_URL` / `OPENAI_BASE_URL`. +- **Settings → Voice** (per browser, localStorage): engine, language + (auto-detect per message or forced), browser voice/rate/pitch, neural + voice id, and one-tap previews of the admin-set lines. - **Admin Center → Features → Voice previews**: the default playback lines - (`voice_previews` setting, 1–6 lines; empty list resets to the built-ins), - also served to users via `GET /api/tools`. + (`voice_previews` setting, 1–6 lines `{text, lang, rate, pitch, timbre, + voice}`, each with its OWN voice character — masculine/feminine/any + timbre hint, speed, pitch, optional server voice id — so previews sound + different from each other; timbre matching is best-effort over the + browser's voice names, rate/pitch always apply; empty list resets to the + built-ins), also served to users via `GET /api/tools`. Users can adopt + any preview's character as their own playback voice from Settings → Voice. ## Personal providers (BYOK) diff --git a/frontend/src/components/AdminCenter.tsx b/frontend/src/components/AdminCenter.tsx index 0cfd029..8ebd345 100644 --- a/frontend/src/components/AdminCenter.tsx +++ b/frontend/src/components/AdminCenter.tsx @@ -321,6 +321,13 @@ export function AdminCenter({ initialTab }: { initialTab?: string | null }) { toggle({ usernameChangeEnabled: v })} /> +
+
+

{t("center.ttsTitle")}

+

{t("center.ttsDesc")}

+
+ toggle({ ttsEnabled: v })} /> +
{saving &&

{t("common.saving")}

} {!settings && !error &&

{t("common.loading")}

} @@ -444,6 +451,20 @@ function VoicePreviewsCard() { /> + + + patch(i, { voice: e.target.value })} + onBlur={() => commit(i)} + onKeyDown={(e) => { if (e.key === "Enter") (e.target as HTMLInputElement).blur(); }} + /> + +