natural voice

This commit is contained in:
k
2026-09-24 11:31:43 +02:00
parent 562ab5191f
commit 8d72defb82
16 changed files with 778 additions and 73 deletions
+22 -1
View File
@@ -321,6 +321,13 @@ export function AdminCenter({ initialTab }: { initialTab?: string | null }) {
</div>
<Switch label={t("center.usernameChangeTitle")} checked={settings?.usernameChangeEnabled ?? false} onChange={(v) => toggle({ usernameChangeEnabled: v })} />
</div>
<div className="flex items-center gap-4 rounded-3xl border border-stone-200/70 bg-white p-5 shadow-sm dark:border-zinc-800 dark:bg-zinc-900">
<div className="min-w-0 flex-1">
<p className="text-sm font-medium">{t("center.ttsTitle")}</p>
<p className="text-sm opacity-60">{t("center.ttsDesc")}</p>
</div>
<Switch label={t("center.ttsTitle")} checked={settings?.ttsEnabled ?? false} onChange={(v) => toggle({ ttsEnabled: v })} />
</div>
{saving && <p className="flex items-center gap-2 text-sm opacity-60"><Spinner size={14} /> {t("common.saving")}</p>}
{!settings && !error && <p className="flex items-center gap-2 text-sm opacity-60"><Spinner size={14} /> {t("common.loading")}</p>}
<VoicePreviewsCard />
@@ -444,6 +451,20 @@ function VoicePreviewsCard() {
/>
</Field>
</span>
<span className="w-40">
<Field label={t("vfeat.voice")} hint={t("vfeat.voiceHint")}>
<Input
value={line.voice}
disabled={busy}
maxLength={64}
placeholder={t("vfeat.voicePh")}
aria-label={t("vfeat.voice")}
onChange={(e) => patch(i, { voice: e.target.value })}
onBlur={() => commit(i)}
onKeyDown={(e) => { if (e.key === "Enter") (e.target as HTMLInputElement).blur(); }}
/>
</Field>
</span>
<label className="min-w-32 flex-1 text-xs opacity-80">
<span className="mb-1 block font-medium">{t("voice.rate")} · {line.rate.toFixed(2)}×</span>
<input
@@ -478,7 +499,7 @@ function VoicePreviewsCard() {
variant="secondary"
size="sm"
disabled={busy}
onClick={() => setLines((prev) => [...prev, { text: "", lang: "auto", rate: 1, pitch: 1, timbre: "any" as const }])}
onClick={() => setLines((prev) => [...prev, { text: "", lang: "auto", rate: 1, pitch: 1, timbre: "any" as const, voice: "" }])}
>
<Plus size={14} /> {t("vfeat.add")}
</Button>
+18 -9
View File
@@ -7,7 +7,7 @@ import { Markdown } from "./Markdown.tsx";
import { UserMessageContent } from "./MessageContent.tsx";
import { cn } from "../lib/cn.ts";
import { useFeatures } from "../lib/features.ts";
import { fmtDuration, getVoicePrefs, listTtsVoices, recognizerCtor, speak, stopSpeak, sttSupported, type Recognizer } from "../lib/voice.ts";
import { fmtDuration, getVoicePrefs, listTtsVoices, playText, recognizerCtor, stopPlayback, sttSupported, type Recognizer } from "../lib/voice.ts";
import { useT, type StringKey } from "../lib/i18n.ts";
const REPORT_REASONS: ReportReason[] = ["copyright", "gore", "falseinfo", "bug"];
@@ -279,6 +279,7 @@ export function Chat({
// TTS playback: which assistant bubble is speaking (null = silent).
const [speakingKey, setSpeakingKey] = useState<number | null>(null);
const voicesRef = useRef<SpeechSynthesisVoice[]>([]);
const ttsServerRef = useRef(false);
// Voice input: live recording session (null = composer idle).
const [rec, setRec] = useState<{
stream: MediaStream;
@@ -296,12 +297,13 @@ export function Chat({
useEffect(() => {
listTtsVoices().then((v) => { voicesRef.current = v; });
return () => stopSpeak();
api.ttsInfo().then((r) => { ttsServerRef.current = r.available; }).catch(() => {});
return () => stopPlayback();
}, []);
// New message / conversation: stop any playback (never leak speech).
useEffect(() => {
stopSpeak();
stopPlayback();
setSpeakingKey(null);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [conv?.id]);
@@ -309,22 +311,29 @@ export function Chat({
useEffect(() => {
return () => {
cancelRec(false);
stopSpeak();
stopPlayback();
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
function toggleSpeak(index: number, text: string) {
if (speakingKey === index) {
stopSpeak();
stopPlayback();
setSpeakingKey(null);
return;
}
const ok = speak(text, getVoicePrefs(), voicesRef.current, {
const prefs = getVoicePrefs();
playText({
text,
engine: prefs.engine,
browser: { voices: voicesRef.current, lang: prefs.lang, rate: prefs.rate, pitch: prefs.pitch, voiceURI: prefs.voiceURI },
serverVoice: prefs.serverVoice ?? undefined,
serverAvailable: ttsServerRef.current,
fetchAudio: (t, v) => api.ttsSpeak(t, v),
onend: () => setSpeakingKey(null),
onerror: () => setSpeakingKey(null),
});
if (ok) setSpeakingKey(index);
setSpeakingKey(index);
}
function cleanupRec(r: NonNullable<typeof rec>) {
@@ -446,7 +455,7 @@ export function Chat({
// A fresh message ends any playback (the glow never sticks to a stale row).
const msgCount = messages.length;
useEffect(() => {
stopSpeak();
stopPlayback();
setSpeakingKey(null);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [msgCount]);
@@ -464,7 +473,7 @@ export function Chat({
if (readOnly) return;
const text = draft.trim();
if (!text || sending) return;
stopSpeak();
stopPlayback();
setSpeakingKey(null);
setDraft("");
requestAnimationFrame(autoGrow);
+59 -7
View File
@@ -4,7 +4,7 @@ import { api, type UserProvider, type VoicePreview } from "../lib/api.ts";
import { pushToast } from "../lib/toasts.ts";
import { cn } from "../lib/cn.ts";
import { ACCENT_PRESETS, applyAccent, currentAccent, previewAccent, type AccentState } from "../lib/accent.ts";
import { detectLang, getVoicePrefs, listTtsVoices, pickVoice, setVoicePrefs, speakPreview, stopSpeak, ttsSupported, type VoicePrefs } from "../lib/voice.ts";
import { detectLang, getVoicePrefs, listTtsVoices, pickVoice, playText, setVoicePrefs, stopPlayback, ttsSupported, type VoicePrefs } from "../lib/voice.ts";
import { FEATURES, setFeature, useFeatures } from "../lib/features.ts";
import type { Conversation, FilePreview, User } from "../lib/types.ts";
import { useT, type StringKey } from "../lib/i18n.ts";
@@ -877,12 +877,14 @@ function VoiceSection() {
const [voices, setVoices] = useState<SpeechSynthesisVoice[]>([]);
const [previews, setPreviews] = useState<VoicePreview[]>([]);
const [playing, setPlaying] = useState<number | null>(null);
const [ttsServer, setTtsServer] = useState<{ available: boolean; provider: string | null; voices: string[] } | null>(null);
useEffect(() => {
let alive = true;
listTtsVoices().then((v) => { if (alive) setVoices(v); });
api.tools().then((r) => { if (alive) setPreviews(r.voicePreviews); }).catch(() => {});
return () => { alive = false; stopSpeak(); };
api.ttsInfo().then((r) => { if (alive) setTtsServer({ available: r.available, provider: r.provider, voices: r.voices }); }).catch(() => {});
return () => { alive = false; stopPlayback(); };
}, []);
function update(p: Partial<VoicePrefs>) {
@@ -893,17 +895,23 @@ function VoiceSection() {
function preview(i: number, line: VoicePreview) {
if (playing === i) {
stopSpeak();
stopPlayback();
setPlaying(null);
return;
}
// Each line plays with ITS OWN character (timbre/rate/pitch/lang) —
// never the global playback prefs.
const pick = speakPreview(line.text, line, voices, {
// Each line plays with ITS OWN character; the neural server engine is
// used when selected and available (line.server voice or user default).
playText({
text: line.text,
engine: prefs.engine,
browser: { voices, lang: line.lang, rate: line.rate, pitch: line.pitch, timbre: line.timbre },
serverVoice: line.voice || prefs.serverVoice || undefined,
serverAvailable: ttsServer?.available ?? false,
fetchAudio: (text, voice) => api.ttsSpeak(text, voice),
onend: () => setPlaying(null),
onerror: () => setPlaying(null),
});
if (pick || ttsSupported()) setPlaying(i);
setPlaying(i);
}
/** Adopt a preview's character as the global playback voice. */
@@ -911,6 +919,7 @@ function VoiceSection() {
const lang = line.lang === "auto" ? detectLang(line.text) : line.lang;
const pick = pickVoice(voices, { lang, timbre: line.timbre });
const next: VoicePrefs = {
...prefs,
voiceURI: pick?.voiceURI ?? prefs.voiceURI,
rate: line.rate,
pitch: line.pitch,
@@ -954,6 +963,49 @@ function VoiceSection() {
options={VOICE_LANGS.map((l) => ({ value: l, label: l === "auto" ? t("voice.auto") : l }))}
/>
</Field>
<Field label={t("voice.engine")} hint={ttsServer?.available ? t("voice.engineHintOn", { provider: ttsServer.provider ?? "?" }) : t("voice.engineHintOff")}>
<Picker
ariaLabel={t("voice.engine")}
icon={Volume2}
value={prefs.engine}
onChange={(v) => update({ engine: v as VoicePrefs["engine"] })}
align="left"
options={[
{ value: "auto", label: t("voice.engineAuto") },
{ value: "browser", label: t("voice.engineBrowser") },
{ value: "server", label: t("voice.engineServer") },
]}
/>
</Field>
{(prefs.engine !== "browser") && (
ttsServer?.available ? (
ttsServer.provider === "openai" ? (
<Field label={t("voice.serverVoice")}>
<Picker
ariaLabel={t("voice.serverVoice")}
icon={Volume2}
value={prefs.serverVoice ?? ""}
onChange={(v) => update({ serverVoice: v || null })}
align="left"
options={[{ value: "", label: t("voice.defaultVoice") }, ...ttsServer.voices.map((v) => ({ value: v, label: v }))]}
/>
</Field>
) : ttsServer.provider === "elevenlabs" ? (
<Field label={t("voice.serverVoice")} hint={t("voice.serverVoiceHint")}>
<Input
value={prefs.serverVoice ?? ""}
onChange={(e) => update({ serverVoice: e.target.value.trim() || null })}
placeholder="pNInz6obpgDQGcFmaJgB"
maxLength={64}
/>
</Field>
) : (
<p className="rounded-2xl bg-stone-100 px-4 py-2.5 text-sm opacity-70 dark:bg-zinc-800">{t("voice.piperNote")}</p>
)
) : (
<p className="rounded-2xl bg-amber-500/10 px-4 py-2.5 text-sm text-amber-700 dark:text-amber-400">{t("voice.noServer")}</p>
)
)}
<Field label={t("voice.voice")}>
<Picker
ariaLabel={t("voice.voice")}
+24
View File
@@ -74,6 +74,7 @@ export interface VoicePreview {
rate: number;
pitch: number;
timbre: "masculine" | "feminine" | "any";
voice: string;
}
export interface PlatformInfo {
@@ -102,6 +103,7 @@ export interface AdminSettings {
recoveryLimitMax: number;
recoveryLimitWindowMin: number;
voicePreviews: VoicePreview[];
ttsEnabled: boolean;
}
export interface AdminSettingsPatch {
@@ -115,6 +117,7 @@ export interface AdminSettingsPatch {
recoveryLimitMax?: number;
recoveryLimitWindowMin?: number;
voicePreviews?: VoicePreview[];
ttsEnabled?: boolean;
}
export interface OllamaNode {
@@ -442,6 +445,27 @@ export const api = {
tools: () => req<{ tools: { name: string; description: string; available: boolean; reason: string | null }[]; reportsEnabled: boolean; usernameChangeEnabled: boolean; voicePreviews: VoicePreview[] }>("/api/tools"),
/** Neural TTS provider status (null = browser speech fallback). */
ttsInfo: () => req<{ provider: "elevenlabs" | "openai" | "piper" | null; voices: string[]; available: boolean; reason: string | null }>("/api/tools/tts/info"),
/** Synthesize text server-side (neural voice). Returns audio bytes. */
ttsSpeak: async (text: string, voice?: string): Promise<Blob> => {
const res = await fetch("/api/tools/tts", {
method: "POST",
headers: { "Content-Type": "application/json", Authorization: `Bearer ${getToken()}` },
body: JSON.stringify({ text, ...(voice ? { voice } : {}) }),
});
if (res.status === 401) {
clearToken();
window.location.reload();
throw new ApiError(401, "Session expired.");
}
if (!res.ok) {
const data = (await res.json().catch(() => ({}))) as { error?: string };
throw new ApiError(res.status, data.error || `TTS failed (${res.status})`);
}
return await res.blob();
},
/** Send an image to the platform OCR tool. Returns editable text (never the raw image). */
ocrImage: async (file: File): Promise<{ text: string; truncated: boolean; chars: number }> => {
const res = await fetch("/api/tools/ocr", {
+96
View File
@@ -605,6 +605,22 @@ const en = {
"vfeat.timbre": "Timbre",
"vfeat.rate": "Speed",
"vfeat.pitch": "Pitch",
"vfeat.voice": "Server voice",
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
"vfeat.voicePh": "auto",
"center.ttsTitle": "Neural voice playback",
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
"voice.engine": "Playback engine",
"voice.engineAuto": "Auto (neural if available)",
"voice.engineBrowser": "Browser speech",
"voice.engineServer": "Neural server voice",
"voice.engineHintOn": "Neural TTS via {provider}",
"voice.engineHintOff": "No neural provider configured — browser speech only",
"voice.serverVoice": "Neural voice",
"voice.serverVoiceHint": "ElevenLabs voice id",
"voice.piperNote": "Local piper voice (single configured model).",
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
"msg.copy": "Copy message",
@@ -1297,6 +1313,22 @@ const es: Dict = {
"vfeat.timbre": "Timbre",
"vfeat.rate": "Speed",
"vfeat.pitch": "Pitch",
"vfeat.voice": "Server voice",
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
"vfeat.voicePh": "auto",
"center.ttsTitle": "Neural voice playback",
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
"voice.engine": "Playback engine",
"voice.engineAuto": "Auto (neural if available)",
"voice.engineBrowser": "Browser speech",
"voice.engineServer": "Neural server voice",
"voice.engineHintOn": "Neural TTS via {provider}",
"voice.engineHintOff": "No neural provider configured — browser speech only",
"voice.serverVoice": "Neural voice",
"voice.serverVoiceHint": "ElevenLabs voice id",
"voice.piperNote": "Local piper voice (single configured model).",
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
"msg.copy": "Copiar mensaje",
@@ -1984,6 +2016,22 @@ const fr: Dict = {
"vfeat.timbre": "Timbre",
"vfeat.rate": "Speed",
"vfeat.pitch": "Pitch",
"vfeat.voice": "Server voice",
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
"vfeat.voicePh": "auto",
"center.ttsTitle": "Neural voice playback",
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
"voice.engine": "Playback engine",
"voice.engineAuto": "Auto (neural if available)",
"voice.engineBrowser": "Browser speech",
"voice.engineServer": "Neural server voice",
"voice.engineHintOn": "Neural TTS via {provider}",
"voice.engineHintOff": "No neural provider configured — browser speech only",
"voice.serverVoice": "Neural voice",
"voice.serverVoiceHint": "ElevenLabs voice id",
"voice.piperNote": "Local piper voice (single configured model).",
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
"msg.copy": "Copier le message",
@@ -2659,6 +2707,22 @@ const ru: Dict = {
"vfeat.timbre": "Timbre",
"vfeat.rate": "Speed",
"vfeat.pitch": "Pitch",
"vfeat.voice": "Server voice",
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
"vfeat.voicePh": "auto",
"center.ttsTitle": "Neural voice playback",
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
"voice.engine": "Playback engine",
"voice.engineAuto": "Auto (neural if available)",
"voice.engineBrowser": "Browser speech",
"voice.engineServer": "Neural server voice",
"voice.engineHintOn": "Neural TTS via {provider}",
"voice.engineHintOff": "No neural provider configured — browser speech only",
"voice.serverVoice": "Neural voice",
"voice.serverVoiceHint": "ElevenLabs voice id",
"voice.piperNote": "Local piper voice (single configured model).",
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
"msg.copy": "Скопировать сообщение",
@@ -3339,6 +3403,22 @@ const it: Dict = {
"vfeat.timbre": "Timbre",
"vfeat.rate": "Speed",
"vfeat.pitch": "Pitch",
"vfeat.voice": "Server voice",
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
"vfeat.voicePh": "auto",
"center.ttsTitle": "Neural voice playback",
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
"voice.engine": "Playback engine",
"voice.engineAuto": "Auto (neural if available)",
"voice.engineBrowser": "Browser speech",
"voice.engineServer": "Neural server voice",
"voice.engineHintOn": "Neural TTS via {provider}",
"voice.engineHintOff": "No neural provider configured — browser speech only",
"voice.serverVoice": "Neural voice",
"voice.serverVoiceHint": "ElevenLabs voice id",
"voice.piperNote": "Local piper voice (single configured model).",
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
"msg.copy": "Copia messaggio",
@@ -4010,6 +4090,22 @@ const pt: Dict = {
"vfeat.timbre": "Timbre",
"vfeat.rate": "Speed",
"vfeat.pitch": "Pitch",
"vfeat.voice": "Server voice",
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
"vfeat.voicePh": "auto",
"center.ttsTitle": "Neural voice playback",
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
"voice.engine": "Playback engine",
"voice.engineAuto": "Auto (neural if available)",
"voice.engineBrowser": "Browser speech",
"voice.engineServer": "Neural server voice",
"voice.engineHintOn": "Neural TTS via {provider}",
"voice.engineHintOff": "No neural provider configured — browser speech only",
"voice.serverVoice": "Neural voice",
"voice.serverVoiceHint": "ElevenLabs voice id",
"voice.piperNote": "Local piper voice (single configured model).",
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
"msg.copy": "Copiar mensagem",
+159 -39
View File
@@ -15,21 +15,28 @@ export interface VoicePrefs {
pitch: number;
/** "auto" detects per message, otherwise forces a BCP-47 prefix. */
lang: string;
/** Playback engine: neural server TTS when available, else browser. */
engine: "auto" | "browser" | "server";
/** Server neural voice id (ElevenLabs voice_id / OpenAI voice name). */
serverVoice: string | null;
}
const PREFS_KEY = "ka:voice";
export const DEFAULT_VOICE_PREFS: VoicePrefs = { voiceURI: null, rate: 1, pitch: 1, lang: "auto" };
export const DEFAULT_VOICE_PREFS: VoicePrefs = { voiceURI: null, rate: 1, pitch: 1, lang: "auto", engine: "auto", serverVoice: null };
export function getVoicePrefs(): VoicePrefs {
try {
const raw = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "null") as Partial<VoicePrefs> | null;
if (!raw || typeof raw !== "object") return { ...DEFAULT_VOICE_PREFS };
const engine = raw.engine === "browser" || raw.engine === "server" ? raw.engine : "auto";
return {
voiceURI: typeof raw.voiceURI === "string" ? raw.voiceURI : null,
rate: typeof raw.rate === "number" && raw.rate >= 0.5 && raw.rate <= 2 ? raw.rate : 1,
pitch: typeof raw.pitch === "number" && raw.pitch >= 0.5 && raw.pitch <= 2 ? raw.pitch : 1,
lang: typeof raw.lang === "string" && raw.lang.length <= 12 ? raw.lang : "auto",
engine,
serverVoice: typeof raw.serverVoice === "string" && raw.serverVoice.length > 0 ? raw.serverVoice : null,
};
} catch {
return { ...DEFAULT_VOICE_PREFS };
@@ -83,13 +90,6 @@ export interface SpeakOpts {
export type VoiceTimbre = "masculine" | "feminine" | "any";
export interface PreviewVoice {
lang: string;
rate: number;
pitch: number;
timbre: VoiceTimbre;
}
// Name fragments hinting a masculine / feminine voice (Chrome, Edge,
// Safari and Android voices carry such markers; absence falls back to
// any same-language voice — never an error).
@@ -158,43 +158,163 @@ export function stopSpeak(): void {
}
}
/**
* Play a preview line with ITS OWN voice character (timbre + rate + pitch),
* never the user's global playback prefs — previews must sound different
* from each other. Returns the resolved voice (null when unsupported) so
* callers can offer "use this voice" adoption.
*/
export function speakPreview(
text: string,
preview: PreviewVoice,
voices: SpeechSynthesisVoice[],
opts: SpeakOpts = {},
): SpeechSynthesisVoice | null {
if (!ttsSupported()) return null;
const synth = window.speechSynthesis;
// --- unified playback (neural server audio with browser fallback) ---
export interface PlayHandle {
/** Engine that ended up playing (may differ from the request in auto). */
engine: "server" | "browser";
stop: () => void;
done: Promise<void>;
}
let currentAudio: HTMLAudioElement | null = null;
function stopCurrentAudio(): void {
if (currentAudio) {
try {
currentAudio.pause();
const url = currentAudio.src;
currentAudio.removeAttribute("src");
currentAudio.load();
if (url.startsWith("blob:")) URL.revokeObjectURL(url);
} catch {
// ignore
}
currentAudio = null;
}
try {
synth.cancel();
window.speechSynthesis?.cancel();
} catch {
// ignore
}
const lang = preview.lang === "auto" ? detectLang(text) : preview.lang;
const utter = new SpeechSynthesisUtterance(text.slice(0, 2000));
utter.lang = lang;
utter.rate = preview.rate;
utter.pitch = preview.pitch;
const pick = pickVoice(voices, { lang, timbre: preview.timbre });
if (pick) utter.voice = pick;
utter.onend = () => opts.onend?.();
utter.onerror = () => opts.onerror?.("error");
try {
synth.speak(utter);
} catch {
return null;
}
return pick;
}
// --- speech recognition (STT) ---
/** Stop whatever is playing (server audio or browser speech). */
export function stopPlayback(): void {
stopCurrentAudio();
}
export interface PlayRequest {
text: string;
/** "server" forces neural, "browser" forces synthesis, "auto" tries neural first. */
engine: "auto" | "browser" | "server";
browser: {
voices: SpeechSynthesisVoice[];
/** "auto" detects per text, otherwise a BCP-47 prefix. */
lang: string;
rate: number;
pitch: number;
/** Explicit voice URI (user pref); otherwise timbre-matched. */
voiceURI?: string | null;
/** Preview character timbre (ignored when voiceURI is set). */
timbre?: VoiceTimbre;
};
/** Server neural voice id (preview line voice or user serverVoice). */
serverVoice?: string | null;
/** True when the platform TTS tool is available. */
serverAvailable: boolean;
fetchAudio: (text: string, voice?: string) => Promise<Blob>;
onstart?: () => void;
onend?: () => void;
onerror?: (msg: string) => void;
}
/**
* Play one text with the best engine: neural server audio when requested
* and available (shared <audio> element, user speed), otherwise the
* browser character voice. Single global playback — a new play stops the
* previous one. `done` resolves when playback fully ends (fallback
* included); it never rejects (errors go to onend/onerror like speech).
*/
export function playText(req: PlayRequest): PlayHandle {
stopPlayback();
const wantServer = req.serverAvailable && req.engine !== "browser";
if (wantServer) return playServer(req, req.engine === "auto");
return playBrowser(req);
}
function playServer(req: PlayRequest, allowFallback: boolean): PlayHandle {
let settled = false;
const stop = () => {
settled = true;
stopCurrentAudio();
};
const done = (async () => {
try {
const blob = await req.fetchAudio(req.text.slice(0, 2000), req.serverVoice ?? undefined);
if (settled) return;
await playAudioBlob(blob, req, () => settled);
} catch (e) {
if (settled) return;
if (allowFallback) {
await playBrowserAudio(req, () => settled);
return;
}
req.onerror?.(e instanceof Error ? e.message : "playback failed");
}
})();
return { engine: "server", stop, done };
}
function playAudioBlob(blob: Blob, req: PlayRequest, cancelled: () => boolean): Promise<void> {
return new Promise((resolve) => {
const url = URL.createObjectURL(blob);
const audio = new Audio(url);
audio.playbackRate = req.browser.rate;
currentAudio = audio;
const finish = (ok: boolean) => {
if (currentAudio === audio) stopCurrentAudio();
else {
try {
URL.revokeObjectURL(url);
} catch {
// ignore
}
}
if (!cancelled() && ok) req.onend?.();
resolve();
};
audio.onended = () => finish(true);
audio.onerror = () => finish(false);
req.onstart?.();
audio.play().catch(() => finish(false));
});
}
function playBrowser(req: PlayRequest): PlayHandle {
let settled = false;
const stop = () => {
settled = true;
stopSpeak();
};
const done = playBrowserAudio(req, () => settled);
return { engine: "browser", stop, done };
}
function playBrowserAudio(req: PlayRequest, cancelled: () => boolean): Promise<void> {
return new Promise((resolve) => {
const b = req.browser;
const lang = b.lang === "auto" ? detectLang(req.text) : b.lang;
const explicit = b.voiceURI ? b.voices.find((v) => v.voiceURI === b.voiceURI) : undefined;
const pick = explicit ?? pickVoice(b.voices, { lang, timbre: b.timbre ?? "any" });
const finish = (ok: boolean, msg?: string) => {
if (cancelled()) {
resolve();
return;
}
if (ok) req.onend?.();
else req.onerror?.(msg ?? "error");
resolve();
};
const ok = speak(
req.text,
{ voiceURI: pick?.voiceURI ?? null, rate: b.rate, pitch: b.pitch, lang, engine: "browser", serverVoice: null },
b.voices,
{ onstart: () => req.onstart?.(), onend: () => finish(true), onerror: (m) => finish(false, m) },
);
if (!ok) finish(false, "unsupported");
});
}
export interface Recognizer {
lang: string;