natural voice
This commit is contained in:
@@ -25,6 +25,21 @@ CDN_DIR=./cdn
|
|||||||
OCR_LANG=eng
|
OCR_LANG=eng
|
||||||
OCR_MAX_CHARS=100000
|
OCR_MAX_CHARS=100000
|
||||||
|
|
||||||
|
# --- platform tools: neural TTS (first configured wins, TTS_PROVIDER forces one) ---
|
||||||
|
# ElevenLabs (most natural): https://elevenlabs.io/app/settings/api-keys
|
||||||
|
#ELEVENLABS_API_KEY=
|
||||||
|
#ELEVENLABS_VOICE=
|
||||||
|
# OpenAI (reuses the LLM key when set, tts-1 + alloy/echo/fable/onyx/nova/shimmer)
|
||||||
|
#OPENAI_API_KEY=
|
||||||
|
#TTS_VOICE=alloy
|
||||||
|
# Local piper (offline, like tesseract): PIPER_BIN + onnx model
|
||||||
|
#PIPER_BIN=piper
|
||||||
|
#PIPER_MODEL=/var/lib/kisassistant/piper/en_US-lessac-medium.onnx
|
||||||
|
# Test/development overrides (base URLs, default = api.elevenlabs.io / api.openai.com)
|
||||||
|
#ELEVENLABS_BASE_URL=
|
||||||
|
#OPENAI_BASE_URL=
|
||||||
|
#TTS_PROVIDER=auto
|
||||||
|
|
||||||
# Per-driver debug logging to stdout ([driver:name] lines).
|
# Per-driver debug logging to stdout ([driver:name] lines).
|
||||||
DRIVER_DEBUG=true
|
DRIVER_DEBUG=true
|
||||||
|
|
||||||
|
|||||||
@@ -68,6 +68,8 @@ from the **Accounts** panel (each gets a one-time access key to hand over).
|
|||||||
| GET | `/api/admin/ollama/nodes/:id/status`, `…/stats?window=` | admin | live probe (version, running, VRAM) / load charts 1h–30d |
|
| GET | `/api/admin/ollama/nodes/:id/status`, `…/stats?window=` | admin | live probe (version, running, VRAM) / load charts 1h–30d |
|
||||||
| GET | `/api/admin/ollama/probe?host=` | admin | test a host before registering it |
|
| GET | `/api/admin/ollama/probe?host=` | admin | test a host before registering it |
|
||||||
| GET | `/api/admin/platform` | admin | this host facts: hostname, kernel, uptime, load, memory, IPs |
|
| GET | `/api/admin/platform` | admin | this host facts: hostname, kernel, uptime, load, memory, IPs |
|
||||||
|
| GET | `/api/tools/tts/info` | Bearer | neural TTS provider + voices (null = browser fallback) |
|
||||||
|
| POST | `/api/tools/tts` | Bearer | synthesize `{text, voice?}` → audio bytes (cached, 30/hour) |
|
||||||
| GET | `/api/admin/ollama/status` | admin | Ollama connectivity probe: reachability, version, on-disk models |
|
| GET | `/api/admin/ollama/status` | admin | Ollama connectivity probe: reachability, version, on-disk models |
|
||||||
| POST | `/api/access/request` | public | reserve username+email with a motivation → ticket (`/review/:id`) |
|
| POST | `/api/access/request` | public | reserve username+email with a motivation → ticket (`/review/:id`) |
|
||||||
| GET/POST | `/api/access/ticket/:id` (+`/message`) | ticket bearer | follow + reply on an access request |
|
| GET/POST | `/api/access/ticket/:id` (+`/message`) | ticket bearer | follow + reply on an access request |
|
||||||
|
|||||||
+3
-1
@@ -92,7 +92,9 @@ No router library — a tiny history wrapper (`lib/route.ts`: `navigate()`,
|
|||||||
also repaints the SVG favicon (data URL) so the tab icon matches the dress.
|
also repaints the SVG favicon (data URL) so the tab icon matches the dress.
|
||||||
- Voice (`lib/voice.ts`, no dependency): `Chat.tsx` composer mic with live
|
- Voice (`lib/voice.ts`, no dependency): `Chat.tsx` composer mic with live
|
||||||
waveform + speech-recognition transcript (editable before send), per-bubble
|
waveform + speech-recognition transcript (editable before send), per-bubble
|
||||||
TTS playback with glow, `Settings → Voice` prefs (voice/lang/speed/pitch,
|
playback with glow through a unified player (`playText`: neural server
|
||||||
|
audio via `POST /api/tools/tts` with browser-synthesis fallback),
|
||||||
|
`Settings → Voice` prefs (engine/lang/voice/speed/pitch/neural voice,
|
||||||
localStorage), admin-editable preview lines.
|
localStorage), admin-editable preview lines.
|
||||||
|
|
||||||
## Theming
|
## Theming
|
||||||
|
|||||||
+24
-8
@@ -193,15 +193,31 @@ Disk meters are intentionally absent (models live on the Ollama hosts).
|
|||||||
it away. Unsupported browsers get a disabled mic with an explanatory
|
it away. Unsupported browsers get a disabled mic with an explanatory
|
||||||
tooltip. No audio leaves the browser (no server STT endpoint yet —
|
tooltip. No audio leaves the browser (no server STT endpoint yet —
|
||||||
whisper.cpp via `WHISPER_BIN` is the documented next step).
|
whisper.cpp via `WHISPER_BIN` is the documented next step).
|
||||||
- **Message playback**: every assistant bubble has a Listen button
|
- **Message playback**: every assistant bubble has a Listen button with a
|
||||||
(speechSynthesis, Siri-style breathing glow while speaking, click again to
|
Siri-style breathing glow while speaking (click again to stop; playback
|
||||||
stop; playback stops on send / new message / conversation switch).
|
stops on send / new message / conversation switch). Engine per user
|
||||||
- **Settings → Voice** (per browser, localStorage): language (auto-detect
|
(Settings → Voice): **auto** (neural server voice when available, browser
|
||||||
per message via `detectLang`, or forced), voice picker grouped by locale,
|
fallback), **browser**, or **server**.
|
||||||
speed, pitch, and one-tap previews of the admin-set lines.
|
- **Neural server TTS** (`POST /api/tools/tts` `{text, voice?}` → audio
|
||||||
|
bytes, `GET /api/tools/tts/info` → provider + voices): ElevenLabs
|
||||||
|
(any voice_id, most natural) → OpenAI `tts-1` (6 stock voices, reuses the
|
||||||
|
LLM key) → local piper (`PIPER_BIN` + `PIPER_MODEL`, offline WAV).
|
||||||
|
`TTS_PROVIDER` forces one; `TTS_VOICE` / `ELEVENLABS_VOICE` set defaults.
|
||||||
|
Output is cached on disk (sha of provider|voice|text, 200 files cap),
|
||||||
|
rate-limited to 30 jobs/hour/account (429), kill-switched via
|
||||||
|
`tools_tts_enabled` (Admin Center → Features). Test overrides:
|
||||||
|
`ELEVENLABS_BASE_URL` / `OPENAI_BASE_URL`.
|
||||||
|
- **Settings → Voice** (per browser, localStorage): engine, language
|
||||||
|
(auto-detect per message or forced), browser voice/rate/pitch, neural
|
||||||
|
voice id, and one-tap previews of the admin-set lines.
|
||||||
- **Admin Center → Features → Voice previews**: the default playback lines
|
- **Admin Center → Features → Voice previews**: the default playback lines
|
||||||
(`voice_previews` setting, 1–6 lines; empty list resets to the built-ins),
|
(`voice_previews` setting, 1–6 lines `{text, lang, rate, pitch, timbre,
|
||||||
also served to users via `GET /api/tools`.
|
voice}`, each with its OWN voice character — masculine/feminine/any
|
||||||
|
timbre hint, speed, pitch, optional server voice id — so previews sound
|
||||||
|
different from each other; timbre matching is best-effort over the
|
||||||
|
browser's voice names, rate/pitch always apply; empty list resets to the
|
||||||
|
built-ins), also served to users via `GET /api/tools`. Users can adopt
|
||||||
|
any preview's character as their own playback voice from Settings → Voice.
|
||||||
|
|
||||||
## Personal providers (BYOK)
|
## Personal providers (BYOK)
|
||||||
|
|
||||||
|
|||||||
@@ -321,6 +321,13 @@ export function AdminCenter({ initialTab }: { initialTab?: string | null }) {
|
|||||||
</div>
|
</div>
|
||||||
<Switch label={t("center.usernameChangeTitle")} checked={settings?.usernameChangeEnabled ?? false} onChange={(v) => toggle({ usernameChangeEnabled: v })} />
|
<Switch label={t("center.usernameChangeTitle")} checked={settings?.usernameChangeEnabled ?? false} onChange={(v) => toggle({ usernameChangeEnabled: v })} />
|
||||||
</div>
|
</div>
|
||||||
|
<div className="flex items-center gap-4 rounded-3xl border border-stone-200/70 bg-white p-5 shadow-sm dark:border-zinc-800 dark:bg-zinc-900">
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<p className="text-sm font-medium">{t("center.ttsTitle")}</p>
|
||||||
|
<p className="text-sm opacity-60">{t("center.ttsDesc")}</p>
|
||||||
|
</div>
|
||||||
|
<Switch label={t("center.ttsTitle")} checked={settings?.ttsEnabled ?? false} onChange={(v) => toggle({ ttsEnabled: v })} />
|
||||||
|
</div>
|
||||||
{saving && <p className="flex items-center gap-2 text-sm opacity-60"><Spinner size={14} /> {t("common.saving")}</p>}
|
{saving && <p className="flex items-center gap-2 text-sm opacity-60"><Spinner size={14} /> {t("common.saving")}</p>}
|
||||||
{!settings && !error && <p className="flex items-center gap-2 text-sm opacity-60"><Spinner size={14} /> {t("common.loading")}</p>}
|
{!settings && !error && <p className="flex items-center gap-2 text-sm opacity-60"><Spinner size={14} /> {t("common.loading")}</p>}
|
||||||
<VoicePreviewsCard />
|
<VoicePreviewsCard />
|
||||||
@@ -444,6 +451,20 @@ function VoicePreviewsCard() {
|
|||||||
/>
|
/>
|
||||||
</Field>
|
</Field>
|
||||||
</span>
|
</span>
|
||||||
|
<span className="w-40">
|
||||||
|
<Field label={t("vfeat.voice")} hint={t("vfeat.voiceHint")}>
|
||||||
|
<Input
|
||||||
|
value={line.voice}
|
||||||
|
disabled={busy}
|
||||||
|
maxLength={64}
|
||||||
|
placeholder={t("vfeat.voicePh")}
|
||||||
|
aria-label={t("vfeat.voice")}
|
||||||
|
onChange={(e) => patch(i, { voice: e.target.value })}
|
||||||
|
onBlur={() => commit(i)}
|
||||||
|
onKeyDown={(e) => { if (e.key === "Enter") (e.target as HTMLInputElement).blur(); }}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
</span>
|
||||||
<label className="min-w-32 flex-1 text-xs opacity-80">
|
<label className="min-w-32 flex-1 text-xs opacity-80">
|
||||||
<span className="mb-1 block font-medium">{t("voice.rate")} · {line.rate.toFixed(2)}×</span>
|
<span className="mb-1 block font-medium">{t("voice.rate")} · {line.rate.toFixed(2)}×</span>
|
||||||
<input
|
<input
|
||||||
@@ -478,7 +499,7 @@ function VoicePreviewsCard() {
|
|||||||
variant="secondary"
|
variant="secondary"
|
||||||
size="sm"
|
size="sm"
|
||||||
disabled={busy}
|
disabled={busy}
|
||||||
onClick={() => setLines((prev) => [...prev, { text: "", lang: "auto", rate: 1, pitch: 1, timbre: "any" as const }])}
|
onClick={() => setLines((prev) => [...prev, { text: "", lang: "auto", rate: 1, pitch: 1, timbre: "any" as const, voice: "" }])}
|
||||||
>
|
>
|
||||||
<Plus size={14} /> {t("vfeat.add")}
|
<Plus size={14} /> {t("vfeat.add")}
|
||||||
</Button>
|
</Button>
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import { Markdown } from "./Markdown.tsx";
|
|||||||
import { UserMessageContent } from "./MessageContent.tsx";
|
import { UserMessageContent } from "./MessageContent.tsx";
|
||||||
import { cn } from "../lib/cn.ts";
|
import { cn } from "../lib/cn.ts";
|
||||||
import { useFeatures } from "../lib/features.ts";
|
import { useFeatures } from "../lib/features.ts";
|
||||||
import { fmtDuration, getVoicePrefs, listTtsVoices, recognizerCtor, speak, stopSpeak, sttSupported, type Recognizer } from "../lib/voice.ts";
|
import { fmtDuration, getVoicePrefs, listTtsVoices, playText, recognizerCtor, stopPlayback, sttSupported, type Recognizer } from "../lib/voice.ts";
|
||||||
import { useT, type StringKey } from "../lib/i18n.ts";
|
import { useT, type StringKey } from "../lib/i18n.ts";
|
||||||
|
|
||||||
const REPORT_REASONS: ReportReason[] = ["copyright", "gore", "falseinfo", "bug"];
|
const REPORT_REASONS: ReportReason[] = ["copyright", "gore", "falseinfo", "bug"];
|
||||||
@@ -279,6 +279,7 @@ export function Chat({
|
|||||||
// TTS playback: which assistant bubble is speaking (null = silent).
|
// TTS playback: which assistant bubble is speaking (null = silent).
|
||||||
const [speakingKey, setSpeakingKey] = useState<number | null>(null);
|
const [speakingKey, setSpeakingKey] = useState<number | null>(null);
|
||||||
const voicesRef = useRef<SpeechSynthesisVoice[]>([]);
|
const voicesRef = useRef<SpeechSynthesisVoice[]>([]);
|
||||||
|
const ttsServerRef = useRef(false);
|
||||||
// Voice input: live recording session (null = composer idle).
|
// Voice input: live recording session (null = composer idle).
|
||||||
const [rec, setRec] = useState<{
|
const [rec, setRec] = useState<{
|
||||||
stream: MediaStream;
|
stream: MediaStream;
|
||||||
@@ -296,12 +297,13 @@ export function Chat({
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
listTtsVoices().then((v) => { voicesRef.current = v; });
|
listTtsVoices().then((v) => { voicesRef.current = v; });
|
||||||
return () => stopSpeak();
|
api.ttsInfo().then((r) => { ttsServerRef.current = r.available; }).catch(() => {});
|
||||||
|
return () => stopPlayback();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// New message / conversation: stop any playback (never leak speech).
|
// New message / conversation: stop any playback (never leak speech).
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
stopSpeak();
|
stopPlayback();
|
||||||
setSpeakingKey(null);
|
setSpeakingKey(null);
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, [conv?.id]);
|
}, [conv?.id]);
|
||||||
@@ -309,22 +311,29 @@ export function Chat({
|
|||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
return () => {
|
return () => {
|
||||||
cancelRec(false);
|
cancelRec(false);
|
||||||
stopSpeak();
|
stopPlayback();
|
||||||
};
|
};
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
function toggleSpeak(index: number, text: string) {
|
function toggleSpeak(index: number, text: string) {
|
||||||
if (speakingKey === index) {
|
if (speakingKey === index) {
|
||||||
stopSpeak();
|
stopPlayback();
|
||||||
setSpeakingKey(null);
|
setSpeakingKey(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const ok = speak(text, getVoicePrefs(), voicesRef.current, {
|
const prefs = getVoicePrefs();
|
||||||
|
playText({
|
||||||
|
text,
|
||||||
|
engine: prefs.engine,
|
||||||
|
browser: { voices: voicesRef.current, lang: prefs.lang, rate: prefs.rate, pitch: prefs.pitch, voiceURI: prefs.voiceURI },
|
||||||
|
serverVoice: prefs.serverVoice ?? undefined,
|
||||||
|
serverAvailable: ttsServerRef.current,
|
||||||
|
fetchAudio: (t, v) => api.ttsSpeak(t, v),
|
||||||
onend: () => setSpeakingKey(null),
|
onend: () => setSpeakingKey(null),
|
||||||
onerror: () => setSpeakingKey(null),
|
onerror: () => setSpeakingKey(null),
|
||||||
});
|
});
|
||||||
if (ok) setSpeakingKey(index);
|
setSpeakingKey(index);
|
||||||
}
|
}
|
||||||
|
|
||||||
function cleanupRec(r: NonNullable<typeof rec>) {
|
function cleanupRec(r: NonNullable<typeof rec>) {
|
||||||
@@ -446,7 +455,7 @@ export function Chat({
|
|||||||
// A fresh message ends any playback (the glow never sticks to a stale row).
|
// A fresh message ends any playback (the glow never sticks to a stale row).
|
||||||
const msgCount = messages.length;
|
const msgCount = messages.length;
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
stopSpeak();
|
stopPlayback();
|
||||||
setSpeakingKey(null);
|
setSpeakingKey(null);
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, [msgCount]);
|
}, [msgCount]);
|
||||||
@@ -464,7 +473,7 @@ export function Chat({
|
|||||||
if (readOnly) return;
|
if (readOnly) return;
|
||||||
const text = draft.trim();
|
const text = draft.trim();
|
||||||
if (!text || sending) return;
|
if (!text || sending) return;
|
||||||
stopSpeak();
|
stopPlayback();
|
||||||
setSpeakingKey(null);
|
setSpeakingKey(null);
|
||||||
setDraft("");
|
setDraft("");
|
||||||
requestAnimationFrame(autoGrow);
|
requestAnimationFrame(autoGrow);
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { api, type UserProvider, type VoicePreview } from "../lib/api.ts";
|
|||||||
import { pushToast } from "../lib/toasts.ts";
|
import { pushToast } from "../lib/toasts.ts";
|
||||||
import { cn } from "../lib/cn.ts";
|
import { cn } from "../lib/cn.ts";
|
||||||
import { ACCENT_PRESETS, applyAccent, currentAccent, previewAccent, type AccentState } from "../lib/accent.ts";
|
import { ACCENT_PRESETS, applyAccent, currentAccent, previewAccent, type AccentState } from "../lib/accent.ts";
|
||||||
import { detectLang, getVoicePrefs, listTtsVoices, pickVoice, setVoicePrefs, speakPreview, stopSpeak, ttsSupported, type VoicePrefs } from "../lib/voice.ts";
|
import { detectLang, getVoicePrefs, listTtsVoices, pickVoice, playText, setVoicePrefs, stopPlayback, ttsSupported, type VoicePrefs } from "../lib/voice.ts";
|
||||||
import { FEATURES, setFeature, useFeatures } from "../lib/features.ts";
|
import { FEATURES, setFeature, useFeatures } from "../lib/features.ts";
|
||||||
import type { Conversation, FilePreview, User } from "../lib/types.ts";
|
import type { Conversation, FilePreview, User } from "../lib/types.ts";
|
||||||
import { useT, type StringKey } from "../lib/i18n.ts";
|
import { useT, type StringKey } from "../lib/i18n.ts";
|
||||||
@@ -877,12 +877,14 @@ function VoiceSection() {
|
|||||||
const [voices, setVoices] = useState<SpeechSynthesisVoice[]>([]);
|
const [voices, setVoices] = useState<SpeechSynthesisVoice[]>([]);
|
||||||
const [previews, setPreviews] = useState<VoicePreview[]>([]);
|
const [previews, setPreviews] = useState<VoicePreview[]>([]);
|
||||||
const [playing, setPlaying] = useState<number | null>(null);
|
const [playing, setPlaying] = useState<number | null>(null);
|
||||||
|
const [ttsServer, setTtsServer] = useState<{ available: boolean; provider: string | null; voices: string[] } | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
let alive = true;
|
let alive = true;
|
||||||
listTtsVoices().then((v) => { if (alive) setVoices(v); });
|
listTtsVoices().then((v) => { if (alive) setVoices(v); });
|
||||||
api.tools().then((r) => { if (alive) setPreviews(r.voicePreviews); }).catch(() => {});
|
api.tools().then((r) => { if (alive) setPreviews(r.voicePreviews); }).catch(() => {});
|
||||||
return () => { alive = false; stopSpeak(); };
|
api.ttsInfo().then((r) => { if (alive) setTtsServer({ available: r.available, provider: r.provider, voices: r.voices }); }).catch(() => {});
|
||||||
|
return () => { alive = false; stopPlayback(); };
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
function update(p: Partial<VoicePrefs>) {
|
function update(p: Partial<VoicePrefs>) {
|
||||||
@@ -893,17 +895,23 @@ function VoiceSection() {
|
|||||||
|
|
||||||
function preview(i: number, line: VoicePreview) {
|
function preview(i: number, line: VoicePreview) {
|
||||||
if (playing === i) {
|
if (playing === i) {
|
||||||
stopSpeak();
|
stopPlayback();
|
||||||
setPlaying(null);
|
setPlaying(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// Each line plays with ITS OWN character (timbre/rate/pitch/lang) —
|
// Each line plays with ITS OWN character; the neural server engine is
|
||||||
// never the global playback prefs.
|
// used when selected and available (line.server voice or user default).
|
||||||
const pick = speakPreview(line.text, line, voices, {
|
playText({
|
||||||
|
text: line.text,
|
||||||
|
engine: prefs.engine,
|
||||||
|
browser: { voices, lang: line.lang, rate: line.rate, pitch: line.pitch, timbre: line.timbre },
|
||||||
|
serverVoice: line.voice || prefs.serverVoice || undefined,
|
||||||
|
serverAvailable: ttsServer?.available ?? false,
|
||||||
|
fetchAudio: (text, voice) => api.ttsSpeak(text, voice),
|
||||||
onend: () => setPlaying(null),
|
onend: () => setPlaying(null),
|
||||||
onerror: () => setPlaying(null),
|
onerror: () => setPlaying(null),
|
||||||
});
|
});
|
||||||
if (pick || ttsSupported()) setPlaying(i);
|
setPlaying(i);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Adopt a preview's character as the global playback voice. */
|
/** Adopt a preview's character as the global playback voice. */
|
||||||
@@ -911,6 +919,7 @@ function VoiceSection() {
|
|||||||
const lang = line.lang === "auto" ? detectLang(line.text) : line.lang;
|
const lang = line.lang === "auto" ? detectLang(line.text) : line.lang;
|
||||||
const pick = pickVoice(voices, { lang, timbre: line.timbre });
|
const pick = pickVoice(voices, { lang, timbre: line.timbre });
|
||||||
const next: VoicePrefs = {
|
const next: VoicePrefs = {
|
||||||
|
...prefs,
|
||||||
voiceURI: pick?.voiceURI ?? prefs.voiceURI,
|
voiceURI: pick?.voiceURI ?? prefs.voiceURI,
|
||||||
rate: line.rate,
|
rate: line.rate,
|
||||||
pitch: line.pitch,
|
pitch: line.pitch,
|
||||||
@@ -954,6 +963,49 @@ function VoiceSection() {
|
|||||||
options={VOICE_LANGS.map((l) => ({ value: l, label: l === "auto" ? t("voice.auto") : l }))}
|
options={VOICE_LANGS.map((l) => ({ value: l, label: l === "auto" ? t("voice.auto") : l }))}
|
||||||
/>
|
/>
|
||||||
</Field>
|
</Field>
|
||||||
|
<Field label={t("voice.engine")} hint={ttsServer?.available ? t("voice.engineHintOn", { provider: ttsServer.provider ?? "?" }) : t("voice.engineHintOff")}>
|
||||||
|
<Picker
|
||||||
|
ariaLabel={t("voice.engine")}
|
||||||
|
icon={Volume2}
|
||||||
|
value={prefs.engine}
|
||||||
|
onChange={(v) => update({ engine: v as VoicePrefs["engine"] })}
|
||||||
|
align="left"
|
||||||
|
options={[
|
||||||
|
{ value: "auto", label: t("voice.engineAuto") },
|
||||||
|
{ value: "browser", label: t("voice.engineBrowser") },
|
||||||
|
{ value: "server", label: t("voice.engineServer") },
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
{(prefs.engine !== "browser") && (
|
||||||
|
ttsServer?.available ? (
|
||||||
|
ttsServer.provider === "openai" ? (
|
||||||
|
<Field label={t("voice.serverVoice")}>
|
||||||
|
<Picker
|
||||||
|
ariaLabel={t("voice.serverVoice")}
|
||||||
|
icon={Volume2}
|
||||||
|
value={prefs.serverVoice ?? ""}
|
||||||
|
onChange={(v) => update({ serverVoice: v || null })}
|
||||||
|
align="left"
|
||||||
|
options={[{ value: "", label: t("voice.defaultVoice") }, ...ttsServer.voices.map((v) => ({ value: v, label: v }))]}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
) : ttsServer.provider === "elevenlabs" ? (
|
||||||
|
<Field label={t("voice.serverVoice")} hint={t("voice.serverVoiceHint")}>
|
||||||
|
<Input
|
||||||
|
value={prefs.serverVoice ?? ""}
|
||||||
|
onChange={(e) => update({ serverVoice: e.target.value.trim() || null })}
|
||||||
|
placeholder="pNInz6obpgDQGcFmaJgB"
|
||||||
|
maxLength={64}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
) : (
|
||||||
|
<p className="rounded-2xl bg-stone-100 px-4 py-2.5 text-sm opacity-70 dark:bg-zinc-800">{t("voice.piperNote")}</p>
|
||||||
|
)
|
||||||
|
) : (
|
||||||
|
<p className="rounded-2xl bg-amber-500/10 px-4 py-2.5 text-sm text-amber-700 dark:text-amber-400">{t("voice.noServer")}</p>
|
||||||
|
)
|
||||||
|
)}
|
||||||
<Field label={t("voice.voice")}>
|
<Field label={t("voice.voice")}>
|
||||||
<Picker
|
<Picker
|
||||||
ariaLabel={t("voice.voice")}
|
ariaLabel={t("voice.voice")}
|
||||||
|
|||||||
@@ -74,6 +74,7 @@ export interface VoicePreview {
|
|||||||
rate: number;
|
rate: number;
|
||||||
pitch: number;
|
pitch: number;
|
||||||
timbre: "masculine" | "feminine" | "any";
|
timbre: "masculine" | "feminine" | "any";
|
||||||
|
voice: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PlatformInfo {
|
export interface PlatformInfo {
|
||||||
@@ -102,6 +103,7 @@ export interface AdminSettings {
|
|||||||
recoveryLimitMax: number;
|
recoveryLimitMax: number;
|
||||||
recoveryLimitWindowMin: number;
|
recoveryLimitWindowMin: number;
|
||||||
voicePreviews: VoicePreview[];
|
voicePreviews: VoicePreview[];
|
||||||
|
ttsEnabled: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AdminSettingsPatch {
|
export interface AdminSettingsPatch {
|
||||||
@@ -115,6 +117,7 @@ export interface AdminSettingsPatch {
|
|||||||
recoveryLimitMax?: number;
|
recoveryLimitMax?: number;
|
||||||
recoveryLimitWindowMin?: number;
|
recoveryLimitWindowMin?: number;
|
||||||
voicePreviews?: VoicePreview[];
|
voicePreviews?: VoicePreview[];
|
||||||
|
ttsEnabled?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface OllamaNode {
|
export interface OllamaNode {
|
||||||
@@ -442,6 +445,27 @@ export const api = {
|
|||||||
|
|
||||||
tools: () => req<{ tools: { name: string; description: string; available: boolean; reason: string | null }[]; reportsEnabled: boolean; usernameChangeEnabled: boolean; voicePreviews: VoicePreview[] }>("/api/tools"),
|
tools: () => req<{ tools: { name: string; description: string; available: boolean; reason: string | null }[]; reportsEnabled: boolean; usernameChangeEnabled: boolean; voicePreviews: VoicePreview[] }>("/api/tools"),
|
||||||
|
|
||||||
|
/** Neural TTS provider status (null = browser speech fallback). */
|
||||||
|
ttsInfo: () => req<{ provider: "elevenlabs" | "openai" | "piper" | null; voices: string[]; available: boolean; reason: string | null }>("/api/tools/tts/info"),
|
||||||
|
|
||||||
|
/** Synthesize text server-side (neural voice). Returns audio bytes. */
|
||||||
|
ttsSpeak: async (text: string, voice?: string): Promise<Blob> => {
|
||||||
|
const res = await fetch("/api/tools/tts", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json", Authorization: `Bearer ${getToken()}` },
|
||||||
|
body: JSON.stringify({ text, ...(voice ? { voice } : {}) }),
|
||||||
|
});
|
||||||
|
if (res.status === 401) {
|
||||||
|
clearToken();
|
||||||
|
window.location.reload();
|
||||||
|
throw new ApiError(401, "Session expired.");
|
||||||
|
}
|
||||||
|
if (!res.ok) {
|
||||||
|
const data = (await res.json().catch(() => ({}))) as { error?: string };
|
||||||
|
throw new ApiError(res.status, data.error || `TTS failed (${res.status})`);
|
||||||
|
}
|
||||||
|
return await res.blob();
|
||||||
|
},
|
||||||
/** Send an image to the platform OCR tool. Returns editable text (never the raw image). */
|
/** Send an image to the platform OCR tool. Returns editable text (never the raw image). */
|
||||||
ocrImage: async (file: File): Promise<{ text: string; truncated: boolean; chars: number }> => {
|
ocrImage: async (file: File): Promise<{ text: string; truncated: boolean; chars: number }> => {
|
||||||
const res = await fetch("/api/tools/ocr", {
|
const res = await fetch("/api/tools/ocr", {
|
||||||
|
|||||||
@@ -605,6 +605,22 @@ const en = {
|
|||||||
"vfeat.timbre": "Timbre",
|
"vfeat.timbre": "Timbre",
|
||||||
"vfeat.rate": "Speed",
|
"vfeat.rate": "Speed",
|
||||||
"vfeat.pitch": "Pitch",
|
"vfeat.pitch": "Pitch",
|
||||||
|
"vfeat.voice": "Server voice",
|
||||||
|
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||||
|
"vfeat.voicePh": "auto",
|
||||||
|
"center.ttsTitle": "Neural voice playback",
|
||||||
|
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||||
|
"voice.engine": "Playback engine",
|
||||||
|
"voice.engineAuto": "Auto (neural if available)",
|
||||||
|
"voice.engineBrowser": "Browser speech",
|
||||||
|
"voice.engineServer": "Neural server voice",
|
||||||
|
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||||
|
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||||
|
"voice.serverVoice": "Neural voice",
|
||||||
|
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||||
|
"voice.piperNote": "Local piper voice (single configured model).",
|
||||||
|
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"msg.copy": "Copy message",
|
"msg.copy": "Copy message",
|
||||||
@@ -1297,6 +1313,22 @@ const es: Dict = {
|
|||||||
"vfeat.timbre": "Timbre",
|
"vfeat.timbre": "Timbre",
|
||||||
"vfeat.rate": "Speed",
|
"vfeat.rate": "Speed",
|
||||||
"vfeat.pitch": "Pitch",
|
"vfeat.pitch": "Pitch",
|
||||||
|
"vfeat.voice": "Server voice",
|
||||||
|
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||||
|
"vfeat.voicePh": "auto",
|
||||||
|
"center.ttsTitle": "Neural voice playback",
|
||||||
|
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||||
|
"voice.engine": "Playback engine",
|
||||||
|
"voice.engineAuto": "Auto (neural if available)",
|
||||||
|
"voice.engineBrowser": "Browser speech",
|
||||||
|
"voice.engineServer": "Neural server voice",
|
||||||
|
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||||
|
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||||
|
"voice.serverVoice": "Neural voice",
|
||||||
|
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||||
|
"voice.piperNote": "Local piper voice (single configured model).",
|
||||||
|
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"msg.copy": "Copiar mensaje",
|
"msg.copy": "Copiar mensaje",
|
||||||
@@ -1984,6 +2016,22 @@ const fr: Dict = {
|
|||||||
"vfeat.timbre": "Timbre",
|
"vfeat.timbre": "Timbre",
|
||||||
"vfeat.rate": "Speed",
|
"vfeat.rate": "Speed",
|
||||||
"vfeat.pitch": "Pitch",
|
"vfeat.pitch": "Pitch",
|
||||||
|
"vfeat.voice": "Server voice",
|
||||||
|
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||||
|
"vfeat.voicePh": "auto",
|
||||||
|
"center.ttsTitle": "Neural voice playback",
|
||||||
|
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||||
|
"voice.engine": "Playback engine",
|
||||||
|
"voice.engineAuto": "Auto (neural if available)",
|
||||||
|
"voice.engineBrowser": "Browser speech",
|
||||||
|
"voice.engineServer": "Neural server voice",
|
||||||
|
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||||
|
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||||
|
"voice.serverVoice": "Neural voice",
|
||||||
|
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||||
|
"voice.piperNote": "Local piper voice (single configured model).",
|
||||||
|
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"msg.copy": "Copier le message",
|
"msg.copy": "Copier le message",
|
||||||
@@ -2659,6 +2707,22 @@ const ru: Dict = {
|
|||||||
"vfeat.timbre": "Timbre",
|
"vfeat.timbre": "Timbre",
|
||||||
"vfeat.rate": "Speed",
|
"vfeat.rate": "Speed",
|
||||||
"vfeat.pitch": "Pitch",
|
"vfeat.pitch": "Pitch",
|
||||||
|
"vfeat.voice": "Server voice",
|
||||||
|
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||||
|
"vfeat.voicePh": "auto",
|
||||||
|
"center.ttsTitle": "Neural voice playback",
|
||||||
|
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||||
|
"voice.engine": "Playback engine",
|
||||||
|
"voice.engineAuto": "Auto (neural if available)",
|
||||||
|
"voice.engineBrowser": "Browser speech",
|
||||||
|
"voice.engineServer": "Neural server voice",
|
||||||
|
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||||
|
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||||
|
"voice.serverVoice": "Neural voice",
|
||||||
|
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||||
|
"voice.piperNote": "Local piper voice (single configured model).",
|
||||||
|
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"msg.copy": "Скопировать сообщение",
|
"msg.copy": "Скопировать сообщение",
|
||||||
@@ -3339,6 +3403,22 @@ const it: Dict = {
|
|||||||
"vfeat.timbre": "Timbre",
|
"vfeat.timbre": "Timbre",
|
||||||
"vfeat.rate": "Speed",
|
"vfeat.rate": "Speed",
|
||||||
"vfeat.pitch": "Pitch",
|
"vfeat.pitch": "Pitch",
|
||||||
|
"vfeat.voice": "Server voice",
|
||||||
|
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||||
|
"vfeat.voicePh": "auto",
|
||||||
|
"center.ttsTitle": "Neural voice playback",
|
||||||
|
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||||
|
"voice.engine": "Playback engine",
|
||||||
|
"voice.engineAuto": "Auto (neural if available)",
|
||||||
|
"voice.engineBrowser": "Browser speech",
|
||||||
|
"voice.engineServer": "Neural server voice",
|
||||||
|
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||||
|
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||||
|
"voice.serverVoice": "Neural voice",
|
||||||
|
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||||
|
"voice.piperNote": "Local piper voice (single configured model).",
|
||||||
|
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"msg.copy": "Copia messaggio",
|
"msg.copy": "Copia messaggio",
|
||||||
@@ -4010,6 +4090,22 @@ const pt: Dict = {
|
|||||||
"vfeat.timbre": "Timbre",
|
"vfeat.timbre": "Timbre",
|
||||||
"vfeat.rate": "Speed",
|
"vfeat.rate": "Speed",
|
||||||
"vfeat.pitch": "Pitch",
|
"vfeat.pitch": "Pitch",
|
||||||
|
"vfeat.voice": "Server voice",
|
||||||
|
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||||
|
"vfeat.voicePh": "auto",
|
||||||
|
"center.ttsTitle": "Neural voice playback",
|
||||||
|
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||||
|
"voice.engine": "Playback engine",
|
||||||
|
"voice.engineAuto": "Auto (neural if available)",
|
||||||
|
"voice.engineBrowser": "Browser speech",
|
||||||
|
"voice.engineServer": "Neural server voice",
|
||||||
|
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||||
|
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||||
|
"voice.serverVoice": "Neural voice",
|
||||||
|
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||||
|
"voice.piperNote": "Local piper voice (single configured model).",
|
||||||
|
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"msg.copy": "Copiar mensagem",
|
"msg.copy": "Copiar mensagem",
|
||||||
|
|||||||
+156
-36
@@ -15,21 +15,28 @@ export interface VoicePrefs {
|
|||||||
pitch: number;
|
pitch: number;
|
||||||
/** "auto" detects per message, otherwise forces a BCP-47 prefix. */
|
/** "auto" detects per message, otherwise forces a BCP-47 prefix. */
|
||||||
lang: string;
|
lang: string;
|
||||||
|
/** Playback engine: neural server TTS when available, else browser. */
|
||||||
|
engine: "auto" | "browser" | "server";
|
||||||
|
/** Server neural voice id (ElevenLabs voice_id / OpenAI voice name). */
|
||||||
|
serverVoice: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const PREFS_KEY = "ka:voice";
|
const PREFS_KEY = "ka:voice";
|
||||||
|
|
||||||
export const DEFAULT_VOICE_PREFS: VoicePrefs = { voiceURI: null, rate: 1, pitch: 1, lang: "auto" };
|
export const DEFAULT_VOICE_PREFS: VoicePrefs = { voiceURI: null, rate: 1, pitch: 1, lang: "auto", engine: "auto", serverVoice: null };
|
||||||
|
|
||||||
export function getVoicePrefs(): VoicePrefs {
|
export function getVoicePrefs(): VoicePrefs {
|
||||||
try {
|
try {
|
||||||
const raw = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "null") as Partial<VoicePrefs> | null;
|
const raw = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "null") as Partial<VoicePrefs> | null;
|
||||||
if (!raw || typeof raw !== "object") return { ...DEFAULT_VOICE_PREFS };
|
if (!raw || typeof raw !== "object") return { ...DEFAULT_VOICE_PREFS };
|
||||||
|
const engine = raw.engine === "browser" || raw.engine === "server" ? raw.engine : "auto";
|
||||||
return {
|
return {
|
||||||
voiceURI: typeof raw.voiceURI === "string" ? raw.voiceURI : null,
|
voiceURI: typeof raw.voiceURI === "string" ? raw.voiceURI : null,
|
||||||
rate: typeof raw.rate === "number" && raw.rate >= 0.5 && raw.rate <= 2 ? raw.rate : 1,
|
rate: typeof raw.rate === "number" && raw.rate >= 0.5 && raw.rate <= 2 ? raw.rate : 1,
|
||||||
pitch: typeof raw.pitch === "number" && raw.pitch >= 0.5 && raw.pitch <= 2 ? raw.pitch : 1,
|
pitch: typeof raw.pitch === "number" && raw.pitch >= 0.5 && raw.pitch <= 2 ? raw.pitch : 1,
|
||||||
lang: typeof raw.lang === "string" && raw.lang.length <= 12 ? raw.lang : "auto",
|
lang: typeof raw.lang === "string" && raw.lang.length <= 12 ? raw.lang : "auto",
|
||||||
|
engine,
|
||||||
|
serverVoice: typeof raw.serverVoice === "string" && raw.serverVoice.length > 0 ? raw.serverVoice : null,
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
return { ...DEFAULT_VOICE_PREFS };
|
return { ...DEFAULT_VOICE_PREFS };
|
||||||
@@ -83,13 +90,6 @@ export interface SpeakOpts {
|
|||||||
|
|
||||||
export type VoiceTimbre = "masculine" | "feminine" | "any";
|
export type VoiceTimbre = "masculine" | "feminine" | "any";
|
||||||
|
|
||||||
export interface PreviewVoice {
|
|
||||||
lang: string;
|
|
||||||
rate: number;
|
|
||||||
pitch: number;
|
|
||||||
timbre: VoiceTimbre;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Name fragments hinting a masculine / feminine voice (Chrome, Edge,
|
// Name fragments hinting a masculine / feminine voice (Chrome, Edge,
|
||||||
// Safari and Android voices carry such markers; absence falls back to
|
// Safari and Android voices carry such markers; absence falls back to
|
||||||
// any same-language voice — never an error).
|
// any same-language voice — never an error).
|
||||||
@@ -158,43 +158,163 @@ export function stopSpeak(): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
// --- unified playback (neural server audio with browser fallback) ---
|
||||||
* Play a preview line with ITS OWN voice character (timbre + rate + pitch),
|
|
||||||
* never the user's global playback prefs — previews must sound different
|
export interface PlayHandle {
|
||||||
* from each other. Returns the resolved voice (null when unsupported) so
|
/** Engine that ended up playing (may differ from the request in auto). */
|
||||||
* callers can offer "use this voice" adoption.
|
engine: "server" | "browser";
|
||||||
*/
|
stop: () => void;
|
||||||
export function speakPreview(
|
done: Promise<void>;
|
||||||
text: string,
|
}
|
||||||
preview: PreviewVoice,
|
|
||||||
voices: SpeechSynthesisVoice[],
|
let currentAudio: HTMLAudioElement | null = null;
|
||||||
opts: SpeakOpts = {},
|
|
||||||
): SpeechSynthesisVoice | null {
|
function stopCurrentAudio(): void {
|
||||||
if (!ttsSupported()) return null;
|
if (currentAudio) {
|
||||||
const synth = window.speechSynthesis;
|
|
||||||
try {
|
try {
|
||||||
synth.cancel();
|
currentAudio.pause();
|
||||||
|
const url = currentAudio.src;
|
||||||
|
currentAudio.removeAttribute("src");
|
||||||
|
currentAudio.load();
|
||||||
|
if (url.startsWith("blob:")) URL.revokeObjectURL(url);
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
const lang = preview.lang === "auto" ? detectLang(text) : preview.lang;
|
currentAudio = null;
|
||||||
const utter = new SpeechSynthesisUtterance(text.slice(0, 2000));
|
}
|
||||||
utter.lang = lang;
|
|
||||||
utter.rate = preview.rate;
|
|
||||||
utter.pitch = preview.pitch;
|
|
||||||
const pick = pickVoice(voices, { lang, timbre: preview.timbre });
|
|
||||||
if (pick) utter.voice = pick;
|
|
||||||
utter.onend = () => opts.onend?.();
|
|
||||||
utter.onerror = () => opts.onerror?.("error");
|
|
||||||
try {
|
try {
|
||||||
synth.speak(utter);
|
window.speechSynthesis?.cancel();
|
||||||
} catch {
|
} catch {
|
||||||
return null;
|
// ignore
|
||||||
}
|
}
|
||||||
return pick;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- speech recognition (STT) ---
|
/** Stop whatever is playing (server audio or browser speech). */
|
||||||
|
export function stopPlayback(): void {
|
||||||
|
stopCurrentAudio();
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PlayRequest {
|
||||||
|
text: string;
|
||||||
|
/** "server" forces neural, "browser" forces synthesis, "auto" tries neural first. */
|
||||||
|
engine: "auto" | "browser" | "server";
|
||||||
|
browser: {
|
||||||
|
voices: SpeechSynthesisVoice[];
|
||||||
|
/** "auto" detects per text, otherwise a BCP-47 prefix. */
|
||||||
|
lang: string;
|
||||||
|
rate: number;
|
||||||
|
pitch: number;
|
||||||
|
/** Explicit voice URI (user pref); otherwise timbre-matched. */
|
||||||
|
voiceURI?: string | null;
|
||||||
|
/** Preview character timbre (ignored when voiceURI is set). */
|
||||||
|
timbre?: VoiceTimbre;
|
||||||
|
};
|
||||||
|
/** Server neural voice id (preview line voice or user serverVoice). */
|
||||||
|
serverVoice?: string | null;
|
||||||
|
/** True when the platform TTS tool is available. */
|
||||||
|
serverAvailable: boolean;
|
||||||
|
fetchAudio: (text: string, voice?: string) => Promise<Blob>;
|
||||||
|
onstart?: () => void;
|
||||||
|
onend?: () => void;
|
||||||
|
onerror?: (msg: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Play one text with the best engine: neural server audio when requested
|
||||||
|
* and available (shared <audio> element, user speed), otherwise the
|
||||||
|
* browser character voice. Single global playback — a new play stops the
|
||||||
|
* previous one. `done` resolves when playback fully ends (fallback
|
||||||
|
* included); it never rejects (errors go to onend/onerror like speech).
|
||||||
|
*/
|
||||||
|
export function playText(req: PlayRequest): PlayHandle {
|
||||||
|
stopPlayback();
|
||||||
|
const wantServer = req.serverAvailable && req.engine !== "browser";
|
||||||
|
if (wantServer) return playServer(req, req.engine === "auto");
|
||||||
|
return playBrowser(req);
|
||||||
|
}
|
||||||
|
|
||||||
|
function playServer(req: PlayRequest, allowFallback: boolean): PlayHandle {
|
||||||
|
let settled = false;
|
||||||
|
const stop = () => {
|
||||||
|
settled = true;
|
||||||
|
stopCurrentAudio();
|
||||||
|
};
|
||||||
|
const done = (async () => {
|
||||||
|
try {
|
||||||
|
const blob = await req.fetchAudio(req.text.slice(0, 2000), req.serverVoice ?? undefined);
|
||||||
|
if (settled) return;
|
||||||
|
await playAudioBlob(blob, req, () => settled);
|
||||||
|
} catch (e) {
|
||||||
|
if (settled) return;
|
||||||
|
if (allowFallback) {
|
||||||
|
await playBrowserAudio(req, () => settled);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
req.onerror?.(e instanceof Error ? e.message : "playback failed");
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
return { engine: "server", stop, done };
|
||||||
|
}
|
||||||
|
|
||||||
|
function playAudioBlob(blob: Blob, req: PlayRequest, cancelled: () => boolean): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
const audio = new Audio(url);
|
||||||
|
audio.playbackRate = req.browser.rate;
|
||||||
|
currentAudio = audio;
|
||||||
|
const finish = (ok: boolean) => {
|
||||||
|
if (currentAudio === audio) stopCurrentAudio();
|
||||||
|
else {
|
||||||
|
try {
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!cancelled() && ok) req.onend?.();
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
audio.onended = () => finish(true);
|
||||||
|
audio.onerror = () => finish(false);
|
||||||
|
req.onstart?.();
|
||||||
|
audio.play().catch(() => finish(false));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function playBrowser(req: PlayRequest): PlayHandle {
|
||||||
|
let settled = false;
|
||||||
|
const stop = () => {
|
||||||
|
settled = true;
|
||||||
|
stopSpeak();
|
||||||
|
};
|
||||||
|
const done = playBrowserAudio(req, () => settled);
|
||||||
|
return { engine: "browser", stop, done };
|
||||||
|
}
|
||||||
|
|
||||||
|
function playBrowserAudio(req: PlayRequest, cancelled: () => boolean): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const b = req.browser;
|
||||||
|
const lang = b.lang === "auto" ? detectLang(req.text) : b.lang;
|
||||||
|
const explicit = b.voiceURI ? b.voices.find((v) => v.voiceURI === b.voiceURI) : undefined;
|
||||||
|
const pick = explicit ?? pickVoice(b.voices, { lang, timbre: b.timbre ?? "any" });
|
||||||
|
const finish = (ok: boolean, msg?: string) => {
|
||||||
|
if (cancelled()) {
|
||||||
|
resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (ok) req.onend?.();
|
||||||
|
else req.onerror?.(msg ?? "error");
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
const ok = speak(
|
||||||
|
req.text,
|
||||||
|
{ voiceURI: pick?.voiceURI ?? null, rate: b.rate, pitch: b.pitch, lang, engine: "browser", serverVoice: null },
|
||||||
|
b.voices,
|
||||||
|
{ onstart: () => req.onstart?.(), onend: () => finish(true), onerror: (m) => finish(false, m) },
|
||||||
|
);
|
||||||
|
if (!ok) finish(false, "unsupported");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export interface Recognizer {
|
export interface Recognizer {
|
||||||
lang: string;
|
lang: string;
|
||||||
|
|||||||
@@ -40,3 +40,13 @@ Le modèle doit détecter la langue par le texte/etc. l'user peut customiser l'a
|
|||||||
|
|
||||||
Ces exemples sont les default voix preview . dans l'admin center , l'utilisateur peut les modifier /revenir à défaut
|
Ces exemples sont les default voix preview . dans l'admin center , l'utilisateur peut les modifier /revenir à défaut
|
||||||
L'utilisateur doit pouvoir configurée ces ai,vocaux,voix,langue dans une sous-section dédier dans ces paramètres.
|
L'utilisateur doit pouvoir configurée ces ai,vocaux,voix,langue dans une sous-section dédier dans ces paramètres.
|
||||||
|
|
||||||
|
--- actually the favicon have an background svg as the same color of the accenthique theme. should be indepandant .
|
||||||
|
white = day theme
|
||||||
|
black / night theme
|
||||||
|
witht the flower itself following thr accentic color
|
||||||
|
|
||||||
|
|
||||||
|
-- in specify
|
||||||
|
specify the model used in each footer of assistant message
|
||||||
|
like <model> <button>
|
||||||
|
|||||||
@@ -80,6 +80,14 @@ export const config = {
|
|||||||
ocrLang: env("OCR_LANG", "eng"),
|
ocrLang: env("OCR_LANG", "eng"),
|
||||||
ocrMaxChars: envInt("OCR_MAX_CHARS", 100_000),
|
ocrMaxChars: envInt("OCR_MAX_CHARS", 100_000),
|
||||||
|
|
||||||
|
// Neural TTS tool (ElevenLabs / OpenAI / local piper, first configured
|
||||||
|
// wins unless TTS_PROVIDER forces one). Without any of these the tool
|
||||||
|
// reports unavailable and clients fall back to browser speech.
|
||||||
|
elevenlabsApiKey: env("ELEVENLABS_API_KEY", ""),
|
||||||
|
elevenlabsVoice: env("ELEVENLABS_VOICE", ""),
|
||||||
|
piperModel: env("PIPER_MODEL", ""),
|
||||||
|
ttsVoice: env("TTS_VOICE", "alloy"),
|
||||||
|
|
||||||
// Key recovery via email (optional — empty SMTP_HOST disables it).
|
// Key recovery via email (optional — empty SMTP_HOST disables it).
|
||||||
smtpHost: env("SMTP_HOST", ""),
|
smtpHost: env("SMTP_HOST", ""),
|
||||||
smtpPort: envInt("SMTP_PORT", 587),
|
smtpPort: envInt("SMTP_PORT", 587),
|
||||||
|
|||||||
@@ -311,6 +311,11 @@ export async function isOcrToolEnabled(): Promise<boolean> {
|
|||||||
return (await getSetting("tools_ocr_enabled", "1")) === "1";
|
return (await getSetting("tools_ocr_enabled", "1")) === "1";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Admin kill-switch for the neural TTS platform tool (default on). */
|
||||||
|
export async function isTtsToolEnabled(): Promise<boolean> {
|
||||||
|
return (await getSetting("tools_tts_enabled", "1")) === "1";
|
||||||
|
}
|
||||||
|
|
||||||
/** Admin kill-switch for message reports / flag button (default on). */
|
/** Admin kill-switch for message reports / flag button (default on). */
|
||||||
export async function isReportsEnabled(): Promise<boolean> {
|
export async function isReportsEnabled(): Promise<boolean> {
|
||||||
return (await getSetting("reports_enabled", "1")) === "1";
|
return (await getSetting("reports_enabled", "1")) === "1";
|
||||||
|
|||||||
+50
-7
@@ -143,6 +143,7 @@ import {
|
|||||||
writeCdnFile,
|
writeCdnFile,
|
||||||
} from "./cdn.ts";
|
} from "./cdn.ts";
|
||||||
import { OCR_MAX_BYTES, ocrAllowed, recordOcrJob } from "./tools/ocr.ts";
|
import { OCR_MAX_BYTES, ocrAllowed, recordOcrJob } from "./tools/ocr.ts";
|
||||||
|
import { TTS_MAX_CHARS, recordTtsJob, ttsAllowed } from "./tools/tts.ts";
|
||||||
import { tools } from "./tools/registry.ts";
|
import { tools } from "./tools/registry.ts";
|
||||||
import { initOllamaNodes } from "./ollamaNodes.ts";
|
import { initOllamaNodes } from "./ollamaNodes.ts";
|
||||||
|
|
||||||
@@ -236,13 +237,15 @@ export interface VoicePreview {
|
|||||||
rate: number;
|
rate: number;
|
||||||
pitch: number;
|
pitch: number;
|
||||||
timbre: "masculine" | "feminine" | "any";
|
timbre: "masculine" | "feminine" | "any";
|
||||||
|
/** Server neural voice id (ElevenLabs voice_id / OpenAI voice name). Empty = provider default. */
|
||||||
|
voice: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_VOICE_PREVIEWS: VoicePreview[] = [
|
const DEFAULT_VOICE_PREVIEWS: VoicePreview[] = [
|
||||||
{ text: "Salut, euhm, comment je peux t'aider aujourd'hui ?", lang: "fr", rate: 0.85, pitch: 0.75, timbre: "masculine" },
|
{ text: "Salut, euhm, comment je peux t'aider aujourd'hui ?", lang: "fr", rate: 0.85, pitch: 0.75, timbre: "masculine", voice: "" },
|
||||||
{ text: "5 sur 5 je recois ! Que veux-tu ?", lang: "fr", rate: 1, pitch: 0.8, timbre: "masculine" },
|
{ text: "5 sur 5 je recois ! Que veux-tu ?", lang: "fr", rate: 1, pitch: 0.8, timbre: "masculine", voice: "" },
|
||||||
{ text: "Owww, ce chat est trop mignon, oops, pardon. Je me concentre, que puis-je faire pour toi ?", lang: "fr", rate: 1.05, pitch: 1.35, timbre: "feminine" },
|
{ text: "Owww, ce chat est trop mignon, oops, pardon. Je me concentre, que puis-je faire pour toi ?", lang: "fr", rate: 1.05, pitch: 1.35, timbre: "feminine", voice: "" },
|
||||||
{ text: "Hey, que puis-je faire pour toi ?", lang: "fr", rate: 1, pitch: 1.1, timbre: "feminine" },
|
{ text: "Hey, que puis-je faire pour toi ?", lang: "fr", rate: 1, pitch: 1.1, timbre: "feminine", voice: "" },
|
||||||
];
|
];
|
||||||
|
|
||||||
function sanitizePreviewLine(v: unknown): VoicePreview | null {
|
function sanitizePreviewLine(v: unknown): VoicePreview | null {
|
||||||
@@ -250,7 +253,7 @@ function sanitizePreviewLine(v: unknown): VoicePreview | null {
|
|||||||
if (typeof v === "string") {
|
if (typeof v === "string") {
|
||||||
const text = v.trim();
|
const text = v.trim();
|
||||||
if (text.length === 0 || text.length > 500) return null;
|
if (text.length === 0 || text.length > 500) return null;
|
||||||
return { text, lang: "auto", rate: 1, pitch: 1, timbre: "any" };
|
return { text, lang: "auto", rate: 1, pitch: 1, timbre: "any", voice: "" };
|
||||||
}
|
}
|
||||||
if (!v || typeof v !== "object") return null;
|
if (!v || typeof v !== "object") return null;
|
||||||
const o = v as Record<string, unknown>;
|
const o = v as Record<string, unknown>;
|
||||||
@@ -261,7 +264,8 @@ function sanitizePreviewLine(v: unknown): VoicePreview | null {
|
|||||||
const rate = typeof o.rate === "number" && o.rate >= 0.5 && o.rate <= 2 ? o.rate : 1;
|
const rate = typeof o.rate === "number" && o.rate >= 0.5 && o.rate <= 2 ? o.rate : 1;
|
||||||
const pitch = typeof o.pitch === "number" && o.pitch >= 0.5 && o.pitch <= 2 ? o.pitch : 1;
|
const pitch = typeof o.pitch === "number" && o.pitch >= 0.5 && o.pitch <= 2 ? o.pitch : 1;
|
||||||
const timbre = o.timbre === "masculine" || o.timbre === "feminine" ? o.timbre : "any";
|
const timbre = o.timbre === "masculine" || o.timbre === "feminine" ? o.timbre : "any";
|
||||||
return { text, lang, rate, pitch, timbre };
|
const voice = typeof o.voice === "string" && /^[A-Za-z0-9_-]{1,64}$/.test(o.voice.trim()) ? o.voice.trim() : "";
|
||||||
|
return { text, lang, rate, pitch, timbre, voice };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getVoicePreviews(): Promise<VoicePreview[]> {
|
async function getVoicePreviews(): Promise<VoicePreview[]> {
|
||||||
@@ -1031,6 +1035,7 @@ const server = Bun.serve<{ ticketId: string | null; isAdmin: boolean }>({
|
|||||||
recoveryLimitMax: await recoveryLimitMax(),
|
recoveryLimitMax: await recoveryLimitMax(),
|
||||||
recoveryLimitWindowMin: Math.round(await recoveryLimitWindowMs() / 60_000),
|
recoveryLimitWindowMin: Math.round(await recoveryLimitWindowMs() / 60_000),
|
||||||
voicePreviews: await getVoicePreviews(),
|
voicePreviews: await getVoicePreviews(),
|
||||||
|
ttsEnabled: (await getSetting("tools_tts_enabled", "1")) === "1",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1096,10 +1101,14 @@ const server = Bun.serve<{ ticketId: string | null; isAdmin: boolean }>({
|
|||||||
await setSetting("voice_previews", "");
|
await setSetting("voice_previews", "");
|
||||||
} else {
|
} else {
|
||||||
const clean = sanitizeVoicePreviews(body.voicePreviews);
|
const clean = sanitizeVoicePreviews(body.voicePreviews);
|
||||||
if (!clean) return json({ error: "voicePreviews must be 1-6 lines {text, lang?, rate?, pitch?, timbre?}, or [] to reset" }, 400);
|
if (!clean) return json({ error: "voicePreviews must be 1-6 lines {text, lang?, rate?, pitch?, timbre?, voice?}, or [] to reset" }, 400);
|
||||||
await setSetting("voice_previews", JSON.stringify(clean));
|
await setSetting("voice_previews", JSON.stringify(clean));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (body.ttsEnabled !== undefined) {
|
||||||
|
if (typeof body.ttsEnabled !== "boolean") return json({ error: "ttsEnabled must be boolean" }, 400);
|
||||||
|
await setSetting("tools_tts_enabled", body.ttsEnabled ? "1" : "0");
|
||||||
|
}
|
||||||
return json({
|
return json({
|
||||||
settings: {
|
settings: {
|
||||||
registrationEnabled: await isRegistrationEnabled(),
|
registrationEnabled: await isRegistrationEnabled(),
|
||||||
@@ -1112,6 +1121,7 @@ const server = Bun.serve<{ ticketId: string | null; isAdmin: boolean }>({
|
|||||||
recoveryLimitMax: await recoveryLimitMax(),
|
recoveryLimitMax: await recoveryLimitMax(),
|
||||||
recoveryLimitWindowMin: Math.round(await recoveryLimitWindowMs() / 60_000),
|
recoveryLimitWindowMin: Math.round(await recoveryLimitWindowMs() / 60_000),
|
||||||
voicePreviews: await getVoicePreviews(),
|
voicePreviews: await getVoicePreviews(),
|
||||||
|
ttsEnabled: (await getSetting("tools_tts_enabled", "1")) === "1",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1815,6 +1825,39 @@ const server = Bun.serve<{ ticketId: string | null; isAdmin: boolean }>({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- platform tools: neural TTS (natural voices, cached) ---
|
||||||
|
if (path === "/api/tools/tts/info" && req.method === "GET") {
|
||||||
|
const tts = tools.tts;
|
||||||
|
return json({ ...(await tts.providerInfo()), available: await tts.isAvailable(), reason: await tts.unavailableReason() });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (path === "/api/tools/tts" && req.method === "POST") {
|
||||||
|
const tts = tools.tts;
|
||||||
|
if (!await tts.isAvailable()) return json({ error: (await tts.unavailableReason()) ?? "tts unavailable" }, 501);
|
||||||
|
const { ok, body } = await readJson(req);
|
||||||
|
if (!ok || typeof body.text !== "string") return json({ error: "expected { text, voice? }" }, 400);
|
||||||
|
const text = body.text.trim();
|
||||||
|
if (text.length === 0 || text.length > TTS_MAX_CHARS) {
|
||||||
|
return json({ error: `text: 1-${TTS_MAX_CHARS} chars` }, 400);
|
||||||
|
}
|
||||||
|
const rl = ttsAllowed(user.id);
|
||||||
|
if (!rl.ok) {
|
||||||
|
return json({ error: "tts rate limited: 30 jobs per hour", retryAfterSec: rl.retryAfterSec }, 429);
|
||||||
|
}
|
||||||
|
recordTtsJob(user.id);
|
||||||
|
try {
|
||||||
|
const { audio, contentType } = await tts.synthesize(text, typeof body.voice === "string" ? body.voice : undefined);
|
||||||
|
const bytes = audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength) as ArrayBuffer;
|
||||||
|
return new Response(bytes, {
|
||||||
|
headers: { "Content-Type": contentType, "Cache-Control": "private, max-age=86400" },
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
const msg = (e as Error).message;
|
||||||
|
if (/not available|disabled|failed to start/i.test(msg)) return json({ error: msg }, 501);
|
||||||
|
return json({ error: msg }, 502);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (path === "/api/chat" && req.method === "POST") {
|
if (path === "/api/chat" && req.method === "POST") {
|
||||||
const { ok, body } = await readJson(req);
|
const { ok, body } = await readJson(req);
|
||||||
if (!ok) return json({ error: "invalid JSON" }, 400);
|
if (!ok) return json({ error: "invalid JSON" }, 400);
|
||||||
|
|||||||
@@ -1,13 +1,16 @@
|
|||||||
import { OcrTool } from "./ocr.ts";
|
import { OcrTool } from "./ocr.ts";
|
||||||
|
import { TtsTool } from "./tts.ts";
|
||||||
import type { PlatformTool } from "./types.ts";
|
import type { PlatformTool } from "./types.ts";
|
||||||
|
|
||||||
/** Platform tools registry (upload flows always go through tools). */
|
/** Platform tools registry (upload flows always go through tools). */
|
||||||
class ToolRegistry {
|
class ToolRegistry {
|
||||||
readonly ocr = new OcrTool();
|
readonly ocr = new OcrTool();
|
||||||
private tools: PlatformTool[] = [this.ocr];
|
readonly tts = new TtsTool();
|
||||||
|
private tools: PlatformTool[] = [this.ocr, this.tts];
|
||||||
|
|
||||||
async init(): Promise<void> {
|
async init(): Promise<void> {
|
||||||
await this.ocr.init();
|
await this.ocr.init();
|
||||||
|
await this.tts.init();
|
||||||
}
|
}
|
||||||
|
|
||||||
async list(): Promise<{ name: string; description: string; available: boolean; reason: string | null }[]> {
|
async list(): Promise<{ name: string; description: string; available: boolean; reason: string | null }[]> {
|
||||||
|
|||||||
@@ -0,0 +1,279 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { mkdirSync, readdirSync, rmSync, statSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { config } from "../config.ts";
|
||||||
|
import { isTtsToolEnabled } from "../db.ts";
|
||||||
|
import { driverLog, fmtBytes, fmtMs } from "../drivers/log.ts";
|
||||||
|
import type { PlatformTool } from "./types.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Neural TTS tool: turns assistant replies / preview lines into natural
|
||||||
|
* speech (far beyond the browser's robotic speechSynthesis).
|
||||||
|
*
|
||||||
|
* Providers, tried in order (first configured wins; `TTS_PROVIDER` forces
|
||||||
|
* one explicitly):
|
||||||
|
* - elevenlabs (ELEVENLABS_API_KEY [+ ELEVENLABS_VOICE]) — most natural,
|
||||||
|
* any voice_id (20-char ids, clones included).
|
||||||
|
* - openai (OPENAI_API_KEY, reuses the LLM provider key) — tts-1 with the
|
||||||
|
* 6 stock voices (alloy/echo/fable/onyx/nova/shimmer).
|
||||||
|
* - piper (PIPER_BIN + PIPER_MODEL, local binary like tesseract) — offline,
|
||||||
|
* single configured voice, WAV output.
|
||||||
|
*
|
||||||
|
* Output is cached on disk (DATA_DIR/tts-cache, sha of provider|voice|text,
|
||||||
|
* 200 files cap) so repeated lines cost nothing. Without any provider the
|
||||||
|
* tool reports unavailable and the frontend falls back to browser speech.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export type TtsProvider = "elevenlabs" | "openai" | "piper";
|
||||||
|
|
||||||
|
export const OPENAI_VOICES = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] as const;
|
||||||
|
|
||||||
|
const TTS_MAX_CHARS = 2000;
|
||||||
|
const TTS_TIMEOUT_MS = 60_000;
|
||||||
|
const TTS_CACHE_CAP = 200;
|
||||||
|
|
||||||
|
// Abuse protection: neural TTS costs money per byte.
|
||||||
|
const TTS_WINDOW_MS = 3600_000;
|
||||||
|
const TTS_MAX_JOBS = 30;
|
||||||
|
const ttsJobs = new Map<string, number[]>();
|
||||||
|
|
||||||
|
export function ttsAllowed(userId: string): { ok: boolean; retryAfterSec: number } {
|
||||||
|
const now = Date.now();
|
||||||
|
const list = (ttsJobs.get(userId) ?? []).filter((t) => t > now - TTS_WINDOW_MS);
|
||||||
|
ttsJobs.set(userId, list);
|
||||||
|
if (list.length >= TTS_MAX_JOBS) {
|
||||||
|
return { ok: false, retryAfterSec: Math.max(1, Math.ceil((list[0]! + TTS_WINDOW_MS - now) / 1000)) };
|
||||||
|
}
|
||||||
|
return { ok: true, retryAfterSec: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recordTtsJob(userId: string): void {
|
||||||
|
const list = ttsJobs.get(userId) ?? [];
|
||||||
|
list.push(Date.now());
|
||||||
|
ttsJobs.set(userId, list);
|
||||||
|
}
|
||||||
|
|
||||||
|
function elevenlabsBase(): string {
|
||||||
|
return (process.env["ELEVENLABS_BASE_URL"] || "https://api.elevenlabs.io").replace(/\/$/, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function openaiBase(): string {
|
||||||
|
return (process.env["OPENAI_BASE_URL"] || "https://api.openai.com").replace(/\/$/, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function piperBin(): string {
|
||||||
|
return process.env["PIPER_BIN"] || "piper";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function piperAvailable(): Promise<boolean> {
|
||||||
|
if (!config.piperModel) return false;
|
||||||
|
try {
|
||||||
|
const proc = Bun.spawn([piperBin(), "--version"], { stdout: "pipe", stderr: "pipe" });
|
||||||
|
const code = await proc.exited;
|
||||||
|
return code === 0;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Which provider would serve right now (null = none configured). */
|
||||||
|
export async function activeTtsProvider(): Promise<TtsProvider | null> {
|
||||||
|
const forced = (process.env["TTS_PROVIDER"] || "auto").toLowerCase();
|
||||||
|
const eleven = config.elevenlabsApiKey.length > 0;
|
||||||
|
const openai = config.openaiApiKey.length > 0;
|
||||||
|
const piper = await piperAvailable();
|
||||||
|
const pick = (p: TtsProvider): TtsProvider | null =>
|
||||||
|
p === "elevenlabs" ? (eleven ? p : null) : p === "openai" ? (openai ? p : null) : piper ? p : null;
|
||||||
|
if (forced === "elevenlabs" || forced === "openai" || forced === "piper") return pick(forced);
|
||||||
|
return pick("elevenlabs") ?? pick("openai") ?? pick("piper");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sanitizeTtsVoice(provider: TtsProvider, raw: unknown): string | null {
|
||||||
|
if (raw === undefined || raw === null || raw === "") return null;
|
||||||
|
if (typeof raw !== "string") return null;
|
||||||
|
const v = raw.trim();
|
||||||
|
if (provider === "openai") {
|
||||||
|
return (OPENAI_VOICES as readonly string[]).includes(v.toLowerCase()) ? v.toLowerCase() : null;
|
||||||
|
}
|
||||||
|
if (provider === "elevenlabs") {
|
||||||
|
return /^[A-Za-z0-9_-]{1,64}$/.test(v) ? v : null;
|
||||||
|
}
|
||||||
|
return null; // piper: single configured model voice
|
||||||
|
}
|
||||||
|
|
||||||
|
function cacheDir(): string {
|
||||||
|
const dir = join(config.dataDir, "tts-cache");
|
||||||
|
mkdirSync(dir, { recursive: true });
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cacheKey(provider: TtsProvider, voice: string, text: string): string {
|
||||||
|
return createHash("sha256").update(`${provider}|${voice}|${text}`).digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
function cachedPath(key: string, ext: string): string {
|
||||||
|
return join(cacheDir(), `${key}.${ext}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function pruneCache(): void {
|
||||||
|
try {
|
||||||
|
const dir = cacheDir();
|
||||||
|
const files = readdirSync(dir)
|
||||||
|
.map((f) => {
|
||||||
|
try {
|
||||||
|
return { f, mtime: statSync(join(dir, f)).mtimeMs };
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.filter((x): x is { f: string; mtime: number } => x !== null)
|
||||||
|
.sort((a, b) => a.mtime - b.mtime);
|
||||||
|
for (const old of files.slice(0, Math.max(0, files.length - TTS_CACHE_CAP))) {
|
||||||
|
try {
|
||||||
|
rmSync(join(dir, old.f));
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// cache must never break synthesis
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function synthElevenlabs(text: string, voiceId: string): Promise<{ audio: Uint8Array; contentType: string }> {
|
||||||
|
const res = await fetch(`${elevenlabsBase()}/v1/text-to-speech/${encodeURIComponent(voiceId)}`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Accept: "audio/mpeg",
|
||||||
|
"xi-api-key": config.elevenlabsApiKey,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ text, model_id: "eleven_turbo_v2_5" }),
|
||||||
|
signal: AbortSignal.timeout(TTS_TIMEOUT_MS),
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
const err = await res.text().catch(() => "");
|
||||||
|
throw new Error(`elevenlabs tts failed (${res.status}): ${err.slice(0, 200)}`);
|
||||||
|
}
|
||||||
|
return { audio: new Uint8Array(await res.arrayBuffer()), contentType: "audio/mpeg" };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function synthOpenai(text: string, voice: string): Promise<{ audio: Uint8Array; contentType: string }> {
|
||||||
|
const res = await fetch(`${openaiBase()}/v1/audio/speech`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json", Authorization: `Bearer ${config.openaiApiKey}` },
|
||||||
|
body: JSON.stringify({ model: "tts-1", input: text, voice, response_format: "mp3" }),
|
||||||
|
signal: AbortSignal.timeout(TTS_TIMEOUT_MS),
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
const err = await res.text().catch(() => "");
|
||||||
|
throw new Error(`openai tts failed (${res.status}): ${err.slice(0, 200)}`);
|
||||||
|
}
|
||||||
|
return { audio: new Uint8Array(await res.arrayBuffer()), contentType: "audio/mpeg" };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function synthPiper(text: string): Promise<{ audio: Uint8Array; contentType: string }> {
|
||||||
|
const out = join(cacheDir(), `piper-${Date.now().toString(36)}-${Math.floor(Math.random() * 1e6).toString(36)}.wav`);
|
||||||
|
let proc;
|
||||||
|
try {
|
||||||
|
proc = Bun.spawn([piperBin(), "--model", config.piperModel, "--output_file", out], {
|
||||||
|
stdin: new TextEncoder().encode(text),
|
||||||
|
stdout: "pipe",
|
||||||
|
stderr: "pipe",
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
throw new Error(`tts engine failed to start: ${(e as Error).message}`);
|
||||||
|
}
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
try {
|
||||||
|
proc.kill();
|
||||||
|
} catch {
|
||||||
|
// already exited
|
||||||
|
}
|
||||||
|
}, TTS_TIMEOUT_MS);
|
||||||
|
try {
|
||||||
|
const [code, errText] = await Promise.all([proc.exited, new Response(proc.stderr).text()]);
|
||||||
|
if (code !== 0) throw new Error(`piper failed (exit ${code}): ${errText.trim().slice(0, 200)}`);
|
||||||
|
const file = Bun.file(out);
|
||||||
|
if (!(await file.exists())) throw new Error("piper produced no output");
|
||||||
|
return { audio: new Uint8Array(await file.arrayBuffer()), contentType: "audio/wav" };
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
try {
|
||||||
|
rmSync(out);
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class TtsTool implements PlatformTool {
|
||||||
|
readonly name = "tts";
|
||||||
|
readonly description = "Neural text-to-speech (ElevenLabs / OpenAI / local piper) for natural voice playback.";
|
||||||
|
|
||||||
|
async init(): Promise<void> {
|
||||||
|
const p = await activeTtsProvider();
|
||||||
|
driverLog("tools", `tts engine: ${p ?? "none configured — browser speech fallback only"}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async isAvailable(): Promise<boolean> {
|
||||||
|
return (await activeTtsProvider()) !== null && (await isTtsToolEnabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
async unavailableReason(): Promise<string | null> {
|
||||||
|
if ((await activeTtsProvider()) === null) {
|
||||||
|
return "no TTS provider configured (ELEVENLABS_API_KEY, OPENAI_API_KEY or PIPER_BIN+PIPER_MODEL). Browser speech is used instead.";
|
||||||
|
}
|
||||||
|
if (!(await isTtsToolEnabled())) return "TTS is disabled by the administrator (Admin Center → Features).";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async providerInfo(): Promise<{ provider: TtsProvider | null; voices: string[] }> {
|
||||||
|
const p = await activeTtsProvider();
|
||||||
|
if (!p) return { provider: null, voices: [] };
|
||||||
|
if (p === "openai") return { provider: p, voices: [...OPENAI_VOICES] };
|
||||||
|
if (p === "elevenlabs") return { provider: p, voices: config.elevenlabsVoice ? [config.elevenlabsVoice] : [] };
|
||||||
|
return { provider: p, voices: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
defaultVoice(provider: TtsProvider): string {
|
||||||
|
if (provider === "openai") {
|
||||||
|
const v = sanitizeTtsVoice("openai", config.ttsVoice);
|
||||||
|
return v ?? "alloy";
|
||||||
|
}
|
||||||
|
if (provider === "elevenlabs") return config.elevenlabsVoice || "pNInz6obpgDQGcFmaJgB";
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
async synthesize(text: string, voice?: string): Promise<{ audio: Uint8Array; contentType: string; provider: TtsProvider; fromCache: boolean }> {
|
||||||
|
const provider = await activeTtsProvider();
|
||||||
|
if (!provider) throw new Error((await this.unavailableReason()) ?? "tts unavailable");
|
||||||
|
if (!(await isTtsToolEnabled())) throw new Error("tts disabled");
|
||||||
|
const clean = text.trim().slice(0, TTS_MAX_CHARS);
|
||||||
|
if (!clean) throw new Error("empty text");
|
||||||
|
const resolved = sanitizeTtsVoice(provider, voice) ?? this.defaultVoice(provider);
|
||||||
|
const ext = provider === "piper" ? "wav" : "mp3";
|
||||||
|
const key = cacheKey(provider, resolved, clean);
|
||||||
|
const hit = Bun.file(cachedPath(key, ext));
|
||||||
|
if (await hit.exists()) {
|
||||||
|
return {
|
||||||
|
audio: new Uint8Array(await hit.arrayBuffer()),
|
||||||
|
contentType: provider === "piper" ? "audio/wav" : "audio/mpeg",
|
||||||
|
provider,
|
||||||
|
fromCache: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const t0 = Date.now();
|
||||||
|
const out = provider === "elevenlabs"
|
||||||
|
? await synthElevenlabs(clean, resolved)
|
||||||
|
: provider === "openai"
|
||||||
|
? await synthOpenai(clean, resolved)
|
||||||
|
: await synthPiper(clean);
|
||||||
|
await Bun.write(cachedPath(key, ext), out.audio);
|
||||||
|
pruneCache();
|
||||||
|
driverLog("tools", `tts ${provider} voice=${resolved || "default"}: ${fmtBytes(out.audio.length)} in ${fmtMs(Date.now() - t0)}`);
|
||||||
|
return { ...out, provider, fromCache: false };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export { TTS_MAX_CHARS };
|
||||||
Reference in New Issue
Block a user