natural voice
This commit is contained in:
@@ -25,6 +25,21 @@ CDN_DIR=./cdn
|
||||
OCR_LANG=eng
|
||||
OCR_MAX_CHARS=100000
|
||||
|
||||
# --- platform tools: neural TTS (first configured wins, TTS_PROVIDER forces one) ---
|
||||
# ElevenLabs (most natural): https://elevenlabs.io/app/settings/api-keys
|
||||
#ELEVENLABS_API_KEY=
|
||||
#ELEVENLABS_VOICE=
|
||||
# OpenAI (reuses the LLM key when set, tts-1 + alloy/echo/fable/onyx/nova/shimmer)
|
||||
#OPENAI_API_KEY=
|
||||
#TTS_VOICE=alloy
|
||||
# Local piper (offline, like tesseract): PIPER_BIN + onnx model
|
||||
#PIPER_BIN=piper
|
||||
#PIPER_MODEL=/var/lib/kisassistant/piper/en_US-lessac-medium.onnx
|
||||
# Test/development overrides (base URLs, default = api.elevenlabs.io / api.openai.com)
|
||||
#ELEVENLABS_BASE_URL=
|
||||
#OPENAI_BASE_URL=
|
||||
#TTS_PROVIDER=auto
|
||||
|
||||
# Per-driver debug logging to stdout ([driver:name] lines).
|
||||
DRIVER_DEBUG=true
|
||||
|
||||
|
||||
@@ -68,6 +68,8 @@ from the **Accounts** panel (each gets a one-time access key to hand over).
|
||||
| GET | `/api/admin/ollama/nodes/:id/status`, `…/stats?window=` | admin | live probe (version, running, VRAM) / load charts 1h–30d |
|
||||
| GET | `/api/admin/ollama/probe?host=` | admin | test a host before registering it |
|
||||
| GET | `/api/admin/platform` | admin | this host facts: hostname, kernel, uptime, load, memory, IPs |
|
||||
| GET | `/api/tools/tts/info` | Bearer | neural TTS provider + voices (null = browser fallback) |
|
||||
| POST | `/api/tools/tts` | Bearer | synthesize `{text, voice?}` → audio bytes (cached, 30/hour) |
|
||||
| GET | `/api/admin/ollama/status` | admin | Ollama connectivity probe: reachability, version, on-disk models |
|
||||
| POST | `/api/access/request` | public | reserve username+email with a motivation → ticket (`/review/:id`) |
|
||||
| GET/POST | `/api/access/ticket/:id` (+`/message`) | ticket bearer | follow + reply on an access request |
|
||||
|
||||
+3
-1
@@ -92,7 +92,9 @@ No router library — a tiny history wrapper (`lib/route.ts`: `navigate()`,
|
||||
also repaints the SVG favicon (data URL) so the tab icon matches the dress.
|
||||
- Voice (`lib/voice.ts`, no dependency): `Chat.tsx` composer mic with live
|
||||
waveform + speech-recognition transcript (editable before send), per-bubble
|
||||
TTS playback with glow, `Settings → Voice` prefs (voice/lang/speed/pitch,
|
||||
playback with glow through a unified player (`playText`: neural server
|
||||
audio via `POST /api/tools/tts` with browser-synthesis fallback),
|
||||
`Settings → Voice` prefs (engine/lang/voice/speed/pitch/neural voice,
|
||||
localStorage), admin-editable preview lines.
|
||||
|
||||
## Theming
|
||||
|
||||
+24
-8
@@ -193,15 +193,31 @@ Disk meters are intentionally absent (models live on the Ollama hosts).
|
||||
it away. Unsupported browsers get a disabled mic with an explanatory
|
||||
tooltip. No audio leaves the browser (no server STT endpoint yet —
|
||||
whisper.cpp via `WHISPER_BIN` is the documented next step).
|
||||
- **Message playback**: every assistant bubble has a Listen button
|
||||
(speechSynthesis, Siri-style breathing glow while speaking, click again to
|
||||
stop; playback stops on send / new message / conversation switch).
|
||||
- **Settings → Voice** (per browser, localStorage): language (auto-detect
|
||||
per message via `detectLang`, or forced), voice picker grouped by locale,
|
||||
speed, pitch, and one-tap previews of the admin-set lines.
|
||||
- **Message playback**: every assistant bubble has a Listen button with a
|
||||
Siri-style breathing glow while speaking (click again to stop; playback
|
||||
stops on send / new message / conversation switch). Engine per user
|
||||
(Settings → Voice): **auto** (neural server voice when available, browser
|
||||
fallback), **browser**, or **server**.
|
||||
- **Neural server TTS** (`POST /api/tools/tts` `{text, voice?}` → audio
|
||||
bytes, `GET /api/tools/tts/info` → provider + voices): ElevenLabs
|
||||
(any voice_id, most natural) → OpenAI `tts-1` (6 stock voices, reuses the
|
||||
LLM key) → local piper (`PIPER_BIN` + `PIPER_MODEL`, offline WAV).
|
||||
`TTS_PROVIDER` forces one; `TTS_VOICE` / `ELEVENLABS_VOICE` set defaults.
|
||||
Output is cached on disk (sha of provider|voice|text, 200 files cap),
|
||||
rate-limited to 30 jobs/hour/account (429), kill-switched via
|
||||
`tools_tts_enabled` (Admin Center → Features). Test overrides:
|
||||
`ELEVENLABS_BASE_URL` / `OPENAI_BASE_URL`.
|
||||
- **Settings → Voice** (per browser, localStorage): engine, language
|
||||
(auto-detect per message or forced), browser voice/rate/pitch, neural
|
||||
voice id, and one-tap previews of the admin-set lines.
|
||||
- **Admin Center → Features → Voice previews**: the default playback lines
|
||||
(`voice_previews` setting, 1–6 lines; empty list resets to the built-ins),
|
||||
also served to users via `GET /api/tools`.
|
||||
(`voice_previews` setting, 1–6 lines `{text, lang, rate, pitch, timbre,
|
||||
voice}`, each with its OWN voice character — masculine/feminine/any
|
||||
timbre hint, speed, pitch, optional server voice id — so previews sound
|
||||
different from each other; timbre matching is best-effort over the
|
||||
browser's voice names, rate/pitch always apply; empty list resets to the
|
||||
built-ins), also served to users via `GET /api/tools`. Users can adopt
|
||||
any preview's character as their own playback voice from Settings → Voice.
|
||||
|
||||
## Personal providers (BYOK)
|
||||
|
||||
|
||||
@@ -321,6 +321,13 @@ export function AdminCenter({ initialTab }: { initialTab?: string | null }) {
|
||||
</div>
|
||||
<Switch label={t("center.usernameChangeTitle")} checked={settings?.usernameChangeEnabled ?? false} onChange={(v) => toggle({ usernameChangeEnabled: v })} />
|
||||
</div>
|
||||
<div className="flex items-center gap-4 rounded-3xl border border-stone-200/70 bg-white p-5 shadow-sm dark:border-zinc-800 dark:bg-zinc-900">
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="text-sm font-medium">{t("center.ttsTitle")}</p>
|
||||
<p className="text-sm opacity-60">{t("center.ttsDesc")}</p>
|
||||
</div>
|
||||
<Switch label={t("center.ttsTitle")} checked={settings?.ttsEnabled ?? false} onChange={(v) => toggle({ ttsEnabled: v })} />
|
||||
</div>
|
||||
{saving && <p className="flex items-center gap-2 text-sm opacity-60"><Spinner size={14} /> {t("common.saving")}</p>}
|
||||
{!settings && !error && <p className="flex items-center gap-2 text-sm opacity-60"><Spinner size={14} /> {t("common.loading")}</p>}
|
||||
<VoicePreviewsCard />
|
||||
@@ -444,6 +451,20 @@ function VoicePreviewsCard() {
|
||||
/>
|
||||
</Field>
|
||||
</span>
|
||||
<span className="w-40">
|
||||
<Field label={t("vfeat.voice")} hint={t("vfeat.voiceHint")}>
|
||||
<Input
|
||||
value={line.voice}
|
||||
disabled={busy}
|
||||
maxLength={64}
|
||||
placeholder={t("vfeat.voicePh")}
|
||||
aria-label={t("vfeat.voice")}
|
||||
onChange={(e) => patch(i, { voice: e.target.value })}
|
||||
onBlur={() => commit(i)}
|
||||
onKeyDown={(e) => { if (e.key === "Enter") (e.target as HTMLInputElement).blur(); }}
|
||||
/>
|
||||
</Field>
|
||||
</span>
|
||||
<label className="min-w-32 flex-1 text-xs opacity-80">
|
||||
<span className="mb-1 block font-medium">{t("voice.rate")} · {line.rate.toFixed(2)}×</span>
|
||||
<input
|
||||
@@ -478,7 +499,7 @@ function VoicePreviewsCard() {
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
disabled={busy}
|
||||
onClick={() => setLines((prev) => [...prev, { text: "", lang: "auto", rate: 1, pitch: 1, timbre: "any" as const }])}
|
||||
onClick={() => setLines((prev) => [...prev, { text: "", lang: "auto", rate: 1, pitch: 1, timbre: "any" as const, voice: "" }])}
|
||||
>
|
||||
<Plus size={14} /> {t("vfeat.add")}
|
||||
</Button>
|
||||
|
||||
@@ -7,7 +7,7 @@ import { Markdown } from "./Markdown.tsx";
|
||||
import { UserMessageContent } from "./MessageContent.tsx";
|
||||
import { cn } from "../lib/cn.ts";
|
||||
import { useFeatures } from "../lib/features.ts";
|
||||
import { fmtDuration, getVoicePrefs, listTtsVoices, recognizerCtor, speak, stopSpeak, sttSupported, type Recognizer } from "../lib/voice.ts";
|
||||
import { fmtDuration, getVoicePrefs, listTtsVoices, playText, recognizerCtor, stopPlayback, sttSupported, type Recognizer } from "../lib/voice.ts";
|
||||
import { useT, type StringKey } from "../lib/i18n.ts";
|
||||
|
||||
const REPORT_REASONS: ReportReason[] = ["copyright", "gore", "falseinfo", "bug"];
|
||||
@@ -279,6 +279,7 @@ export function Chat({
|
||||
// TTS playback: which assistant bubble is speaking (null = silent).
|
||||
const [speakingKey, setSpeakingKey] = useState<number | null>(null);
|
||||
const voicesRef = useRef<SpeechSynthesisVoice[]>([]);
|
||||
const ttsServerRef = useRef(false);
|
||||
// Voice input: live recording session (null = composer idle).
|
||||
const [rec, setRec] = useState<{
|
||||
stream: MediaStream;
|
||||
@@ -296,12 +297,13 @@ export function Chat({
|
||||
|
||||
useEffect(() => {
|
||||
listTtsVoices().then((v) => { voicesRef.current = v; });
|
||||
return () => stopSpeak();
|
||||
api.ttsInfo().then((r) => { ttsServerRef.current = r.available; }).catch(() => {});
|
||||
return () => stopPlayback();
|
||||
}, []);
|
||||
|
||||
// New message / conversation: stop any playback (never leak speech).
|
||||
useEffect(() => {
|
||||
stopSpeak();
|
||||
stopPlayback();
|
||||
setSpeakingKey(null);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [conv?.id]);
|
||||
@@ -309,22 +311,29 @@ export function Chat({
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
cancelRec(false);
|
||||
stopSpeak();
|
||||
stopPlayback();
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
function toggleSpeak(index: number, text: string) {
|
||||
if (speakingKey === index) {
|
||||
stopSpeak();
|
||||
stopPlayback();
|
||||
setSpeakingKey(null);
|
||||
return;
|
||||
}
|
||||
const ok = speak(text, getVoicePrefs(), voicesRef.current, {
|
||||
const prefs = getVoicePrefs();
|
||||
playText({
|
||||
text,
|
||||
engine: prefs.engine,
|
||||
browser: { voices: voicesRef.current, lang: prefs.lang, rate: prefs.rate, pitch: prefs.pitch, voiceURI: prefs.voiceURI },
|
||||
serverVoice: prefs.serverVoice ?? undefined,
|
||||
serverAvailable: ttsServerRef.current,
|
||||
fetchAudio: (t, v) => api.ttsSpeak(t, v),
|
||||
onend: () => setSpeakingKey(null),
|
||||
onerror: () => setSpeakingKey(null),
|
||||
});
|
||||
if (ok) setSpeakingKey(index);
|
||||
setSpeakingKey(index);
|
||||
}
|
||||
|
||||
function cleanupRec(r: NonNullable<typeof rec>) {
|
||||
@@ -446,7 +455,7 @@ export function Chat({
|
||||
// A fresh message ends any playback (the glow never sticks to a stale row).
|
||||
const msgCount = messages.length;
|
||||
useEffect(() => {
|
||||
stopSpeak();
|
||||
stopPlayback();
|
||||
setSpeakingKey(null);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [msgCount]);
|
||||
@@ -464,7 +473,7 @@ export function Chat({
|
||||
if (readOnly) return;
|
||||
const text = draft.trim();
|
||||
if (!text || sending) return;
|
||||
stopSpeak();
|
||||
stopPlayback();
|
||||
setSpeakingKey(null);
|
||||
setDraft("");
|
||||
requestAnimationFrame(autoGrow);
|
||||
|
||||
@@ -4,7 +4,7 @@ import { api, type UserProvider, type VoicePreview } from "../lib/api.ts";
|
||||
import { pushToast } from "../lib/toasts.ts";
|
||||
import { cn } from "../lib/cn.ts";
|
||||
import { ACCENT_PRESETS, applyAccent, currentAccent, previewAccent, type AccentState } from "../lib/accent.ts";
|
||||
import { detectLang, getVoicePrefs, listTtsVoices, pickVoice, setVoicePrefs, speakPreview, stopSpeak, ttsSupported, type VoicePrefs } from "../lib/voice.ts";
|
||||
import { detectLang, getVoicePrefs, listTtsVoices, pickVoice, playText, setVoicePrefs, stopPlayback, ttsSupported, type VoicePrefs } from "../lib/voice.ts";
|
||||
import { FEATURES, setFeature, useFeatures } from "../lib/features.ts";
|
||||
import type { Conversation, FilePreview, User } from "../lib/types.ts";
|
||||
import { useT, type StringKey } from "../lib/i18n.ts";
|
||||
@@ -877,12 +877,14 @@ function VoiceSection() {
|
||||
const [voices, setVoices] = useState<SpeechSynthesisVoice[]>([]);
|
||||
const [previews, setPreviews] = useState<VoicePreview[]>([]);
|
||||
const [playing, setPlaying] = useState<number | null>(null);
|
||||
const [ttsServer, setTtsServer] = useState<{ available: boolean; provider: string | null; voices: string[] } | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let alive = true;
|
||||
listTtsVoices().then((v) => { if (alive) setVoices(v); });
|
||||
api.tools().then((r) => { if (alive) setPreviews(r.voicePreviews); }).catch(() => {});
|
||||
return () => { alive = false; stopSpeak(); };
|
||||
api.ttsInfo().then((r) => { if (alive) setTtsServer({ available: r.available, provider: r.provider, voices: r.voices }); }).catch(() => {});
|
||||
return () => { alive = false; stopPlayback(); };
|
||||
}, []);
|
||||
|
||||
function update(p: Partial<VoicePrefs>) {
|
||||
@@ -893,17 +895,23 @@ function VoiceSection() {
|
||||
|
||||
function preview(i: number, line: VoicePreview) {
|
||||
if (playing === i) {
|
||||
stopSpeak();
|
||||
stopPlayback();
|
||||
setPlaying(null);
|
||||
return;
|
||||
}
|
||||
// Each line plays with ITS OWN character (timbre/rate/pitch/lang) —
|
||||
// never the global playback prefs.
|
||||
const pick = speakPreview(line.text, line, voices, {
|
||||
// Each line plays with ITS OWN character; the neural server engine is
|
||||
// used when selected and available (line.server voice or user default).
|
||||
playText({
|
||||
text: line.text,
|
||||
engine: prefs.engine,
|
||||
browser: { voices, lang: line.lang, rate: line.rate, pitch: line.pitch, timbre: line.timbre },
|
||||
serverVoice: line.voice || prefs.serverVoice || undefined,
|
||||
serverAvailable: ttsServer?.available ?? false,
|
||||
fetchAudio: (text, voice) => api.ttsSpeak(text, voice),
|
||||
onend: () => setPlaying(null),
|
||||
onerror: () => setPlaying(null),
|
||||
});
|
||||
if (pick || ttsSupported()) setPlaying(i);
|
||||
setPlaying(i);
|
||||
}
|
||||
|
||||
/** Adopt a preview's character as the global playback voice. */
|
||||
@@ -911,6 +919,7 @@ function VoiceSection() {
|
||||
const lang = line.lang === "auto" ? detectLang(line.text) : line.lang;
|
||||
const pick = pickVoice(voices, { lang, timbre: line.timbre });
|
||||
const next: VoicePrefs = {
|
||||
...prefs,
|
||||
voiceURI: pick?.voiceURI ?? prefs.voiceURI,
|
||||
rate: line.rate,
|
||||
pitch: line.pitch,
|
||||
@@ -954,6 +963,49 @@ function VoiceSection() {
|
||||
options={VOICE_LANGS.map((l) => ({ value: l, label: l === "auto" ? t("voice.auto") : l }))}
|
||||
/>
|
||||
</Field>
|
||||
<Field label={t("voice.engine")} hint={ttsServer?.available ? t("voice.engineHintOn", { provider: ttsServer.provider ?? "?" }) : t("voice.engineHintOff")}>
|
||||
<Picker
|
||||
ariaLabel={t("voice.engine")}
|
||||
icon={Volume2}
|
||||
value={prefs.engine}
|
||||
onChange={(v) => update({ engine: v as VoicePrefs["engine"] })}
|
||||
align="left"
|
||||
options={[
|
||||
{ value: "auto", label: t("voice.engineAuto") },
|
||||
{ value: "browser", label: t("voice.engineBrowser") },
|
||||
{ value: "server", label: t("voice.engineServer") },
|
||||
]}
|
||||
/>
|
||||
</Field>
|
||||
{(prefs.engine !== "browser") && (
|
||||
ttsServer?.available ? (
|
||||
ttsServer.provider === "openai" ? (
|
||||
<Field label={t("voice.serverVoice")}>
|
||||
<Picker
|
||||
ariaLabel={t("voice.serverVoice")}
|
||||
icon={Volume2}
|
||||
value={prefs.serverVoice ?? ""}
|
||||
onChange={(v) => update({ serverVoice: v || null })}
|
||||
align="left"
|
||||
options={[{ value: "", label: t("voice.defaultVoice") }, ...ttsServer.voices.map((v) => ({ value: v, label: v }))]}
|
||||
/>
|
||||
</Field>
|
||||
) : ttsServer.provider === "elevenlabs" ? (
|
||||
<Field label={t("voice.serverVoice")} hint={t("voice.serverVoiceHint")}>
|
||||
<Input
|
||||
value={prefs.serverVoice ?? ""}
|
||||
onChange={(e) => update({ serverVoice: e.target.value.trim() || null })}
|
||||
placeholder="pNInz6obpgDQGcFmaJgB"
|
||||
maxLength={64}
|
||||
/>
|
||||
</Field>
|
||||
) : (
|
||||
<p className="rounded-2xl bg-stone-100 px-4 py-2.5 text-sm opacity-70 dark:bg-zinc-800">{t("voice.piperNote")}</p>
|
||||
)
|
||||
) : (
|
||||
<p className="rounded-2xl bg-amber-500/10 px-4 py-2.5 text-sm text-amber-700 dark:text-amber-400">{t("voice.noServer")}</p>
|
||||
)
|
||||
)}
|
||||
<Field label={t("voice.voice")}>
|
||||
<Picker
|
||||
ariaLabel={t("voice.voice")}
|
||||
|
||||
@@ -74,6 +74,7 @@ export interface VoicePreview {
|
||||
rate: number;
|
||||
pitch: number;
|
||||
timbre: "masculine" | "feminine" | "any";
|
||||
voice: string;
|
||||
}
|
||||
|
||||
export interface PlatformInfo {
|
||||
@@ -102,6 +103,7 @@ export interface AdminSettings {
|
||||
recoveryLimitMax: number;
|
||||
recoveryLimitWindowMin: number;
|
||||
voicePreviews: VoicePreview[];
|
||||
ttsEnabled: boolean;
|
||||
}
|
||||
|
||||
export interface AdminSettingsPatch {
|
||||
@@ -115,6 +117,7 @@ export interface AdminSettingsPatch {
|
||||
recoveryLimitMax?: number;
|
||||
recoveryLimitWindowMin?: number;
|
||||
voicePreviews?: VoicePreview[];
|
||||
ttsEnabled?: boolean;
|
||||
}
|
||||
|
||||
export interface OllamaNode {
|
||||
@@ -442,6 +445,27 @@ export const api = {
|
||||
|
||||
tools: () => req<{ tools: { name: string; description: string; available: boolean; reason: string | null }[]; reportsEnabled: boolean; usernameChangeEnabled: boolean; voicePreviews: VoicePreview[] }>("/api/tools"),
|
||||
|
||||
/** Neural TTS provider status (null = browser speech fallback). */
|
||||
ttsInfo: () => req<{ provider: "elevenlabs" | "openai" | "piper" | null; voices: string[]; available: boolean; reason: string | null }>("/api/tools/tts/info"),
|
||||
|
||||
/** Synthesize text server-side (neural voice). Returns audio bytes. */
|
||||
ttsSpeak: async (text: string, voice?: string): Promise<Blob> => {
|
||||
const res = await fetch("/api/tools/tts", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${getToken()}` },
|
||||
body: JSON.stringify({ text, ...(voice ? { voice } : {}) }),
|
||||
});
|
||||
if (res.status === 401) {
|
||||
clearToken();
|
||||
window.location.reload();
|
||||
throw new ApiError(401, "Session expired.");
|
||||
}
|
||||
if (!res.ok) {
|
||||
const data = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new ApiError(res.status, data.error || `TTS failed (${res.status})`);
|
||||
}
|
||||
return await res.blob();
|
||||
},
|
||||
/** Send an image to the platform OCR tool. Returns editable text (never the raw image). */
|
||||
ocrImage: async (file: File): Promise<{ text: string; truncated: boolean; chars: number }> => {
|
||||
const res = await fetch("/api/tools/ocr", {
|
||||
|
||||
@@ -605,6 +605,22 @@ const en = {
|
||||
"vfeat.timbre": "Timbre",
|
||||
"vfeat.rate": "Speed",
|
||||
"vfeat.pitch": "Pitch",
|
||||
"vfeat.voice": "Server voice",
|
||||
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||
"vfeat.voicePh": "auto",
|
||||
"center.ttsTitle": "Neural voice playback",
|
||||
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||
"voice.engine": "Playback engine",
|
||||
"voice.engineAuto": "Auto (neural if available)",
|
||||
"voice.engineBrowser": "Browser speech",
|
||||
"voice.engineServer": "Neural server voice",
|
||||
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||
"voice.serverVoice": "Neural voice",
|
||||
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||
"voice.piperNote": "Local piper voice (single configured model).",
|
||||
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||
|
||||
|
||||
|
||||
"msg.copy": "Copy message",
|
||||
@@ -1297,6 +1313,22 @@ const es: Dict = {
|
||||
"vfeat.timbre": "Timbre",
|
||||
"vfeat.rate": "Speed",
|
||||
"vfeat.pitch": "Pitch",
|
||||
"vfeat.voice": "Server voice",
|
||||
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||
"vfeat.voicePh": "auto",
|
||||
"center.ttsTitle": "Neural voice playback",
|
||||
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||
"voice.engine": "Playback engine",
|
||||
"voice.engineAuto": "Auto (neural if available)",
|
||||
"voice.engineBrowser": "Browser speech",
|
||||
"voice.engineServer": "Neural server voice",
|
||||
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||
"voice.serverVoice": "Neural voice",
|
||||
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||
"voice.piperNote": "Local piper voice (single configured model).",
|
||||
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||
|
||||
|
||||
|
||||
"msg.copy": "Copiar mensaje",
|
||||
@@ -1984,6 +2016,22 @@ const fr: Dict = {
|
||||
"vfeat.timbre": "Timbre",
|
||||
"vfeat.rate": "Speed",
|
||||
"vfeat.pitch": "Pitch",
|
||||
"vfeat.voice": "Server voice",
|
||||
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||
"vfeat.voicePh": "auto",
|
||||
"center.ttsTitle": "Neural voice playback",
|
||||
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||
"voice.engine": "Playback engine",
|
||||
"voice.engineAuto": "Auto (neural if available)",
|
||||
"voice.engineBrowser": "Browser speech",
|
||||
"voice.engineServer": "Neural server voice",
|
||||
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||
"voice.serverVoice": "Neural voice",
|
||||
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||
"voice.piperNote": "Local piper voice (single configured model).",
|
||||
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||
|
||||
|
||||
|
||||
"msg.copy": "Copier le message",
|
||||
@@ -2659,6 +2707,22 @@ const ru: Dict = {
|
||||
"vfeat.timbre": "Timbre",
|
||||
"vfeat.rate": "Speed",
|
||||
"vfeat.pitch": "Pitch",
|
||||
"vfeat.voice": "Server voice",
|
||||
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||
"vfeat.voicePh": "auto",
|
||||
"center.ttsTitle": "Neural voice playback",
|
||||
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||
"voice.engine": "Playback engine",
|
||||
"voice.engineAuto": "Auto (neural if available)",
|
||||
"voice.engineBrowser": "Browser speech",
|
||||
"voice.engineServer": "Neural server voice",
|
||||
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||
"voice.serverVoice": "Neural voice",
|
||||
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||
"voice.piperNote": "Local piper voice (single configured model).",
|
||||
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||
|
||||
|
||||
|
||||
"msg.copy": "Скопировать сообщение",
|
||||
@@ -3339,6 +3403,22 @@ const it: Dict = {
|
||||
"vfeat.timbre": "Timbre",
|
||||
"vfeat.rate": "Speed",
|
||||
"vfeat.pitch": "Pitch",
|
||||
"vfeat.voice": "Server voice",
|
||||
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||
"vfeat.voicePh": "auto",
|
||||
"center.ttsTitle": "Neural voice playback",
|
||||
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||
"voice.engine": "Playback engine",
|
||||
"voice.engineAuto": "Auto (neural if available)",
|
||||
"voice.engineBrowser": "Browser speech",
|
||||
"voice.engineServer": "Neural server voice",
|
||||
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||
"voice.serverVoice": "Neural voice",
|
||||
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||
"voice.piperNote": "Local piper voice (single configured model).",
|
||||
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||
|
||||
|
||||
|
||||
"msg.copy": "Copia messaggio",
|
||||
@@ -4010,6 +4090,22 @@ const pt: Dict = {
|
||||
"vfeat.timbre": "Timbre",
|
||||
"vfeat.rate": "Speed",
|
||||
"vfeat.pitch": "Pitch",
|
||||
"vfeat.voice": "Server voice",
|
||||
"vfeat.voiceHint": "ElevenLabs id / OpenAI name, empty = default",
|
||||
"vfeat.voicePh": "auto",
|
||||
"center.ttsTitle": "Neural voice playback",
|
||||
"center.ttsDesc": "Server text-to-speech for message playback (ElevenLabs / OpenAI / piper). Off = browser speech only.",
|
||||
"voice.engine": "Playback engine",
|
||||
"voice.engineAuto": "Auto (neural if available)",
|
||||
"voice.engineBrowser": "Browser speech",
|
||||
"voice.engineServer": "Neural server voice",
|
||||
"voice.engineHintOn": "Neural TTS via {provider}",
|
||||
"voice.engineHintOff": "No neural provider configured — browser speech only",
|
||||
"voice.serverVoice": "Neural voice",
|
||||
"voice.serverVoiceHint": "ElevenLabs voice id",
|
||||
"voice.piperNote": "Local piper voice (single configured model).",
|
||||
"voice.noServer": "Configure ELEVENLABS_API_KEY, OPENAI_API_KEY or piper to unlock neural voices.",
|
||||
|
||||
|
||||
|
||||
"msg.copy": "Copiar mensagem",
|
||||
|
||||
+156
-36
@@ -15,21 +15,28 @@ export interface VoicePrefs {
|
||||
pitch: number;
|
||||
/** "auto" detects per message, otherwise forces a BCP-47 prefix. */
|
||||
lang: string;
|
||||
/** Playback engine: neural server TTS when available, else browser. */
|
||||
engine: "auto" | "browser" | "server";
|
||||
/** Server neural voice id (ElevenLabs voice_id / OpenAI voice name). */
|
||||
serverVoice: string | null;
|
||||
}
|
||||
|
||||
const PREFS_KEY = "ka:voice";
|
||||
|
||||
export const DEFAULT_VOICE_PREFS: VoicePrefs = { voiceURI: null, rate: 1, pitch: 1, lang: "auto" };
|
||||
export const DEFAULT_VOICE_PREFS: VoicePrefs = { voiceURI: null, rate: 1, pitch: 1, lang: "auto", engine: "auto", serverVoice: null };
|
||||
|
||||
export function getVoicePrefs(): VoicePrefs {
|
||||
try {
|
||||
const raw = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "null") as Partial<VoicePrefs> | null;
|
||||
if (!raw || typeof raw !== "object") return { ...DEFAULT_VOICE_PREFS };
|
||||
const engine = raw.engine === "browser" || raw.engine === "server" ? raw.engine : "auto";
|
||||
return {
|
||||
voiceURI: typeof raw.voiceURI === "string" ? raw.voiceURI : null,
|
||||
rate: typeof raw.rate === "number" && raw.rate >= 0.5 && raw.rate <= 2 ? raw.rate : 1,
|
||||
pitch: typeof raw.pitch === "number" && raw.pitch >= 0.5 && raw.pitch <= 2 ? raw.pitch : 1,
|
||||
lang: typeof raw.lang === "string" && raw.lang.length <= 12 ? raw.lang : "auto",
|
||||
engine,
|
||||
serverVoice: typeof raw.serverVoice === "string" && raw.serverVoice.length > 0 ? raw.serverVoice : null,
|
||||
};
|
||||
} catch {
|
||||
return { ...DEFAULT_VOICE_PREFS };
|
||||
@@ -83,13 +90,6 @@ export interface SpeakOpts {
|
||||
|
||||
export type VoiceTimbre = "masculine" | "feminine" | "any";
|
||||
|
||||
export interface PreviewVoice {
|
||||
lang: string;
|
||||
rate: number;
|
||||
pitch: number;
|
||||
timbre: VoiceTimbre;
|
||||
}
|
||||
|
||||
// Name fragments hinting a masculine / feminine voice (Chrome, Edge,
|
||||
// Safari and Android voices carry such markers; absence falls back to
|
||||
// any same-language voice — never an error).
|
||||
@@ -158,43 +158,163 @@ export function stopSpeak(): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Play a preview line with ITS OWN voice character (timbre + rate + pitch),
|
||||
* never the user's global playback prefs — previews must sound different
|
||||
* from each other. Returns the resolved voice (null when unsupported) so
|
||||
* callers can offer "use this voice" adoption.
|
||||
*/
|
||||
export function speakPreview(
|
||||
text: string,
|
||||
preview: PreviewVoice,
|
||||
voices: SpeechSynthesisVoice[],
|
||||
opts: SpeakOpts = {},
|
||||
): SpeechSynthesisVoice | null {
|
||||
if (!ttsSupported()) return null;
|
||||
const synth = window.speechSynthesis;
|
||||
// --- unified playback (neural server audio with browser fallback) ---
|
||||
|
||||
export interface PlayHandle {
|
||||
/** Engine that ended up playing (may differ from the request in auto). */
|
||||
engine: "server" | "browser";
|
||||
stop: () => void;
|
||||
done: Promise<void>;
|
||||
}
|
||||
|
||||
let currentAudio: HTMLAudioElement | null = null;
|
||||
|
||||
function stopCurrentAudio(): void {
|
||||
if (currentAudio) {
|
||||
try {
|
||||
synth.cancel();
|
||||
currentAudio.pause();
|
||||
const url = currentAudio.src;
|
||||
currentAudio.removeAttribute("src");
|
||||
currentAudio.load();
|
||||
if (url.startsWith("blob:")) URL.revokeObjectURL(url);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
const lang = preview.lang === "auto" ? detectLang(text) : preview.lang;
|
||||
const utter = new SpeechSynthesisUtterance(text.slice(0, 2000));
|
||||
utter.lang = lang;
|
||||
utter.rate = preview.rate;
|
||||
utter.pitch = preview.pitch;
|
||||
const pick = pickVoice(voices, { lang, timbre: preview.timbre });
|
||||
if (pick) utter.voice = pick;
|
||||
utter.onend = () => opts.onend?.();
|
||||
utter.onerror = () => opts.onerror?.("error");
|
||||
currentAudio = null;
|
||||
}
|
||||
try {
|
||||
synth.speak(utter);
|
||||
window.speechSynthesis?.cancel();
|
||||
} catch {
|
||||
return null;
|
||||
// ignore
|
||||
}
|
||||
return pick;
|
||||
}
|
||||
|
||||
// --- speech recognition (STT) ---
|
||||
/** Stop whatever is playing (server audio or browser speech). */
|
||||
export function stopPlayback(): void {
|
||||
stopCurrentAudio();
|
||||
}
|
||||
|
||||
export interface PlayRequest {
|
||||
text: string;
|
||||
/** "server" forces neural, "browser" forces synthesis, "auto" tries neural first. */
|
||||
engine: "auto" | "browser" | "server";
|
||||
browser: {
|
||||
voices: SpeechSynthesisVoice[];
|
||||
/** "auto" detects per text, otherwise a BCP-47 prefix. */
|
||||
lang: string;
|
||||
rate: number;
|
||||
pitch: number;
|
||||
/** Explicit voice URI (user pref); otherwise timbre-matched. */
|
||||
voiceURI?: string | null;
|
||||
/** Preview character timbre (ignored when voiceURI is set). */
|
||||
timbre?: VoiceTimbre;
|
||||
};
|
||||
/** Server neural voice id (preview line voice or user serverVoice). */
|
||||
serverVoice?: string | null;
|
||||
/** True when the platform TTS tool is available. */
|
||||
serverAvailable: boolean;
|
||||
fetchAudio: (text: string, voice?: string) => Promise<Blob>;
|
||||
onstart?: () => void;
|
||||
onend?: () => void;
|
||||
onerror?: (msg: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Play one text with the best engine: neural server audio when requested
|
||||
* and available (shared <audio> element, user speed), otherwise the
|
||||
* browser character voice. Single global playback — a new play stops the
|
||||
* previous one. `done` resolves when playback fully ends (fallback
|
||||
* included); it never rejects (errors go to onend/onerror like speech).
|
||||
*/
|
||||
export function playText(req: PlayRequest): PlayHandle {
|
||||
stopPlayback();
|
||||
const wantServer = req.serverAvailable && req.engine !== "browser";
|
||||
if (wantServer) return playServer(req, req.engine === "auto");
|
||||
return playBrowser(req);
|
||||
}
|
||||
|
||||
function playServer(req: PlayRequest, allowFallback: boolean): PlayHandle {
|
||||
let settled = false;
|
||||
const stop = () => {
|
||||
settled = true;
|
||||
stopCurrentAudio();
|
||||
};
|
||||
const done = (async () => {
|
||||
try {
|
||||
const blob = await req.fetchAudio(req.text.slice(0, 2000), req.serverVoice ?? undefined);
|
||||
if (settled) return;
|
||||
await playAudioBlob(blob, req, () => settled);
|
||||
} catch (e) {
|
||||
if (settled) return;
|
||||
if (allowFallback) {
|
||||
await playBrowserAudio(req, () => settled);
|
||||
return;
|
||||
}
|
||||
req.onerror?.(e instanceof Error ? e.message : "playback failed");
|
||||
}
|
||||
})();
|
||||
return { engine: "server", stop, done };
|
||||
}
|
||||
|
||||
function playAudioBlob(blob: Blob, req: PlayRequest, cancelled: () => boolean): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const audio = new Audio(url);
|
||||
audio.playbackRate = req.browser.rate;
|
||||
currentAudio = audio;
|
||||
const finish = (ok: boolean) => {
|
||||
if (currentAudio === audio) stopCurrentAudio();
|
||||
else {
|
||||
try {
|
||||
URL.revokeObjectURL(url);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
if (!cancelled() && ok) req.onend?.();
|
||||
resolve();
|
||||
};
|
||||
audio.onended = () => finish(true);
|
||||
audio.onerror = () => finish(false);
|
||||
req.onstart?.();
|
||||
audio.play().catch(() => finish(false));
|
||||
});
|
||||
}
|
||||
|
||||
function playBrowser(req: PlayRequest): PlayHandle {
|
||||
let settled = false;
|
||||
const stop = () => {
|
||||
settled = true;
|
||||
stopSpeak();
|
||||
};
|
||||
const done = playBrowserAudio(req, () => settled);
|
||||
return { engine: "browser", stop, done };
|
||||
}
|
||||
|
||||
function playBrowserAudio(req: PlayRequest, cancelled: () => boolean): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const b = req.browser;
|
||||
const lang = b.lang === "auto" ? detectLang(req.text) : b.lang;
|
||||
const explicit = b.voiceURI ? b.voices.find((v) => v.voiceURI === b.voiceURI) : undefined;
|
||||
const pick = explicit ?? pickVoice(b.voices, { lang, timbre: b.timbre ?? "any" });
|
||||
const finish = (ok: boolean, msg?: string) => {
|
||||
if (cancelled()) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
if (ok) req.onend?.();
|
||||
else req.onerror?.(msg ?? "error");
|
||||
resolve();
|
||||
};
|
||||
const ok = speak(
|
||||
req.text,
|
||||
{ voiceURI: pick?.voiceURI ?? null, rate: b.rate, pitch: b.pitch, lang, engine: "browser", serverVoice: null },
|
||||
b.voices,
|
||||
{ onstart: () => req.onstart?.(), onend: () => finish(true), onerror: (m) => finish(false, m) },
|
||||
);
|
||||
if (!ok) finish(false, "unsupported");
|
||||
});
|
||||
}
|
||||
|
||||
export interface Recognizer {
|
||||
lang: string;
|
||||
|
||||
@@ -40,3 +40,13 @@ Le modèle doit détecter la langue par le texte/etc. l'user peut customiser l'a
|
||||
|
||||
Ces exemples sont les default voix preview . dans l'admin center , l'utilisateur peut les modifier /revenir à défaut
|
||||
L'utilisateur doit pouvoir configurée ces ai,vocaux,voix,langue dans une sous-section dédier dans ces paramètres.
|
||||
|
||||
--- actually the favicon have an background svg as the same color of the accenthique theme. should be indepandant .
|
||||
white = day theme
|
||||
black / night theme
|
||||
witht the flower itself following thr accentic color
|
||||
|
||||
|
||||
-- in specify
|
||||
specify the model used in each footer of assistant message
|
||||
like <model> <button>
|
||||
|
||||
@@ -80,6 +80,14 @@ export const config = {
|
||||
ocrLang: env("OCR_LANG", "eng"),
|
||||
ocrMaxChars: envInt("OCR_MAX_CHARS", 100_000),
|
||||
|
||||
// Neural TTS tool (ElevenLabs / OpenAI / local piper, first configured
|
||||
// wins unless TTS_PROVIDER forces one). Without any of these the tool
|
||||
// reports unavailable and clients fall back to browser speech.
|
||||
elevenlabsApiKey: env("ELEVENLABS_API_KEY", ""),
|
||||
elevenlabsVoice: env("ELEVENLABS_VOICE", ""),
|
||||
piperModel: env("PIPER_MODEL", ""),
|
||||
ttsVoice: env("TTS_VOICE", "alloy"),
|
||||
|
||||
// Key recovery via email (optional — empty SMTP_HOST disables it).
|
||||
smtpHost: env("SMTP_HOST", ""),
|
||||
smtpPort: envInt("SMTP_PORT", 587),
|
||||
|
||||
@@ -311,6 +311,11 @@ export async function isOcrToolEnabled(): Promise<boolean> {
|
||||
return (await getSetting("tools_ocr_enabled", "1")) === "1";
|
||||
}
|
||||
|
||||
/** Admin kill-switch for the neural TTS platform tool (default on). */
|
||||
export async function isTtsToolEnabled(): Promise<boolean> {
|
||||
return (await getSetting("tools_tts_enabled", "1")) === "1";
|
||||
}
|
||||
|
||||
/** Admin kill-switch for message reports / flag button (default on). */
|
||||
export async function isReportsEnabled(): Promise<boolean> {
|
||||
return (await getSetting("reports_enabled", "1")) === "1";
|
||||
|
||||
+50
-7
@@ -143,6 +143,7 @@ import {
|
||||
writeCdnFile,
|
||||
} from "./cdn.ts";
|
||||
import { OCR_MAX_BYTES, ocrAllowed, recordOcrJob } from "./tools/ocr.ts";
|
||||
import { TTS_MAX_CHARS, recordTtsJob, ttsAllowed } from "./tools/tts.ts";
|
||||
import { tools } from "./tools/registry.ts";
|
||||
import { initOllamaNodes } from "./ollamaNodes.ts";
|
||||
|
||||
@@ -236,13 +237,15 @@ export interface VoicePreview {
|
||||
rate: number;
|
||||
pitch: number;
|
||||
timbre: "masculine" | "feminine" | "any";
|
||||
/** Server neural voice id (ElevenLabs voice_id / OpenAI voice name). Empty = provider default. */
|
||||
voice: string;
|
||||
}
|
||||
|
||||
const DEFAULT_VOICE_PREVIEWS: VoicePreview[] = [
|
||||
{ text: "Salut, euhm, comment je peux t'aider aujourd'hui ?", lang: "fr", rate: 0.85, pitch: 0.75, timbre: "masculine" },
|
||||
{ text: "5 sur 5 je recois ! Que veux-tu ?", lang: "fr", rate: 1, pitch: 0.8, timbre: "masculine" },
|
||||
{ text: "Owww, ce chat est trop mignon, oops, pardon. Je me concentre, que puis-je faire pour toi ?", lang: "fr", rate: 1.05, pitch: 1.35, timbre: "feminine" },
|
||||
{ text: "Hey, que puis-je faire pour toi ?", lang: "fr", rate: 1, pitch: 1.1, timbre: "feminine" },
|
||||
{ text: "Salut, euhm, comment je peux t'aider aujourd'hui ?", lang: "fr", rate: 0.85, pitch: 0.75, timbre: "masculine", voice: "" },
|
||||
{ text: "5 sur 5 je recois ! Que veux-tu ?", lang: "fr", rate: 1, pitch: 0.8, timbre: "masculine", voice: "" },
|
||||
{ text: "Owww, ce chat est trop mignon, oops, pardon. Je me concentre, que puis-je faire pour toi ?", lang: "fr", rate: 1.05, pitch: 1.35, timbre: "feminine", voice: "" },
|
||||
{ text: "Hey, que puis-je faire pour toi ?", lang: "fr", rate: 1, pitch: 1.1, timbre: "feminine", voice: "" },
|
||||
];
|
||||
|
||||
function sanitizePreviewLine(v: unknown): VoicePreview | null {
|
||||
@@ -250,7 +253,7 @@ function sanitizePreviewLine(v: unknown): VoicePreview | null {
|
||||
if (typeof v === "string") {
|
||||
const text = v.trim();
|
||||
if (text.length === 0 || text.length > 500) return null;
|
||||
return { text, lang: "auto", rate: 1, pitch: 1, timbre: "any" };
|
||||
return { text, lang: "auto", rate: 1, pitch: 1, timbre: "any", voice: "" };
|
||||
}
|
||||
if (!v || typeof v !== "object") return null;
|
||||
const o = v as Record<string, unknown>;
|
||||
@@ -261,7 +264,8 @@ function sanitizePreviewLine(v: unknown): VoicePreview | null {
|
||||
const rate = typeof o.rate === "number" && o.rate >= 0.5 && o.rate <= 2 ? o.rate : 1;
|
||||
const pitch = typeof o.pitch === "number" && o.pitch >= 0.5 && o.pitch <= 2 ? o.pitch : 1;
|
||||
const timbre = o.timbre === "masculine" || o.timbre === "feminine" ? o.timbre : "any";
|
||||
return { text, lang, rate, pitch, timbre };
|
||||
const voice = typeof o.voice === "string" && /^[A-Za-z0-9_-]{1,64}$/.test(o.voice.trim()) ? o.voice.trim() : "";
|
||||
return { text, lang, rate, pitch, timbre, voice };
|
||||
}
|
||||
|
||||
async function getVoicePreviews(): Promise<VoicePreview[]> {
|
||||
@@ -1031,6 +1035,7 @@ const server = Bun.serve<{ ticketId: string | null; isAdmin: boolean }>({
|
||||
recoveryLimitMax: await recoveryLimitMax(),
|
||||
recoveryLimitWindowMin: Math.round(await recoveryLimitWindowMs() / 60_000),
|
||||
voicePreviews: await getVoicePreviews(),
|
||||
ttsEnabled: (await getSetting("tools_tts_enabled", "1")) === "1",
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1096,10 +1101,14 @@ const server = Bun.serve<{ ticketId: string | null; isAdmin: boolean }>({
|
||||
await setSetting("voice_previews", "");
|
||||
} else {
|
||||
const clean = sanitizeVoicePreviews(body.voicePreviews);
|
||||
if (!clean) return json({ error: "voicePreviews must be 1-6 lines {text, lang?, rate?, pitch?, timbre?}, or [] to reset" }, 400);
|
||||
if (!clean) return json({ error: "voicePreviews must be 1-6 lines {text, lang?, rate?, pitch?, timbre?, voice?}, or [] to reset" }, 400);
|
||||
await setSetting("voice_previews", JSON.stringify(clean));
|
||||
}
|
||||
}
|
||||
if (body.ttsEnabled !== undefined) {
|
||||
if (typeof body.ttsEnabled !== "boolean") return json({ error: "ttsEnabled must be boolean" }, 400);
|
||||
await setSetting("tools_tts_enabled", body.ttsEnabled ? "1" : "0");
|
||||
}
|
||||
return json({
|
||||
settings: {
|
||||
registrationEnabled: await isRegistrationEnabled(),
|
||||
@@ -1112,6 +1121,7 @@ const server = Bun.serve<{ ticketId: string | null; isAdmin: boolean }>({
|
||||
recoveryLimitMax: await recoveryLimitMax(),
|
||||
recoveryLimitWindowMin: Math.round(await recoveryLimitWindowMs() / 60_000),
|
||||
voicePreviews: await getVoicePreviews(),
|
||||
ttsEnabled: (await getSetting("tools_tts_enabled", "1")) === "1",
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1815,6 +1825,39 @@ const server = Bun.serve<{ ticketId: string | null; isAdmin: boolean }>({
|
||||
}
|
||||
}
|
||||
|
||||
// --- platform tools: neural TTS (natural voices, cached) ---
|
||||
if (path === "/api/tools/tts/info" && req.method === "GET") {
|
||||
const tts = tools.tts;
|
||||
return json({ ...(await tts.providerInfo()), available: await tts.isAvailable(), reason: await tts.unavailableReason() });
|
||||
}
|
||||
|
||||
if (path === "/api/tools/tts" && req.method === "POST") {
|
||||
const tts = tools.tts;
|
||||
if (!await tts.isAvailable()) return json({ error: (await tts.unavailableReason()) ?? "tts unavailable" }, 501);
|
||||
const { ok, body } = await readJson(req);
|
||||
if (!ok || typeof body.text !== "string") return json({ error: "expected { text, voice? }" }, 400);
|
||||
const text = body.text.trim();
|
||||
if (text.length === 0 || text.length > TTS_MAX_CHARS) {
|
||||
return json({ error: `text: 1-${TTS_MAX_CHARS} chars` }, 400);
|
||||
}
|
||||
const rl = ttsAllowed(user.id);
|
||||
if (!rl.ok) {
|
||||
return json({ error: "tts rate limited: 30 jobs per hour", retryAfterSec: rl.retryAfterSec }, 429);
|
||||
}
|
||||
recordTtsJob(user.id);
|
||||
try {
|
||||
const { audio, contentType } = await tts.synthesize(text, typeof body.voice === "string" ? body.voice : undefined);
|
||||
const bytes = audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength) as ArrayBuffer;
|
||||
return new Response(bytes, {
|
||||
headers: { "Content-Type": contentType, "Cache-Control": "private, max-age=86400" },
|
||||
});
|
||||
} catch (e) {
|
||||
const msg = (e as Error).message;
|
||||
if (/not available|disabled|failed to start/i.test(msg)) return json({ error: msg }, 501);
|
||||
return json({ error: msg }, 502);
|
||||
}
|
||||
}
|
||||
|
||||
if (path === "/api/chat" && req.method === "POST") {
|
||||
const { ok, body } = await readJson(req);
|
||||
if (!ok) return json({ error: "invalid JSON" }, 400);
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import { OcrTool } from "./ocr.ts";
|
||||
import { TtsTool } from "./tts.ts";
|
||||
import type { PlatformTool } from "./types.ts";
|
||||
|
||||
/** Platform tools registry (upload flows always go through tools). */
|
||||
class ToolRegistry {
|
||||
readonly ocr = new OcrTool();
|
||||
private tools: PlatformTool[] = [this.ocr];
|
||||
readonly tts = new TtsTool();
|
||||
private tools: PlatformTool[] = [this.ocr, this.tts];
|
||||
|
||||
async init(): Promise<void> {
|
||||
await this.ocr.init();
|
||||
await this.tts.init();
|
||||
}
|
||||
|
||||
async list(): Promise<{ name: string; description: string; available: boolean; reason: string | null }[]> {
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, readdirSync, rmSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { config } from "../config.ts";
|
||||
import { isTtsToolEnabled } from "../db.ts";
|
||||
import { driverLog, fmtBytes, fmtMs } from "../drivers/log.ts";
|
||||
import type { PlatformTool } from "./types.ts";
|
||||
|
||||
/**
|
||||
* Neural TTS tool: turns assistant replies / preview lines into natural
|
||||
* speech (far beyond the browser's robotic speechSynthesis).
|
||||
*
|
||||
* Providers, tried in order (first configured wins; `TTS_PROVIDER` forces
|
||||
* one explicitly):
|
||||
* - elevenlabs (ELEVENLABS_API_KEY [+ ELEVENLABS_VOICE]) — most natural,
|
||||
* any voice_id (20-char ids, clones included).
|
||||
* - openai (OPENAI_API_KEY, reuses the LLM provider key) — tts-1 with the
|
||||
* 6 stock voices (alloy/echo/fable/onyx/nova/shimmer).
|
||||
* - piper (PIPER_BIN + PIPER_MODEL, local binary like tesseract) — offline,
|
||||
* single configured voice, WAV output.
|
||||
*
|
||||
* Output is cached on disk (DATA_DIR/tts-cache, sha of provider|voice|text,
|
||||
* 200 files cap) so repeated lines cost nothing. Without any provider the
|
||||
* tool reports unavailable and the frontend falls back to browser speech.
|
||||
*/
|
||||
|
||||
export type TtsProvider = "elevenlabs" | "openai" | "piper";
|
||||
|
||||
export const OPENAI_VOICES = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] as const;
|
||||
|
||||
const TTS_MAX_CHARS = 2000;
|
||||
const TTS_TIMEOUT_MS = 60_000;
|
||||
const TTS_CACHE_CAP = 200;
|
||||
|
||||
// Abuse protection: neural TTS costs money per byte.
|
||||
const TTS_WINDOW_MS = 3600_000;
|
||||
const TTS_MAX_JOBS = 30;
|
||||
const ttsJobs = new Map<string, number[]>();
|
||||
|
||||
export function ttsAllowed(userId: string): { ok: boolean; retryAfterSec: number } {
|
||||
const now = Date.now();
|
||||
const list = (ttsJobs.get(userId) ?? []).filter((t) => t > now - TTS_WINDOW_MS);
|
||||
ttsJobs.set(userId, list);
|
||||
if (list.length >= TTS_MAX_JOBS) {
|
||||
return { ok: false, retryAfterSec: Math.max(1, Math.ceil((list[0]! + TTS_WINDOW_MS - now) / 1000)) };
|
||||
}
|
||||
return { ok: true, retryAfterSec: 0 };
|
||||
}
|
||||
|
||||
export function recordTtsJob(userId: string): void {
|
||||
const list = ttsJobs.get(userId) ?? [];
|
||||
list.push(Date.now());
|
||||
ttsJobs.set(userId, list);
|
||||
}
|
||||
|
||||
function elevenlabsBase(): string {
|
||||
return (process.env["ELEVENLABS_BASE_URL"] || "https://api.elevenlabs.io").replace(/\/$/, "");
|
||||
}
|
||||
|
||||
function openaiBase(): string {
|
||||
return (process.env["OPENAI_BASE_URL"] || "https://api.openai.com").replace(/\/$/, "");
|
||||
}
|
||||
|
||||
function piperBin(): string {
|
||||
return process.env["PIPER_BIN"] || "piper";
|
||||
}
|
||||
|
||||
async function piperAvailable(): Promise<boolean> {
|
||||
if (!config.piperModel) return false;
|
||||
try {
|
||||
const proc = Bun.spawn([piperBin(), "--version"], { stdout: "pipe", stderr: "pipe" });
|
||||
const code = await proc.exited;
|
||||
return code === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Which provider would serve right now (null = none configured). */
|
||||
export async function activeTtsProvider(): Promise<TtsProvider | null> {
|
||||
const forced = (process.env["TTS_PROVIDER"] || "auto").toLowerCase();
|
||||
const eleven = config.elevenlabsApiKey.length > 0;
|
||||
const openai = config.openaiApiKey.length > 0;
|
||||
const piper = await piperAvailable();
|
||||
const pick = (p: TtsProvider): TtsProvider | null =>
|
||||
p === "elevenlabs" ? (eleven ? p : null) : p === "openai" ? (openai ? p : null) : piper ? p : null;
|
||||
if (forced === "elevenlabs" || forced === "openai" || forced === "piper") return pick(forced);
|
||||
return pick("elevenlabs") ?? pick("openai") ?? pick("piper");
|
||||
}
|
||||
|
||||
export function sanitizeTtsVoice(provider: TtsProvider, raw: unknown): string | null {
|
||||
if (raw === undefined || raw === null || raw === "") return null;
|
||||
if (typeof raw !== "string") return null;
|
||||
const v = raw.trim();
|
||||
if (provider === "openai") {
|
||||
return (OPENAI_VOICES as readonly string[]).includes(v.toLowerCase()) ? v.toLowerCase() : null;
|
||||
}
|
||||
if (provider === "elevenlabs") {
|
||||
return /^[A-Za-z0-9_-]{1,64}$/.test(v) ? v : null;
|
||||
}
|
||||
return null; // piper: single configured model voice
|
||||
}
|
||||
|
||||
function cacheDir(): string {
|
||||
const dir = join(config.dataDir, "tts-cache");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
function cacheKey(provider: TtsProvider, voice: string, text: string): string {
|
||||
return createHash("sha256").update(`${provider}|${voice}|${text}`).digest("hex");
|
||||
}
|
||||
|
||||
function cachedPath(key: string, ext: string): string {
|
||||
return join(cacheDir(), `${key}.${ext}`);
|
||||
}
|
||||
|
||||
function pruneCache(): void {
|
||||
try {
|
||||
const dir = cacheDir();
|
||||
const files = readdirSync(dir)
|
||||
.map((f) => {
|
||||
try {
|
||||
return { f, mtime: statSync(join(dir, f)).mtimeMs };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((x): x is { f: string; mtime: number } => x !== null)
|
||||
.sort((a, b) => a.mtime - b.mtime);
|
||||
for (const old of files.slice(0, Math.max(0, files.length - TTS_CACHE_CAP))) {
|
||||
try {
|
||||
rmSync(join(dir, old.f));
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// cache must never break synthesis
|
||||
}
|
||||
}
|
||||
|
||||
async function synthElevenlabs(text: string, voiceId: string): Promise<{ audio: Uint8Array; contentType: string }> {
|
||||
const res = await fetch(`${elevenlabsBase()}/v1/text-to-speech/${encodeURIComponent(voiceId)}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "audio/mpeg",
|
||||
"xi-api-key": config.elevenlabsApiKey,
|
||||
},
|
||||
body: JSON.stringify({ text, model_id: "eleven_turbo_v2_5" }),
|
||||
signal: AbortSignal.timeout(TTS_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.text().catch(() => "");
|
||||
throw new Error(`elevenlabs tts failed (${res.status}): ${err.slice(0, 200)}`);
|
||||
}
|
||||
return { audio: new Uint8Array(await res.arrayBuffer()), contentType: "audio/mpeg" };
|
||||
}
|
||||
|
||||
async function synthOpenai(text: string, voice: string): Promise<{ audio: Uint8Array; contentType: string }> {
|
||||
const res = await fetch(`${openaiBase()}/v1/audio/speech`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${config.openaiApiKey}` },
|
||||
body: JSON.stringify({ model: "tts-1", input: text, voice, response_format: "mp3" }),
|
||||
signal: AbortSignal.timeout(TTS_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.text().catch(() => "");
|
||||
throw new Error(`openai tts failed (${res.status}): ${err.slice(0, 200)}`);
|
||||
}
|
||||
return { audio: new Uint8Array(await res.arrayBuffer()), contentType: "audio/mpeg" };
|
||||
}
|
||||
|
||||
async function synthPiper(text: string): Promise<{ audio: Uint8Array; contentType: string }> {
|
||||
const out = join(cacheDir(), `piper-${Date.now().toString(36)}-${Math.floor(Math.random() * 1e6).toString(36)}.wav`);
|
||||
let proc;
|
||||
try {
|
||||
proc = Bun.spawn([piperBin(), "--model", config.piperModel, "--output_file", out], {
|
||||
stdin: new TextEncoder().encode(text),
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`tts engine failed to start: ${(e as Error).message}`);
|
||||
}
|
||||
const timeout = setTimeout(() => {
|
||||
try {
|
||||
proc.kill();
|
||||
} catch {
|
||||
// already exited
|
||||
}
|
||||
}, TTS_TIMEOUT_MS);
|
||||
try {
|
||||
const [code, errText] = await Promise.all([proc.exited, new Response(proc.stderr).text()]);
|
||||
if (code !== 0) throw new Error(`piper failed (exit ${code}): ${errText.trim().slice(0, 200)}`);
|
||||
const file = Bun.file(out);
|
||||
if (!(await file.exists())) throw new Error("piper produced no output");
|
||||
return { audio: new Uint8Array(await file.arrayBuffer()), contentType: "audio/wav" };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
try {
|
||||
rmSync(out);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class TtsTool implements PlatformTool {
|
||||
readonly name = "tts";
|
||||
readonly description = "Neural text-to-speech (ElevenLabs / OpenAI / local piper) for natural voice playback.";
|
||||
|
||||
async init(): Promise<void> {
|
||||
const p = await activeTtsProvider();
|
||||
driverLog("tools", `tts engine: ${p ?? "none configured — browser speech fallback only"}`);
|
||||
}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return (await activeTtsProvider()) !== null && (await isTtsToolEnabled());
|
||||
}
|
||||
|
||||
async unavailableReason(): Promise<string | null> {
|
||||
if ((await activeTtsProvider()) === null) {
|
||||
return "no TTS provider configured (ELEVENLABS_API_KEY, OPENAI_API_KEY or PIPER_BIN+PIPER_MODEL). Browser speech is used instead.";
|
||||
}
|
||||
if (!(await isTtsToolEnabled())) return "TTS is disabled by the administrator (Admin Center → Features).";
|
||||
return null;
|
||||
}
|
||||
|
||||
async providerInfo(): Promise<{ provider: TtsProvider | null; voices: string[] }> {
|
||||
const p = await activeTtsProvider();
|
||||
if (!p) return { provider: null, voices: [] };
|
||||
if (p === "openai") return { provider: p, voices: [...OPENAI_VOICES] };
|
||||
if (p === "elevenlabs") return { provider: p, voices: config.elevenlabsVoice ? [config.elevenlabsVoice] : [] };
|
||||
return { provider: p, voices: [] };
|
||||
}
|
||||
|
||||
defaultVoice(provider: TtsProvider): string {
|
||||
if (provider === "openai") {
|
||||
const v = sanitizeTtsVoice("openai", config.ttsVoice);
|
||||
return v ?? "alloy";
|
||||
}
|
||||
if (provider === "elevenlabs") return config.elevenlabsVoice || "pNInz6obpgDQGcFmaJgB";
|
||||
return "";
|
||||
}
|
||||
|
||||
async synthesize(text: string, voice?: string): Promise<{ audio: Uint8Array; contentType: string; provider: TtsProvider; fromCache: boolean }> {
|
||||
const provider = await activeTtsProvider();
|
||||
if (!provider) throw new Error((await this.unavailableReason()) ?? "tts unavailable");
|
||||
if (!(await isTtsToolEnabled())) throw new Error("tts disabled");
|
||||
const clean = text.trim().slice(0, TTS_MAX_CHARS);
|
||||
if (!clean) throw new Error("empty text");
|
||||
const resolved = sanitizeTtsVoice(provider, voice) ?? this.defaultVoice(provider);
|
||||
const ext = provider === "piper" ? "wav" : "mp3";
|
||||
const key = cacheKey(provider, resolved, clean);
|
||||
const hit = Bun.file(cachedPath(key, ext));
|
||||
if (await hit.exists()) {
|
||||
return {
|
||||
audio: new Uint8Array(await hit.arrayBuffer()),
|
||||
contentType: provider === "piper" ? "audio/wav" : "audio/mpeg",
|
||||
provider,
|
||||
fromCache: true,
|
||||
};
|
||||
}
|
||||
const t0 = Date.now();
|
||||
const out = provider === "elevenlabs"
|
||||
? await synthElevenlabs(clean, resolved)
|
||||
: provider === "openai"
|
||||
? await synthOpenai(clean, resolved)
|
||||
: await synthPiper(clean);
|
||||
await Bun.write(cachedPath(key, ext), out.audio);
|
||||
pruneCache();
|
||||
driverLog("tools", `tts ${provider} voice=${resolved || "default"}: ${fmtBytes(out.audio.length)} in ${fmtMs(Date.now() - t0)}`);
|
||||
return { ...out, provider, fromCache: false };
|
||||
}
|
||||
}
|
||||
|
||||
export { TTS_MAX_CHARS };
|
||||
Reference in New Issue
Block a user