Compare commits
4
Commits
d37f4efd69
...
0e44bf6364
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0e44bf6364 | ||
|
|
db14c9bf21 | ||
|
|
d05dad2e48 | ||
|
|
56628fa103 |
@@ -10,6 +10,12 @@ SESSION_TTL_HOURS=720
|
|||||||
# Data dir: holds kisassistant.db (SQLite: users, sessions, chats, shares)
|
# Data dir: holds kisassistant.db (SQLite: users, sessions, chats, shares)
|
||||||
DATA_DIR=./data
|
DATA_DIR=./data
|
||||||
|
|
||||||
|
# Per-driver debug logging to stdout ([driver:name] lines).
|
||||||
|
DRIVER_DEBUG=true
|
||||||
|
|
||||||
|
# Public docs wiki (Gitea wiki pages of this repo). GET /wiki redirects here.
|
||||||
|
WIKI_URL=https://git.kisakay.com/k/chat/wiki
|
||||||
|
|
||||||
# --- rate limit on /api/auth/login (per IP, sliding window) ---
|
# --- rate limit on /api/auth/login (per IP, sliding window) ---
|
||||||
RATE_LIMIT_MAX=5
|
RATE_LIMIT_MAX=5
|
||||||
RATE_LIMIT_WINDOW_MIN=10
|
RATE_LIMIT_WINDOW_MIN=10
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
--- HIGH PRIORITY
|
||||||
|
Being abble to :
|
||||||
|
Disable features / Enable it:
|
||||||
|
ie : thinking, deep search, etc...
|
||||||
|
|
||||||
|
Downloading models throught a catalog : ollama only
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
--- VERY HIGH PRIORITY
|
||||||
|
fait en sorte de pouvoir upload un avatar (Servir un cdn en local via fichier upload sur le serveur dans un dossier à côté de la codebase)
|
||||||
|
|
||||||
|
cdn comme sa
|
||||||
|
|
||||||
|
http<remote>/cdn/<avatar>/<id of the account>.<extension>
|
||||||
|
|
||||||
|
pouvoir gérer dans le code proprement les routes cdn . FU (futur usage/futurproof)
|
||||||
|
http<?s>://<root host>/cdn/<x>/<key>.<fileTypeAllowed>
|
||||||
|
|
||||||
|
Extension authorisé ONLY:
|
||||||
|
jpg/jpeg/webp/png
|
||||||
|
|
||||||
|
BLACKLISTE SYSTÈMATHIQUEMENT LE RESTE.
|
||||||
|
Pas que un detect typeFile avec l'extension.
|
||||||
|
Logique propre de détection du fichier *magic byte, looking bytes, etc*
|
||||||
|
|
||||||
|
Changer l 'avatar à plusieurs règles:
|
||||||
|
Rate-limit per change / per account de:
|
||||||
|
5 fois maximum en 2 heures
|
||||||
|
(window time: 2 hours, max count 5)
|
||||||
|
file length maxi: 5MB
|
||||||
|
|
||||||
|
SYSTEM DE CHANGEMENT DE MDP:
|
||||||
|
Donc avoir une upload box, propre, adapative etc
|
||||||
|
|
||||||
|
---- MEDIUM PRIORITY
|
||||||
|
ADAPTATION MOBILE:
|
||||||
|
GÉRER LES MODALS, LES BOXES, LES BOARDING, LES CHAT BOXES AUSSI POUR PETIT ÉCRAN, ADAPTER LEUR WEBVIEW PORTAGE, DIALOG BOX, CONFIRMATON, TRAY CHANNELS SIDE LIST...
|
||||||
|
DRAG AND SNAP LIKE PHONES, SCROLLING FOR OPENGING/EXITING ETC..
|
||||||
|
|
||||||
|
--- LOW PRIORITY
|
||||||
|
Système de service mails aussi pour les comptes (optionel) pour recovery passwords, dans le page de login:
|
||||||
|
- Recover password with emails. smtp server,
|
||||||
+14
-1
@@ -22,7 +22,20 @@ browser ──► Bun.serve (src/index.ts)
|
|||||||
are auto-titled from it.
|
are auto-titled from it.
|
||||||
4. Non-streaming → `driver.chat()` → JSON. Streaming → `driver.chatStream()`
|
4. Non-streaming → `driver.chat()` → JSON. Streaming → `driver.chatStream()`
|
||||||
→ SSE events `token` / `done` / `error`. The full reply is persisted after a
|
→ SSE events `token` / `done` / `error`. The full reply is persisted after a
|
||||||
successful stream (partial text + `[interrupted]` marker on failure).
|
successful stream; on error the partial text is kept as-is (no internal
|
||||||
|
markers). Server logs each stream open/done/abort per driver when
|
||||||
|
`DRIVER_DEBUG=true`.
|
||||||
|
|
||||||
|
Streaming guarantees (ChatGPT-style, no buffering):
|
||||||
|
|
||||||
|
- Tokens are forwarded via `controller.enqueue` the moment they arrive from the
|
||||||
|
LLM; disconnects are detected (`req.signal`, guarded `enqueue`,
|
||||||
|
stream `cancel()`) and propagated upstream via an `AbortController` passed as
|
||||||
|
`opts.signal`, so the model stops generating when the tab closes.
|
||||||
|
- `Bun.serve` runs with `idleTimeout: 0` — the default 10 s would kill slow
|
||||||
|
streams (cold model load, thinking pauses) mid-generation.
|
||||||
|
- Responses carry `X-Accel-Buffering: no` and the NixOS nginx vhost sets
|
||||||
|
`proxy_buffering off`, otherwise the proxy would buffer the whole reply.
|
||||||
|
|
||||||
## Database schema
|
## Database schema
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Copy `.env.example` to `.env`. Full list:
|
|||||||
| `APP_PASSWORD` | *(required)* | **admin** access key — generate with `openssl rand -base64 24` |
|
| `APP_PASSWORD` | *(required)* | **admin** access key — generate with `openssl rand -base64 24` |
|
||||||
| `SESSION_TTL_HOURS` | `720` | Bearer session lifetime (30 days) |
|
| `SESSION_TTL_HOURS` | `720` | Bearer session lifetime (30 days) |
|
||||||
| `DATA_DIR` | `./data` | holds `kisassistant.db` (SQLite) |
|
| `DATA_DIR` | `./data` | holds `kisassistant.db` (SQLite) |
|
||||||
|
| `WIKI_URL` | `https://git.kisakay.com/k/chat/wiki` | remote docs wiki — `GET /wiki` redirects (302) there |
|
||||||
| `RATE_LIMIT_MAX` / `RATE_LIMIT_WINDOW_MIN` | `5` / `10` | login rate limit per IP |
|
| `RATE_LIMIT_MAX` / `RATE_LIMIT_WINDOW_MIN` | `5` / `10` | login rate limit per IP |
|
||||||
| `OLLAMA_HOST` / `OLLAMA_ENABLED` | `http://10.66.66.4:11434` / `true` | local-model driver |
|
| `OLLAMA_HOST` / `OLLAMA_ENABLED` | `http://10.66.66.4:11434` / `true` | local-model driver |
|
||||||
| `MISTRAL_*`, `DEEPSEEK_*`, `ANTHROPIC_*`, `OPENAI_*` | disabled | API drivers: set `*_ENABLED=true` **and** `*_API_KEY` |
|
| `MISTRAL_*`, `DEEPSEEK_*`, `ANTHROPIC_*`, `OPENAI_*` | disabled | API drivers: set `*_ENABLED=true` **and** `*_API_KEY` |
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { useEffect, useRef, useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
import { Bot, Cpu, PanelLeftOpen, SendHorizontal, Share2, User as UserIcon } from "lucide-react";
|
import { BookOpen, Bot, Cpu, PanelLeftOpen, SendHorizontal, Share2, User as UserIcon } from "lucide-react";
|
||||||
import type { ChatMessage, Conversation, DriverModel, User } from "../lib/types.ts";
|
import type { ChatMessage, Conversation, DriverModel, User } from "../lib/types.ts";
|
||||||
import { Avatar, IconButton, LOGO_URL, Logo, Picker, type PickerGroup, Spinner } from "./ui.tsx";
|
import { Avatar, IconButton, LOGO_URL, Logo, Picker, type PickerGroup, Spinner } from "./ui.tsx";
|
||||||
import { Markdown } from "./Markdown.tsx";
|
import { Markdown } from "./Markdown.tsx";
|
||||||
@@ -180,7 +180,12 @@ export function Chat({
|
|||||||
<SendHorizontal size={17} />
|
<SendHorizontal size={17} />
|
||||||
</button>
|
</button>
|
||||||
</form>
|
</form>
|
||||||
<p className="py-2 text-center text-xs opacity-50">No cookies — a Bearer key is stored in this browser only.</p>
|
<p className="py-2 text-center text-xs opacity-50">
|
||||||
|
<a href="/wiki" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 underline-offset-2 hover:underline">
|
||||||
|
<BookOpen size={12} />
|
||||||
|
Wiki & docs
|
||||||
|
</a>
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -118,10 +118,13 @@ export function Picker({
|
|||||||
const width = Math.min(Math.max(r.width, 208), 288);
|
const width = Math.min(Math.max(r.width, 208), 288);
|
||||||
let left = align === "right" ? r.right - width : r.left;
|
let left = align === "right" ? r.right - width : r.left;
|
||||||
left = Math.max(8, Math.min(left, window.innerWidth - width - 8));
|
left = Math.max(8, Math.min(left, window.innerWidth - width - 8));
|
||||||
const maxH = 288; // matches max-h-72
|
// Use the popover's real height (not the max-h-72 cap) so short lists
|
||||||
|
// (e.g. 3 theme options) stay glued under the button instead of flipping
|
||||||
|
// above with a visible gap.
|
||||||
|
const popH = Math.min(popRef.current?.offsetHeight || 288, 288);
|
||||||
const below = r.bottom + 8;
|
const below = r.bottom + 8;
|
||||||
const top = below + maxH > window.innerHeight && r.top - maxH - 8 > 8
|
const top = below + popH > window.innerHeight && r.top - popH - 8 > 8
|
||||||
? Math.max(8, r.top - maxH - 8) // flip above when there is no room below
|
? Math.max(8, r.top - popH - 8) // flip above when there is no room below
|
||||||
: below;
|
: below;
|
||||||
setPos({ top, left, width });
|
setPos({ top, left, width });
|
||||||
return true;
|
return true;
|
||||||
|
|||||||
@@ -75,6 +75,8 @@ in {
|
|||||||
enableACME = true;
|
enableACME = true;
|
||||||
locations."/" = {
|
locations."/" = {
|
||||||
proxyPass = "http://${cfg.host}:${toString cfg.port}";
|
proxyPass = "http://${cfg.host}:${toString cfg.port}";
|
||||||
|
# Never buffer SSE (/api/chat streams tokens like ChatGPT).
|
||||||
|
extraConfig = "proxy_buffering off;";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -37,6 +37,13 @@ export const config = {
|
|||||||
|
|
||||||
openaiApiKey: env("OPENAI_API_KEY", ""),
|
openaiApiKey: env("OPENAI_API_KEY", ""),
|
||||||
openaiEnabled: envBool("OPENAI_ENABLED", false),
|
openaiEnabled: envBool("OPENAI_ENABLED", false),
|
||||||
|
|
||||||
|
// Public docs wiki (served by the hosted git forge, e.g. Gitea wiki pages).
|
||||||
|
// The app redirects GET /wiki there (302) so the target stays configurable.
|
||||||
|
wikiUrl: env("WIKI_URL", "https://git.kisakay.com/k/chat/wiki"),
|
||||||
|
|
||||||
|
// Per-driver debug logging to stdout ([driver:name] lines).
|
||||||
|
driverDebug: envBool("DRIVER_DEBUG", true),
|
||||||
};
|
};
|
||||||
|
|
||||||
export function assertConfig(): void {
|
export function assertConfig(): void {
|
||||||
|
|||||||
+28
-3
@@ -1,5 +1,6 @@
|
|||||||
import type { ChatMessage, ChatOptions, DriverModel, LLMDriver } from "./types.ts";
|
import type { ChatMessage, ChatOptions, DriverModel, LLMDriver } from "./types.ts";
|
||||||
import { DriverDisabledError } from "./types.ts";
|
import { DriverDisabledError } from "./types.ts";
|
||||||
|
import { driverLog, fmtBytes, fmtMs } from "./log.ts";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Base class for OpenAI-compatible HTTP API drivers
|
* Base class for OpenAI-compatible HTTP API drivers
|
||||||
@@ -35,11 +36,12 @@ export abstract class ApiDriverBase implements LLMDriver {
|
|||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const data = (await res.json()) as { data?: { id: string }[] };
|
const data = (await res.json()) as { data?: { id: string }[] };
|
||||||
if (Array.isArray(data.data) && data.data.length > 0) {
|
if (Array.isArray(data.data) && data.data.length > 0) {
|
||||||
|
driverLog(this.name, `listModels -> ${data.data.length} model(s) via /models`);
|
||||||
return data.data.map((m) => ({ id: `${this.name}:${m.id}`, name: m.id, driver: this.name, label: m.id }));
|
return data.data.map((m) => ({ id: `${this.name}:${m.id}`, name: m.id, driver: this.name, label: m.id }));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch {
|
} catch (e) {
|
||||||
// fall through to static list
|
driverLog(this.name, `listModels: /models failed (${(e as Error).message}), using static list`);
|
||||||
}
|
}
|
||||||
return this.defaultModels.map((m) => ({ id: `${this.name}:${m}`, name: m, driver: this.name, label: m }));
|
return this.defaultModels.map((m) => ({ id: `${this.name}:${m}`, name: m, driver: this.name, label: m }));
|
||||||
}
|
}
|
||||||
@@ -53,6 +55,8 @@ export abstract class ApiDriverBase implements LLMDriver {
|
|||||||
|
|
||||||
async *chatStream(messages: ChatMessage[], opts: ChatOptions): AsyncGenerator<string, void, void> {
|
async *chatStream(messages: ChatMessage[], opts: ChatOptions): AsyncGenerator<string, void, void> {
|
||||||
this.ensureEnabled();
|
this.ensureEnabled();
|
||||||
|
const t0 = Date.now();
|
||||||
|
driverLog(this.name, `chat model=${opts.model} messages=${messages.length} stream=open`);
|
||||||
const res = await fetch(`${this.baseUrl}/chat/completions`, {
|
const res = await fetch(`${this.baseUrl}/chat/completions`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` },
|
headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` },
|
||||||
@@ -61,11 +65,15 @@ export abstract class ApiDriverBase implements LLMDriver {
|
|||||||
});
|
});
|
||||||
if (!res.ok || !res.body) {
|
if (!res.ok || !res.body) {
|
||||||
const text = await res.text().catch(() => "");
|
const text = await res.text().catch(() => "");
|
||||||
|
driverLog(this.name, `chat model=${opts.model} ERROR: upstream ${res.status} ${text.slice(0, 120)}`);
|
||||||
throw new Error(`${this.name} chat failed (${res.status}): ${text.slice(0, 300)}`);
|
throw new Error(`${this.name} chat failed (${res.status}): ${text.slice(0, 300)}`);
|
||||||
}
|
}
|
||||||
const reader = res.body.getReader();
|
const reader = res.body.getReader();
|
||||||
const decoder = new TextDecoder();
|
const decoder = new TextDecoder();
|
||||||
let buf = "";
|
let buf = "";
|
||||||
|
let chunks = 0;
|
||||||
|
let chars = 0;
|
||||||
|
let status = "done";
|
||||||
try {
|
try {
|
||||||
while (true) {
|
while (true) {
|
||||||
const { done, value } = await reader.read();
|
const { done, value } = await reader.read();
|
||||||
@@ -78,11 +86,16 @@ export abstract class ApiDriverBase implements LLMDriver {
|
|||||||
const t = line.trim();
|
const t = line.trim();
|
||||||
if (!t.startsWith("data:")) continue;
|
if (!t.startsWith("data:")) continue;
|
||||||
const payload = t.slice(5).trim();
|
const payload = t.slice(5).trim();
|
||||||
if (payload === "[DONE]") return;
|
if (payload === "[DONE]") {
|
||||||
|
driverLog(this.name, `chat model=${opts.model} done: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
const obj = JSON.parse(payload) as { choices?: { delta?: { content?: string } }[] };
|
const obj = JSON.parse(payload) as { choices?: { delta?: { content?: string } }[] };
|
||||||
const token = obj.choices?.[0]?.delta?.content ?? "";
|
const token = obj.choices?.[0]?.delta?.content ?? "";
|
||||||
if (token) {
|
if (token) {
|
||||||
|
chunks++;
|
||||||
|
chars += token.length;
|
||||||
opts.onToken?.(token);
|
opts.onToken?.(token);
|
||||||
yield token;
|
yield token;
|
||||||
}
|
}
|
||||||
@@ -92,8 +105,20 @@ export abstract class ApiDriverBase implements LLMDriver {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
driverLog(this.name, `chat model=${opts.model} upstream closed: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
|
||||||
|
} catch (e) {
|
||||||
|
status = (e as Error).name === "AbortError" ? "aborted-by-caller" : `ERROR: ${(e as Error).message}`;
|
||||||
|
throw e;
|
||||||
} finally {
|
} finally {
|
||||||
|
try {
|
||||||
|
await reader.cancel();
|
||||||
|
} catch {
|
||||||
|
// already closed / consumed — fine
|
||||||
|
}
|
||||||
reader.releaseLock();
|
reader.releaseLock();
|
||||||
|
if (status !== "done") {
|
||||||
|
driverLog(this.name, `chat model=${opts.model} ${status} after ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
import { config } from "../config.ts";
|
||||||
|
|
||||||
|
/** Concise per-driver debug logging to stdout (toggle with DRIVER_DEBUG). */
|
||||||
|
export function driverLog(driver: string, msg: string): void {
|
||||||
|
if (!config.driverDebug) return;
|
||||||
|
const t = new Date().toISOString().slice(11, 23); // HH:MM:SS.mmm
|
||||||
|
console.log(`[${t}][driver:${driver}] ${msg}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function fmtBytes(n: number): string {
|
||||||
|
if (n < 1024) return `${n}B`;
|
||||||
|
return `${(n / 1024).toFixed(1)}KB`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function fmtMs(ms: number): string {
|
||||||
|
if (ms < 1000) return `${Math.round(ms)}ms`;
|
||||||
|
return `${(ms / 1000).toFixed(1)}s`;
|
||||||
|
}
|
||||||
+33
-2
@@ -1,4 +1,5 @@
|
|||||||
import { config } from "../config.ts";
|
import { config } from "../config.ts";
|
||||||
|
import { driverLog, fmtBytes, fmtMs } from "./log.ts";
|
||||||
import type { ChatMessage, ChatOptions, DriverModel, LLMDriver } from "./types.ts";
|
import type { ChatMessage, ChatOptions, DriverModel, LLMDriver } from "./types.ts";
|
||||||
|
|
||||||
interface OllamaTag {
|
interface OllamaTag {
|
||||||
@@ -16,10 +17,15 @@ export class OllamaDriver implements LLMDriver {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async listModels(): Promise<DriverModel[]> {
|
async listModels(): Promise<DriverModel[]> {
|
||||||
|
const t0 = Date.now();
|
||||||
const res = await fetch(`${this.host}/api/tags`, { signal: AbortSignal.timeout(8000) });
|
const res = await fetch(`${this.host}/api/tags`, { signal: AbortSignal.timeout(8000) });
|
||||||
if (!res.ok) throw new Error(`ollama /api/tags failed: ${res.status}`);
|
if (!res.ok) {
|
||||||
|
driverLog(this.name, `listModels ERROR: /api/tags -> ${res.status} (${fmtMs(Date.now() - t0)})`);
|
||||||
|
throw new Error(`ollama /api/tags failed: ${res.status}`);
|
||||||
|
}
|
||||||
const data = (await res.json()) as { models?: OllamaTag[] };
|
const data = (await res.json()) as { models?: OllamaTag[] };
|
||||||
const models = data.models ?? [];
|
const models = data.models ?? [];
|
||||||
|
driverLog(this.name, `listModels -> ${models.length} model(s) [${models.map((m) => m.name).join(", ")}] (${fmtMs(Date.now() - t0)})`);
|
||||||
return models.map((m) => ({
|
return models.map((m) => ({
|
||||||
id: `ollama:${m.name}`,
|
id: `ollama:${m.name}`,
|
||||||
name: m.name,
|
name: m.name,
|
||||||
@@ -35,6 +41,8 @@ export class OllamaDriver implements LLMDriver {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async *chatStream(messages: ChatMessage[], opts: ChatOptions): AsyncGenerator<string, void, void> {
|
async *chatStream(messages: ChatMessage[], opts: ChatOptions): AsyncGenerator<string, void, void> {
|
||||||
|
const t0 = Date.now();
|
||||||
|
driverLog(this.name, `chat model=${opts.model} messages=${messages.length} stream=open`);
|
||||||
const res = await fetch(`${this.host}/api/chat`, {
|
const res = await fetch(`${this.host}/api/chat`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
@@ -43,11 +51,15 @@ export class OllamaDriver implements LLMDriver {
|
|||||||
});
|
});
|
||||||
if (!res.ok || !res.body) {
|
if (!res.ok || !res.body) {
|
||||||
const text = await res.text().catch(() => "");
|
const text = await res.text().catch(() => "");
|
||||||
|
driverLog(this.name, `chat model=${opts.model} ERROR: upstream ${res.status} ${text.slice(0, 120)}`);
|
||||||
throw new Error(`ollama chat failed (${res.status}): ${text.slice(0, 300)}`);
|
throw new Error(`ollama chat failed (${res.status}): ${text.slice(0, 300)}`);
|
||||||
}
|
}
|
||||||
const reader = res.body.getReader();
|
const reader = res.body.getReader();
|
||||||
const decoder = new TextDecoder();
|
const decoder = new TextDecoder();
|
||||||
let buf = "";
|
let buf = "";
|
||||||
|
let chunks = 0;
|
||||||
|
let chars = 0;
|
||||||
|
let status = "done";
|
||||||
try {
|
try {
|
||||||
while (true) {
|
while (true) {
|
||||||
const { done, value } = await reader.read();
|
const { done, value } = await reader.read();
|
||||||
@@ -63,18 +75,37 @@ export class OllamaDriver implements LLMDriver {
|
|||||||
if (obj.error) throw new Error(`ollama error: ${obj.error}`);
|
if (obj.error) throw new Error(`ollama error: ${obj.error}`);
|
||||||
const token = obj.message?.content ?? "";
|
const token = obj.message?.content ?? "";
|
||||||
if (token) {
|
if (token) {
|
||||||
|
chunks++;
|
||||||
|
chars += token.length;
|
||||||
opts.onToken?.(token);
|
opts.onToken?.(token);
|
||||||
yield token;
|
yield token;
|
||||||
}
|
}
|
||||||
if (obj.done) return;
|
if (obj.done) {
|
||||||
|
driverLog(this.name, `chat model=${opts.model} done: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (e instanceof SyntaxError) continue; // partial line, ignore
|
if (e instanceof SyntaxError) continue; // partial line, ignore
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
driverLog(this.name, `chat model=${opts.model} upstream closed: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
|
||||||
|
} catch (e) {
|
||||||
|
status = (e as Error).name === "AbortError" ? "aborted-by-caller" : `ERROR: ${(e as Error).message}`;
|
||||||
|
throw e;
|
||||||
} finally {
|
} finally {
|
||||||
|
// Stop the upstream generation when the consumer goes away
|
||||||
|
// (client disconnect) instead of leaking it in the background.
|
||||||
|
try {
|
||||||
|
await reader.cancel();
|
||||||
|
} catch {
|
||||||
|
// already closed / consumed — fine
|
||||||
|
}
|
||||||
reader.releaseLock();
|
reader.releaseLock();
|
||||||
|
if (status !== "done") {
|
||||||
|
driverLog(this.name, `chat model=${opts.model} ${status} after ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+56
-10
@@ -114,6 +114,10 @@ async function readJson(req: Request): Promise<{ ok: boolean; body: Record<strin
|
|||||||
const server = Bun.serve({
|
const server = Bun.serve({
|
||||||
port: config.port,
|
port: config.port,
|
||||||
hostname: config.host,
|
hostname: config.host,
|
||||||
|
// SSE chat streams can legitimately go silent for a while (cold model load,
|
||||||
|
// slow/thinking models). Bun's default 10s idle timeout would kill them
|
||||||
|
// mid-generation — disable it (0). Nginx in front has its own timeouts.
|
||||||
|
idleTimeout: 0,
|
||||||
async fetch(req, server) {
|
async fetch(req, server) {
|
||||||
const url = new URL(req.url);
|
const url = new URL(req.url);
|
||||||
const path = url.pathname;
|
const path = url.pathname;
|
||||||
@@ -133,6 +137,11 @@ const server = Bun.serve({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- public: /wiki redirects to the remote docs wiki (git forge) ---
|
||||||
|
if ((path === "/wiki" || path === "/wiki/") && req.method === "GET") {
|
||||||
|
return Response.redirect(config.wikiUrl, 302);
|
||||||
|
}
|
||||||
|
|
||||||
// --- auth: login (rate limited) ---
|
// --- auth: login (rate limited) ---
|
||||||
if (path === "/api/auth/login" && req.method === "POST") {
|
if (path === "/api/auth/login" && req.method === "POST") {
|
||||||
const ip = clientIp(req, server);
|
const ip = clientIp(req, server);
|
||||||
@@ -372,7 +381,7 @@ const server = Bun.serve({
|
|||||||
|
|
||||||
if (!wantStream) {
|
if (!wantStream) {
|
||||||
try {
|
try {
|
||||||
const text = await driver.chat(messages, { model });
|
const text = await driver.chat(messages, { model, signal: req.signal });
|
||||||
persistReply(text);
|
persistReply(text);
|
||||||
return json({ model: body.model, message: { role: "assistant", content: text } });
|
return json({ model: body.model, message: { role: "assistant", content: text } });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -380,32 +389,69 @@ const server = Bun.serve({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// True streaming (ChatGPT-style): driver tokens are forwarded as SSE
|
||||||
|
// events the moment they arrive. Client disconnects are propagated
|
||||||
|
// upstream so the model stops generating, and we never write to a
|
||||||
|
// closed controller (the old "[interrupted: Controller is already
|
||||||
|
// closed]" crash).
|
||||||
|
const abortUpstream = new AbortController();
|
||||||
|
const onClientAbort = () => abortUpstream.abort();
|
||||||
|
req.signal.addEventListener("abort", onClientAbort);
|
||||||
|
let clientGone = false;
|
||||||
const stream = new ReadableStream({
|
const stream = new ReadableStream({
|
||||||
async start(controller) {
|
async start(controller) {
|
||||||
const enc = new TextEncoder();
|
const enc = new TextEncoder();
|
||||||
const send = (event: string, data: string) =>
|
const safeSend = (event: string, data: string): boolean => {
|
||||||
|
if (clientGone || req.signal.aborted) return false;
|
||||||
|
try {
|
||||||
controller.enqueue(enc.encode(`event: ${event}\ndata: ${data}\n\n`));
|
controller.enqueue(enc.encode(`event: ${event}\ndata: ${data}\n\n`));
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
// Controller already closed (client gone) — stop pulling upstream.
|
||||||
|
clientGone = true;
|
||||||
|
abortUpstream.abort();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
let full = "";
|
let full = "";
|
||||||
try {
|
try {
|
||||||
for await (const token of driver.chatStream(messages, { model })) {
|
for await (const token of driver.chatStream(messages, { model, signal: abortUpstream.signal })) {
|
||||||
|
if (req.signal.aborted) break;
|
||||||
full += token;
|
full += token;
|
||||||
send("token", JSON.stringify({ token }));
|
if (!safeSend("token", JSON.stringify({ token }))) break;
|
||||||
}
|
}
|
||||||
persistReply(full);
|
persistReply(full);
|
||||||
send("done", "{}");
|
safeSend("done", "{}");
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (full) persistReply(full + "\n\n[interrupted: " + (e as Error).message + "]");
|
// Genuine driver error: keep the partial text as-is (no internal
|
||||||
send("error", JSON.stringify({ error: (e as Error).message }));
|
// markers persisted) and notify the client if still connected.
|
||||||
} finally {
|
if (full) persistReply(full);
|
||||||
controller.close();
|
if (!req.signal.aborted && !clientGone) {
|
||||||
|
safeSend("error", JSON.stringify({ error: (e as Error).message }));
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
req.signal.removeEventListener("abort", onClientAbort);
|
||||||
|
abortUpstream.abort();
|
||||||
|
if (!clientGone) {
|
||||||
|
try {
|
||||||
|
controller.close();
|
||||||
|
} catch {
|
||||||
|
// already closed — fine
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cancel() {
|
||||||
|
clientGone = true;
|
||||||
|
abortUpstream.abort();
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
return new Response(stream, {
|
return new Response(stream, {
|
||||||
headers: {
|
headers: {
|
||||||
"Content-Type": "text/event-stream",
|
"Content-Type": "text/event-stream",
|
||||||
"Cache-Control": "no-cache",
|
"Cache-Control": "no-cache, no-transform",
|
||||||
Connection: "keep-alive",
|
Connection: "keep-alive",
|
||||||
|
"X-Accel-Buffering": "no", // nginx: never buffer SSE
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user