From db14c9bf216b14b5cb3f8a7080eed2ad4cbe920f Mon Sep 17 00:00:00 2001 From: Anais Saraiva Date: Mon, 21 Sep 2026 11:41:51 +0200 Subject: [PATCH] . --- .env.example | 3 +++ another_prompt | 3 +++ module.nix | 2 ++ src/config.ts | 3 +++ src/drivers/apiBase.ts | 31 ++++++++++++++++++++--- src/drivers/log.ts | 18 +++++++++++++ src/drivers/ollama.ts | 35 ++++++++++++++++++++++++-- src/index.ts | 57 ++++++++++++++++++++++++++++++++++-------- 8 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 src/drivers/log.ts diff --git a/.env.example b/.env.example index 27adf53..232b737 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,9 @@ SESSION_TTL_HOURS=720 # Data dir: holds kisassistant.db (SQLite: users, sessions, chats, shares) DATA_DIR=./data +# Per-driver debug logging to stdout ([driver:name] lines). +DRIVER_DEBUG=true + # Public docs wiki (Gitea wiki pages of this repo). GET /wiki redirects here. WIKI_URL=https://git.kisakay.com/k/chat/wiki diff --git a/another_prompt b/another_prompt index a9c345c..9ea12a9 100644 --- a/another_prompt +++ b/another_prompt @@ -4,6 +4,9 @@ cdn comme sa http/cdn//. +pouvoir gérer dans le code proprement les routes cdn . FU (futur usage/futurproof) +http:///cdn//. + Extension authorisé ONLY: jpg/jpeg/webp/png diff --git a/module.nix b/module.nix index 0159899..ea73792 100644 --- a/module.nix +++ b/module.nix @@ -75,6 +75,8 @@ in { enableACME = true; locations."/" = { proxyPass = "http://${cfg.host}:${toString cfg.port}"; + # Never buffer SSE (/api/chat streams tokens like ChatGPT). + extraConfig = "proxy_buffering off;"; }; }; }; diff --git a/src/config.ts b/src/config.ts index 6cb1ce4..e43d4a0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -41,6 +41,9 @@ export const config = { // Public docs wiki (served by the hosted git forge, e.g. Gitea wiki pages). // The app redirects GET /wiki there (302) so the target stays configurable. wikiUrl: env("WIKI_URL", "https://git.kisakay.com/k/chat/wiki"), + + // Per-driver debug logging to stdout ([driver:name] lines). + driverDebug: envBool("DRIVER_DEBUG", true), }; export function assertConfig(): void { diff --git a/src/drivers/apiBase.ts b/src/drivers/apiBase.ts index f80fd30..6ff02bb 100644 --- a/src/drivers/apiBase.ts +++ b/src/drivers/apiBase.ts @@ -1,5 +1,6 @@ import type { ChatMessage, ChatOptions, DriverModel, LLMDriver } from "./types.ts"; import { DriverDisabledError } from "./types.ts"; +import { driverLog, fmtBytes, fmtMs } from "./log.ts"; /** * Base class for OpenAI-compatible HTTP API drivers @@ -35,11 +36,12 @@ export abstract class ApiDriverBase implements LLMDriver { if (res.ok) { const data = (await res.json()) as { data?: { id: string }[] }; if (Array.isArray(data.data) && data.data.length > 0) { + driverLog(this.name, `listModels -> ${data.data.length} model(s) via /models`); return data.data.map((m) => ({ id: `${this.name}:${m.id}`, name: m.id, driver: this.name, label: m.id })); } } - } catch { - // fall through to static list + } catch (e) { + driverLog(this.name, `listModels: /models failed (${(e as Error).message}), using static list`); } return this.defaultModels.map((m) => ({ id: `${this.name}:${m}`, name: m, driver: this.name, label: m })); } @@ -53,6 +55,8 @@ export abstract class ApiDriverBase implements LLMDriver { async *chatStream(messages: ChatMessage[], opts: ChatOptions): AsyncGenerator { this.ensureEnabled(); + const t0 = Date.now(); + driverLog(this.name, `chat model=${opts.model} messages=${messages.length} stream=open`); const res = await fetch(`${this.baseUrl}/chat/completions`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` }, @@ -61,11 +65,15 @@ export abstract class ApiDriverBase implements LLMDriver { }); if (!res.ok || !res.body) { const text = await res.text().catch(() => ""); + driverLog(this.name, `chat model=${opts.model} ERROR: upstream ${res.status} ${text.slice(0, 120)}`); throw new Error(`${this.name} chat failed (${res.status}): ${text.slice(0, 300)}`); } const reader = res.body.getReader(); const decoder = new TextDecoder(); let buf = ""; + let chunks = 0; + let chars = 0; + let status = "done"; try { while (true) { const { done, value } = await reader.read(); @@ -78,11 +86,16 @@ export abstract class ApiDriverBase implements LLMDriver { const t = line.trim(); if (!t.startsWith("data:")) continue; const payload = t.slice(5).trim(); - if (payload === "[DONE]") return; + if (payload === "[DONE]") { + driverLog(this.name, `chat model=${opts.model} done: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`); + return; + } try { const obj = JSON.parse(payload) as { choices?: { delta?: { content?: string } }[] }; const token = obj.choices?.[0]?.delta?.content ?? ""; if (token) { + chunks++; + chars += token.length; opts.onToken?.(token); yield token; } @@ -92,8 +105,20 @@ export abstract class ApiDriverBase implements LLMDriver { } } } + driverLog(this.name, `chat model=${opts.model} upstream closed: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`); + } catch (e) { + status = (e as Error).name === "AbortError" ? "aborted-by-caller" : `ERROR: ${(e as Error).message}`; + throw e; } finally { + try { + await reader.cancel(); + } catch { + // already closed / consumed — fine + } reader.releaseLock(); + if (status !== "done") { + driverLog(this.name, `chat model=${opts.model} ${status} after ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`); + } } } } diff --git a/src/drivers/log.ts b/src/drivers/log.ts new file mode 100644 index 0000000..a0025c0 --- /dev/null +++ b/src/drivers/log.ts @@ -0,0 +1,18 @@ +import { config } from "../config.ts"; + +/** Concise per-driver debug logging to stdout (toggle with DRIVER_DEBUG). */ +export function driverLog(driver: string, msg: string): void { + if (!config.driverDebug) return; + const t = new Date().toISOString().slice(11, 23); // HH:MM:SS.mmm + console.log(`[${t}][driver:${driver}] ${msg}`); +} + +export function fmtBytes(n: number): string { + if (n < 1024) return `${n}B`; + return `${(n / 1024).toFixed(1)}KB`; +} + +export function fmtMs(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + return `${(ms / 1000).toFixed(1)}s`; +} diff --git a/src/drivers/ollama.ts b/src/drivers/ollama.ts index 3c16ae0..ee66bcd 100644 --- a/src/drivers/ollama.ts +++ b/src/drivers/ollama.ts @@ -1,4 +1,5 @@ import { config } from "../config.ts"; +import { driverLog, fmtBytes, fmtMs } from "./log.ts"; import type { ChatMessage, ChatOptions, DriverModel, LLMDriver } from "./types.ts"; interface OllamaTag { @@ -16,10 +17,15 @@ export class OllamaDriver implements LLMDriver { } async listModels(): Promise { + const t0 = Date.now(); const res = await fetch(`${this.host}/api/tags`, { signal: AbortSignal.timeout(8000) }); - if (!res.ok) throw new Error(`ollama /api/tags failed: ${res.status}`); + if (!res.ok) { + driverLog(this.name, `listModels ERROR: /api/tags -> ${res.status} (${fmtMs(Date.now() - t0)})`); + throw new Error(`ollama /api/tags failed: ${res.status}`); + } const data = (await res.json()) as { models?: OllamaTag[] }; const models = data.models ?? []; + driverLog(this.name, `listModels -> ${models.length} model(s) [${models.map((m) => m.name).join(", ")}] (${fmtMs(Date.now() - t0)})`); return models.map((m) => ({ id: `ollama:${m.name}`, name: m.name, @@ -35,6 +41,8 @@ export class OllamaDriver implements LLMDriver { } async *chatStream(messages: ChatMessage[], opts: ChatOptions): AsyncGenerator { + const t0 = Date.now(); + driverLog(this.name, `chat model=${opts.model} messages=${messages.length} stream=open`); const res = await fetch(`${this.host}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -43,11 +51,15 @@ export class OllamaDriver implements LLMDriver { }); if (!res.ok || !res.body) { const text = await res.text().catch(() => ""); + driverLog(this.name, `chat model=${opts.model} ERROR: upstream ${res.status} ${text.slice(0, 120)}`); throw new Error(`ollama chat failed (${res.status}): ${text.slice(0, 300)}`); } const reader = res.body.getReader(); const decoder = new TextDecoder(); let buf = ""; + let chunks = 0; + let chars = 0; + let status = "done"; try { while (true) { const { done, value } = await reader.read(); @@ -63,18 +75,37 @@ export class OllamaDriver implements LLMDriver { if (obj.error) throw new Error(`ollama error: ${obj.error}`); const token = obj.message?.content ?? ""; if (token) { + chunks++; + chars += token.length; opts.onToken?.(token); yield token; } - if (obj.done) return; + if (obj.done) { + driverLog(this.name, `chat model=${opts.model} done: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`); + return; + } } catch (e) { if (e instanceof SyntaxError) continue; // partial line, ignore throw e; } } } + driverLog(this.name, `chat model=${opts.model} upstream closed: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`); + } catch (e) { + status = (e as Error).name === "AbortError" ? "aborted-by-caller" : `ERROR: ${(e as Error).message}`; + throw e; } finally { + // Stop the upstream generation when the consumer goes away + // (client disconnect) instead of leaking it in the background. + try { + await reader.cancel(); + } catch { + // already closed / consumed — fine + } reader.releaseLock(); + if (status !== "done") { + driverLog(this.name, `chat model=${opts.model} ${status} after ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`); + } } } } diff --git a/src/index.ts b/src/index.ts index 97928ee..c0b3902 100644 --- a/src/index.ts +++ b/src/index.ts @@ -377,7 +377,7 @@ const server = Bun.serve({ if (!wantStream) { try { - const text = await driver.chat(messages, { model }); + const text = await driver.chat(messages, { model, signal: req.signal }); persistReply(text); return json({ model: body.model, message: { role: "assistant", content: text } }); } catch (e) { @@ -385,32 +385,69 @@ const server = Bun.serve({ } } + // True streaming (ChatGPT-style): driver tokens are forwarded as SSE + // events the moment they arrive. Client disconnects are propagated + // upstream so the model stops generating, and we never write to a + // closed controller (the old "[interrupted: Controller is already + // closed]" crash). + const abortUpstream = new AbortController(); + const onClientAbort = () => abortUpstream.abort(); + req.signal.addEventListener("abort", onClientAbort); + let clientGone = false; const stream = new ReadableStream({ async start(controller) { const enc = new TextEncoder(); - const send = (event: string, data: string) => - controller.enqueue(enc.encode(`event: ${event}\ndata: ${data}\n\n`)); + const safeSend = (event: string, data: string): boolean => { + if (clientGone || req.signal.aborted) return false; + try { + controller.enqueue(enc.encode(`event: ${event}\ndata: ${data}\n\n`)); + return true; + } catch { + // Controller already closed (client gone) — stop pulling upstream. + clientGone = true; + abortUpstream.abort(); + return false; + } + }; let full = ""; try { - for await (const token of driver.chatStream(messages, { model })) { + for await (const token of driver.chatStream(messages, { model, signal: abortUpstream.signal })) { + if (req.signal.aborted) break; full += token; - send("token", JSON.stringify({ token })); + if (!safeSend("token", JSON.stringify({ token }))) break; } persistReply(full); - send("done", "{}"); + safeSend("done", "{}"); } catch (e) { - if (full) persistReply(full + "\n\n[interrupted: " + (e as Error).message + "]"); - send("error", JSON.stringify({ error: (e as Error).message })); + // Genuine driver error: keep the partial text as-is (no internal + // markers persisted) and notify the client if still connected. + if (full) persistReply(full); + if (!req.signal.aborted && !clientGone) { + safeSend("error", JSON.stringify({ error: (e as Error).message })); + } } finally { - controller.close(); + req.signal.removeEventListener("abort", onClientAbort); + abortUpstream.abort(); + if (!clientGone) { + try { + controller.close(); + } catch { + // already closed — fine + } + } } }, + cancel() { + clientGone = true; + abortUpstream.abort(); + }, }); return new Response(stream, { headers: { "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", + "Cache-Control": "no-cache, no-transform", Connection: "keep-alive", + "X-Accel-Buffering": "no", // nginx: never buffer SSE }, }); }