k/chat
1
Fork 0
Code Issues Pull Requests Wiki Activity

.

Browse Source
This commit is contained in:
k
2026-09-21 11:41:51 +02:00
parent d05dad2e48
commit db14c9bf21
8 changed files with 137 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/drivers/apiBase.ts
+28 -3
View File
@@ -1,5 +1,6 @@
import type { ChatMessage, ChatOptions, DriverModel, LLMDriver } from "./types.ts";
import { DriverDisabledError } from "./types.ts";
import { driverLog, fmtBytes, fmtMs } from "./log.ts";
/**
* Base class for OpenAI-compatible HTTP API drivers
@@ -35,11 +36,12 @@ export abstract class ApiDriverBase implements LLMDriver {
if (res.ok) {
const data = (await res.json()) as { data?: { id: string }[] };
if (Array.isArray(data.data) && data.data.length > 0) {
driverLog(this.name, `listModels -> ${data.data.length} model(s) via /models`);
return data.data.map((m) => ({ id: `${this.name}:${m.id}`, name: m.id, driver: this.name, label: m.id }));
}
}
} catch {
// fall through to static list
} catch (e) {
driverLog(this.name, `listModels: /models failed (${(e as Error).message}), using static list`);
}
return this.defaultModels.map((m) => ({ id: `${this.name}:${m}`, name: m, driver: this.name, label: m }));
}
@@ -53,6 +55,8 @@ export abstract class ApiDriverBase implements LLMDriver {
async *chatStream(messages: ChatMessage[], opts: ChatOptions): AsyncGenerator<string, void, void> {
this.ensureEnabled();
const t0 = Date.now();
driverLog(this.name, `chat model=${opts.model} messages=${messages.length} stream=open`);
const res = await fetch(`${this.baseUrl}/chat/completions`, {
method: "POST",
headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` },
@@ -61,11 +65,15 @@ export abstract class ApiDriverBase implements LLMDriver {
});
if (!res.ok || !res.body) {
const text = await res.text().catch(() => "");
driverLog(this.name, `chat model=${opts.model} ERROR: upstream ${res.status} ${text.slice(0, 120)}`);
throw new Error(`${this.name} chat failed (${res.status}): ${text.slice(0, 300)}`);
}
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = "";
let chunks = 0;
let chars = 0;
let status = "done";
try {
while (true) {
const { done, value } = await reader.read();
@@ -78,11 +86,16 @@ export abstract class ApiDriverBase implements LLMDriver {
const t = line.trim();
if (!t.startsWith("data:")) continue;
const payload = t.slice(5).trim();
if (payload === "[DONE]") return;
if (payload === "[DONE]") {
driverLog(this.name, `chat model=${opts.model} done: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
return;
}
try {
const obj = JSON.parse(payload) as { choices?: { delta?: { content?: string } }[] };
const token = obj.choices?.[0]?.delta?.content ?? "";
if (token) {
chunks++;
chars += token.length;
opts.onToken?.(token);
yield token;
}
@@ -92,8 +105,20 @@ export abstract class ApiDriverBase implements LLMDriver {
}
}
}
driverLog(this.name, `chat model=${opts.model} upstream closed: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
} catch (e) {
status = (e as Error).name === "AbortError" ? "aborted-by-caller" : `ERROR: ${(e as Error).message}`;
throw e;
} finally {
try {
await reader.cancel();
} catch {
// already closed / consumed — fine
}
reader.releaseLock();
if (status !== "done") {
driverLog(this.name, `chat model=${opts.model} ${status} after ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
}
}
}
}
src/drivers/log.ts
+18
View File
@@ -0,0 +1,18 @@
import { config } from "../config.ts";
/** Concise per-driver debug logging to stdout (toggle with DRIVER_DEBUG). */
export function driverLog(driver: string, msg: string): void {
if (!config.driverDebug) return;
const t = new Date().toISOString().slice(11, 23); // HH:MM:SS.mmm
console.log(`[${t}][driver:${driver}] ${msg}`);
}
export function fmtBytes(n: number): string {
if (n < 1024) return `${n}B`;
return `${(n / 1024).toFixed(1)}KB`;
}
export function fmtMs(ms: number): string {
if (ms < 1000) return `${Math.round(ms)}ms`;
return `${(ms / 1000).toFixed(1)}s`;
}
src/drivers/ollama.ts
+33 -2
View File
@@ -1,4 +1,5 @@
import { config } from "../config.ts";
import { driverLog, fmtBytes, fmtMs } from "./log.ts";
import type { ChatMessage, ChatOptions, DriverModel, LLMDriver } from "./types.ts";
interface OllamaTag {
@@ -16,10 +17,15 @@ export class OllamaDriver implements LLMDriver {
}
async listModels(): Promise<DriverModel[]> {
const t0 = Date.now();
const res = await fetch(`${this.host}/api/tags`, { signal: AbortSignal.timeout(8000) });
if (!res.ok) throw new Error(`ollama /api/tags failed: ${res.status}`);
if (!res.ok) {
driverLog(this.name, `listModels ERROR: /api/tags -> ${res.status} (${fmtMs(Date.now() - t0)})`);
throw new Error(`ollama /api/tags failed: ${res.status}`);
}
const data = (await res.json()) as { models?: OllamaTag[] };
const models = data.models ?? [];
driverLog(this.name, `listModels -> ${models.length} model(s) [${models.map((m) => m.name).join(", ")}] (${fmtMs(Date.now() - t0)})`);
return models.map((m) => ({
id: `ollama:${m.name}`,
name: m.name,
@@ -35,6 +41,8 @@ export class OllamaDriver implements LLMDriver {
}
async *chatStream(messages: ChatMessage[], opts: ChatOptions): AsyncGenerator<string, void, void> {
const t0 = Date.now();
driverLog(this.name, `chat model=${opts.model} messages=${messages.length} stream=open`);
const res = await fetch(`${this.host}/api/chat`, {
method: "POST",
headers: { "Content-Type": "application/json" },
@@ -43,11 +51,15 @@ export class OllamaDriver implements LLMDriver {
});
if (!res.ok || !res.body) {
const text = await res.text().catch(() => "");
driverLog(this.name, `chat model=${opts.model} ERROR: upstream ${res.status} ${text.slice(0, 120)}`);
throw new Error(`ollama chat failed (${res.status}): ${text.slice(0, 300)}`);
}
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = "";
let chunks = 0;
let chars = 0;
let status = "done";
try {
while (true) {
const { done, value } = await reader.read();
@@ -63,18 +75,37 @@ export class OllamaDriver implements LLMDriver {
if (obj.error) throw new Error(`ollama error: ${obj.error}`);
const token = obj.message?.content ?? "";
if (token) {
chunks++;
chars += token.length;
opts.onToken?.(token);
yield token;
}
if (obj.done) return;
if (obj.done) {
driverLog(this.name, `chat model=${opts.model} done: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
return;
}
} catch (e) {
if (e instanceof SyntaxError) continue; // partial line, ignore
throw e;
}
}
}
driverLog(this.name, `chat model=${opts.model} upstream closed: ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
} catch (e) {
status = (e as Error).name === "AbortError" ? "aborted-by-caller" : `ERROR: ${(e as Error).message}`;
throw e;
} finally {
// Stop the upstream generation when the consumer goes away
// (client disconnect) instead of leaking it in the background.
try {
await reader.cancel();
} catch {
// already closed / consumed — fine
}
reader.releaseLock();
if (status !== "done") {
driverLog(this.name, `chat model=${opts.model} ${status} after ${chunks} chunks, ${fmtBytes(chars)} in ${fmtMs(Date.now() - t0)}`);
}
}
}
}