diff --git a/scripts/bench-usage.ts b/scripts/bench-usage.ts deleted file mode 100644 index cc8a540..0000000 --- a/scripts/bench-usage.ts +++ /dev/null @@ -1,377 +0,0 @@ -/** - * bench-usage.ts — In-process, no-network benchmark (Option B). - * - * Compares the NEW server-aggregated summary path against the LEGACY - * "fetch-everything-then-re-aggregate" path, in terms of TIME, highlighting - * the daemon-side cache that lives in `getUsageSummary`. - * - * It deliberately SKIPS the HTTP layer. In production: - * - `fetchUsageSummary()` HTTP-calls the daemon, which runs `getUsageSummary()`. - * - `fetchUsage(N)` HTTP-calls the daemon, which runs `driver.list({limit:N})`. - * The real localhost socket round-trip (~1-3 ms) is roughly CONSTANT for both - * paths, so excluding it isolates the actual computational difference — which is - * the whole point. See the takeaway printed at the end. - * - * Run from the repo root: bun scripts/bench-usage.ts [--json] - * - */ - -// @ts-ignore — bun:sqlite is a Bun built-in module (matches storage/bun.ts). -import { Database } from "bun:sqlite"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { rmSync } from "node:fs"; - -import { createBunSqliteUsageTrackingDriver } from "@routstr/sdk/storage/bun"; -import { - getUsageSummary, - __resetUsageSummaryCacheForTest, -} from "../src/daemon/http/usage-summary.ts"; -import { - getTotals, - getModelStats, - getProviderStats, - getClientStats, - getDayStats, - getHourlyToday, - getNpubStats, -} from "../src/tui/usage/data.ts"; -import type { UsageTrackingEntry } from "../src/daemon/types.ts"; - -// ─── Config ────────────────────────────────────────────────────────────────── - -const SIZES = [100, 1000, 2500, 5000, 10000, 20000]; -const SAMPLES = 25; // median of 25, after a warmup -const STEADY_REFRESHES = 10; // the monitor's ~2s refresh loop -const TZ_OFFSET = 300; // synthetic UTC-5; fixed so results are reproducible -const DAY_SPREAD = 45; // spread timestamps over ~45 days - -const jsonMode = process.argv.includes("--json"); - -// ─── Synthetic, sanitized templates (no real DB, no real identifiers) ───────── - -interface Template { - modelId: string; - baseUrl: string; - client: string; - cost: number; - satsCost: number; - promptTokens: number; - completionTokens: number; -} - -const PROVIDER_A = "https://api.example-provider-a.com/"; -const PROVIDER_B = "https://api.example-provider-b.com/"; - -const TEMPLATES: Template[] = [ - { modelId: "minimax-m3", baseUrl: PROVIDER_A, client: "client-a", cost: 0.0021, satsCost: 4, promptTokens: 320, completionTokens: 180 }, - { modelId: "kimi-k2.6", baseUrl: PROVIDER_A, client: "client-b", cost: 0.0140, satsCost: 27, promptTokens: 5200, completionTokens: 2400 }, - { modelId: "deepseek-v4-flash", baseUrl: PROVIDER_B, client: "client-c", cost: 0.0008, satsCost: 1, promptTokens: 90, completionTokens: 60 }, - { modelId: "deepseek-v4-flash", baseUrl: PROVIDER_B, client: "client-a", cost: 0.0420, satsCost: 81, promptTokens: 24000, completionTokens: 13000 }, - { modelId: "kimi-k2.6", baseUrl: PROVIDER_B, client: "client-b", cost: 0.0065, satsCost: 12, promptTokens: 1800, completionTokens: 950 }, -]; - -// Clients array (shape works for both getUsageSummary's ClientEntry and -// data.ts's ClientInfo — both only read clientId / ownerNpub). ~half get a -// synthetic ownerNpub so the npub aggregation path is exercised. -const CLIENT_IDS = [...new Set(TEMPLATES.map((t) => t.client))]; -const clients = CLIENT_IDS.map((id, i) => ({ - clientId: id, - name: id, - apiKey: `apikey-${id}`, - createdAt: 0, - ownerNpub: i % 2 === 0 ? `npub1bench${id.replace(/[^a-z]/g, "")}` : undefined, -})); - -// ─── Row generation ──────────────────────────────────────────────────────────── - -function generateRows(n: number, now: number): UsageTrackingEntry[] { - const rows: UsageTrackingEntry[] = new Array(n); - const spreadMs = DAY_SPREAD * 86_400_000; - for (let i = 0; i < n; i++) { - const t = TEMPLATES[i % TEMPLATES.length]!; - // Spread timestamps backwards from `now` over DAY_SPREAD days, with a - // little jitter so multiple buckets/hours are populated. - const jitter = (i * 97) % 86_400_000; - const timestamp = now - Math.floor((i / n) * spreadMs) - jitter; - const promptTokens = t.promptTokens + (i % 17); - const completionTokens = t.completionTokens + (i % 13); - rows[i] = { - id: `bench-${i}`, - timestamp, - modelId: t.modelId, - baseUrl: t.baseUrl, - requestId: `req-${i}`, - cost: t.cost, - satsCost: t.satsCost, - promptTokens, - completionTokens, - totalTokens: promptTokens + completionTokens, - client: t.client, - sessionId: `sess-${i % 50}`, - tags: [], - }; - } - return rows; -} - -// ─── Seeding (raw transactional insert — fast) ──────────────────────────────── - -const TABLE = "usage_tracking"; - -function seedRawDb(dbPath: string, rows: UsageTrackingEntry[]): void { - const db = new Database(dbPath); - // Match the SDK schema exactly (copied from storage/bun.ts). - db.run(` - CREATE TABLE IF NOT EXISTS ${TABLE} ( - id TEXT PRIMARY KEY, - timestamp INTEGER NOT NULL, - model_id TEXT NOT NULL, - base_url TEXT NOT NULL, - request_id TEXT NOT NULL, - cost REAL NOT NULL, - sats_cost REAL NOT NULL, - prompt_tokens INTEGER NOT NULL, - completion_tokens INTEGER NOT NULL, - total_tokens INTEGER NOT NULL, - client TEXT, - session_id TEXT, - tags TEXT - ) - `); - db.run(`CREATE INDEX IF NOT EXISTS idx_${TABLE}_timestamp ON ${TABLE}(timestamp)`); - db.run(`CREATE INDEX IF NOT EXISTS idx_${TABLE}_model_id ON ${TABLE}(model_id)`); - db.run(`CREATE INDEX IF NOT EXISTS idx_${TABLE}_base_url ON ${TABLE}(base_url)`); - - const insert = db.query(` - INSERT OR REPLACE INTO ${TABLE} ( - id, timestamp, model_id, base_url, request_id, - cost, sats_cost, prompt_tokens, completion_tokens, total_tokens, - client, session_id, tags - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - const seed = db.transaction((batch: UsageTrackingEntry[]) => { - for (const r of batch) { - insert.run( - r.id, r.timestamp, r.modelId, r.baseUrl, r.requestId, - r.cost, r.satsCost, r.promptTokens, r.completionTokens, r.totalTokens, - r.client ?? null, r.sessionId ?? null, JSON.stringify(r.tags ?? []), - ); - } - }); - seed(rows); - db.close(); -} - -// ─── Timing helpers ──────────────────────────────────────────────────────────── - -async function timeMedian(fn: () => Promise, samples = SAMPLES): Promise { - await fn(); // warmup (discarded) - const times: number[] = []; - for (let i = 0; i < samples; i++) { - const start = performance.now(); - await fn(); - times.push(performance.now() - start); - } - times.sort((a, b) => a - b); - return times[Math.floor(times.length / 2)]!; -} - -function runLegacyAggregation(rows: UsageTrackingEntry[]): void { - // Mirror exactly what the monitor's render path does on raw entries. - getTotals(rows); - getModelStats(rows); - getProviderStats(rows); - getClientStats(rows); - getDayStats(rows); - getHourlyToday(rows); - getNpubStats(rows, clients); -} - -// ─── Bench per size ────────────────────────────────────────────────────────── - -interface SizeResult { - size: number; - summaryColdMs: number; - summaryWarmMs: number; - legacyMs: number; - summaryKB: number; - legacyKB: number; - reductionX: number; - steadySummaryMs: number; - steadyLegacyMs: number; - speedupX: number; -} - -async function benchSize(size: number): Promise { - const now = Date.now(); - const dbPath = join(tmpdir(), `routstrd-bench-${size}-${process.pid}-${Date.now()}.db`); - const rows = generateRows(size, now); - seedRawDb(dbPath, rows); - - const driver = await createBunSqliteUsageTrackingDriver({ dbPath }); - - try { - // ── summary cold: reset cache before every timed call ────────────────── - const summaryColdMs = await timeMedian(async () => { - __resetUsageSummaryCacheForTest(); - await getUsageSummary(driver, clients, TZ_OFFSET); - }); - - // ── summary warm: cache already populated (warmup call is itself a hit) ─ - __resetUsageSummaryCacheForTest(); - await getUsageSummary(driver, clients, TZ_OFFSET); // populate once - const summaryWarmMs = await timeMedian(async () => { - await getUsageSummary(driver, clients, TZ_OFFSET); - }); - - // ── legacy: list(limit:N) + full client-side re-aggregation, no cache ── - const legacyMs = await timeMedian(async () => { - const listed = await driver.list({ limit: size }); - runLegacyAggregation(listed); - }); - - // ── payload bytes ────────────────────────────────────────────────────── - __resetUsageSummaryCacheForTest(); - const summaryResult = await getUsageSummary(driver, clients, TZ_OFFSET); - const listedRows = await driver.list({ limit: size }); - const summaryBytes = JSON.stringify(summaryResult).length; - const legacyBytes = JSON.stringify(listedRows).length; - - // ── steady state (the 2s refresh loop, STEADY_REFRESHES iterations) ──── - // summary: 1 cold + (N-1) warm; legacy: N × full cost (no cache). - const steadySummaryMs = - summaryColdMs + (STEADY_REFRESHES - 1) * summaryWarmMs; - const steadyLegacyMs = STEADY_REFRESHES * legacyMs; - - return { - size, - summaryColdMs, - summaryWarmMs, - legacyMs, - summaryKB: summaryBytes / 1024, - legacyKB: legacyBytes / 1024, - reductionX: legacyBytes / summaryBytes, - steadySummaryMs, - steadyLegacyMs, - speedupX: steadyLegacyMs / steadySummaryMs, - }; - } finally { - // driver holds the only open handle now; closing the file by removing it. - try { - rmSync(dbPath, { force: true }); - rmSync(`${dbPath}-shm`, { force: true }); - rmSync(`${dbPath}-wal`, { force: true }); - } catch { /* ignore */ } - } -} - -// ─── Output ────────────────────────────────────────────────────────────────── - -function fmt(n: number, dp = 2): string { - return n.toFixed(dp); -} - -function pad(s: string, width: number): string { - return s.padStart(width); -} - -function printTable(results: SizeResult[]): void { - console.log(""); - console.log("Routstr usage path benchmark — TIME (in-process, no network)"); - console.log("Daemon-side summary cache vs. legacy fetch-all-and-re-aggregate.\n"); - - // ── Payload table (per request) ────────────────────────────── - console.log("Payload per request:"); - const payHeader = [ - pad("size", 7), - pad("summary", 11), - pad("legacy", 11), - pad("reduction", 11), - ].join(" "); - console.log(payHeader); - console.log("-".repeat(payHeader.length)); - for (const r of results) { - console.log([ - pad(String(r.size), 7), - pad(fmt(r.summaryKB, 1) + " KB", 11), - pad(fmt(r.legacyKB, 1) + " KB", 11), - pad(fmt(r.reductionX, 1) + "x", 11), - ].join(" ")); - } - - // ── Time table (ms) ────────────────────────────────────────── - console.log(""); - console.log("Time (ms):"); - const timeHeader = [ - pad("size", 7), - pad("sum cold", 10), - pad("sum warm", 10), - pad("legacy", 10), - pad("steady sum", 12), - pad("steady leg", 12), - pad("speedup", 9), - ].join(" "); - console.log(timeHeader); - console.log("-".repeat(timeHeader.length)); - for (const r of results) { - console.log([ - pad(String(r.size), 7), - pad(fmt(r.summaryColdMs) + "ms", 10), - pad(fmt(r.summaryWarmMs) + "ms", 10), - pad(fmt(r.legacyMs) + "ms", 10), - pad(fmt(r.steadySummaryMs) + "ms", 12), - pad(fmt(r.steadyLegacyMs) + "ms", 12), - pad(fmt(r.speedupX, 1) + "x", 9), - ].join(" ")); - } - - console.log(""); - console.log("Legend (in-process; times = medians of " + SAMPLES + " samples, warmup discarded;"); - console.log("excludes the ~constant localhost HTTP round-trip both paths would add):"); - console.log(" size seeded usage rows in the DB"); - console.log(""); - console.log(" Payload table:"); - console.log(" summary JSON bytes of the /usage/summary response (aggregates + 50 recent rows)"); - console.log(" legacy JSON bytes of the /usage response (all rows)"); - console.log(" reduction legacy / summary"); - console.log(""); - console.log(" Time table:"); - console.log(" sum cold summary build, COLD cache (cache miss: ~15-query rebuild + assemble)"); - console.log(" sum warm summary from WARM cache (count() + return the memoized result)"); - console.log(" legacy list all rows + the client-side re-aggregation the monitor does (no cache)"); - console.log(" steady sum total ms the SUMMARY path spends over " + STEADY_REFRESHES + " refreshes of the monitor's ~2s loop,"); - console.log(" assuming data is unchanged across the window (cache stays valid):"); - console.log(" 1 cold build + " + (STEADY_REFRESHES - 1) + " warm cache hits = cold + " + (STEADY_REFRESHES - 1) + "xwarm. Since warm is ~0.01ms,"); - console.log(" this is in practice ~one cold build."); - console.log(" steady leg same " + STEADY_REFRESHES + " refreshes for LEGACY = legacy x" + STEADY_REFRESHES + " (no cache, full cost each time)"); - console.log(" speedup steady leg / steady sum"); -} - -// ─── Main ──────────────────────────────────────────────────────────────────── - -async function main(): Promise { - const results: SizeResult[] = []; - for (const size of SIZES) { - results.push(await benchSize(size)); - } - - if (jsonMode) { - console.log(JSON.stringify( - { - config: { sizes: SIZES, samples: SAMPLES, steadyRefreshes: STEADY_REFRESHES, tzOffset: TZ_OFFSET }, - note: "In-process; HTTP round-trip (~1-3ms localhost, ~constant for both) excluded.", - results, - }, - null, - 2, - )); - } else { - printTable(results); - } -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/src/tui/usage/data.ts b/src/tui/usage/data.ts index b3e6681..66821af 100644 --- a/src/tui/usage/data.ts +++ b/src/tui/usage/data.ts @@ -1,6 +1,6 @@ import type { UsageTrackingEntry } from "../../daemon/types.ts"; import { callDaemon, isDaemonRunning } from "../../utils/daemon-client.ts"; -import type { ClientStats, DayStats, ModelStats, NpubStats, ProviderStats, UsageStats, UsageSummary } from "./types.ts"; +import type { UsageStats, UsageSummary } from "./types.ts"; export { isDaemonRunning }; @@ -101,17 +101,6 @@ export async function fetchUsageSummary(): Promise { } } -export function getTodayStart(): number { - const now = new Date(); - return new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime(); -} - -export function formatDate(timestamp: number): string { - const d = new Date(timestamp); - const p = (n: number) => String(n).padStart(2, "0"); - return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`; -} - export function formatTime(timestamp: number): string { const d = new Date(timestamp); const p = (n: number) => String(n).padStart(2, "0"); @@ -124,100 +113,6 @@ export function formatNumber(n: number): string { return n.toString(); } -export function getDayStats(entries: UsageTrackingEntry[]): Map { - const days = new Map(); - for (const entry of entries) { - const date = formatDate(entry.timestamp); - const existing = days.get(date) || { date, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 }; - days.set(date, { - ...existing, - requests: existing.requests + 1, - satsCost: existing.satsCost + entry.satsCost, - promptTokens: existing.promptTokens + entry.promptTokens, - completionTokens: existing.completionTokens + entry.completionTokens, - totalTokens: existing.totalTokens + entry.totalTokens, - }); - } - return days; -} - -export function getHourlyToday(entries: UsageTrackingEntry[]): Map { - const todayStart = getTodayStart(); - const hours = new Map(); - for (const entry of entries) { - if (entry.timestamp < todayStart) continue; - const hour = new Date(entry.timestamp).getHours(); - const existing = hours.get(hour) || { - date: formatDate(entry.timestamp), requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, - }; - hours.set(hour, { - ...existing, - requests: existing.requests + 1, - satsCost: existing.satsCost + entry.satsCost, - promptTokens: existing.promptTokens + entry.promptTokens, - completionTokens: existing.completionTokens + entry.completionTokens, - totalTokens: existing.totalTokens + entry.totalTokens, - }); - } - return hours; -} - -export function getModelStats(entries: UsageTrackingEntry[]): ModelStats[] { - const models = new Map(); - for (const entry of entries) { - const existing = models.get(entry.modelId) || { - modelId: entry.modelId, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, - }; - models.set(entry.modelId, { - ...existing, - requests: existing.requests + 1, - satsCost: existing.satsCost + entry.satsCost, - promptTokens: existing.promptTokens + entry.promptTokens, - completionTokens: existing.completionTokens + entry.completionTokens, - totalTokens: existing.totalTokens + entry.totalTokens, - }); - } - return Array.from(models.values()).sort((a, b) => b.satsCost - a.satsCost); -} - -export function getProviderStats(entries: UsageTrackingEntry[]): ProviderStats[] { - const providers = new Map(); - for (const entry of entries) { - const url = entry.baseUrl || "unknown"; - const existing = providers.get(url) || { - baseUrl: url, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, - }; - providers.set(url, { - ...existing, - requests: existing.requests + 1, - satsCost: existing.satsCost + entry.satsCost, - promptTokens: existing.promptTokens + entry.promptTokens, - completionTokens: existing.completionTokens + entry.completionTokens, - totalTokens: existing.totalTokens + entry.totalTokens, - }); - } - return Array.from(providers.values()).sort((a, b) => b.satsCost - a.satsCost); -} - -export function getClientStats(entries: UsageTrackingEntry[]): ClientStats[] { - const clients = new Map(); - for (const entry of entries) { - const client = entry.client || "unknown"; - const existing = clients.get(client) || { - client, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, - }; - clients.set(client, { - ...existing, - requests: existing.requests + 1, - satsCost: existing.satsCost + entry.satsCost, - promptTokens: existing.promptTokens + entry.promptTokens, - completionTokens: existing.completionTokens + entry.completionTokens, - totalTokens: existing.totalTokens + entry.totalTokens, - }); - } - return Array.from(clients.values()).sort((a, b) => b.satsCost - a.satsCost); -} - // ─── Client / Npub helpers ──────────────────────────────────────────── export interface ClientInfo { @@ -251,56 +146,3 @@ export async function fetchClients(): Promise { export function hasAnyNpubs(clients: ClientInfo[]): boolean { return clients.some((c) => !!c.ownerNpub); } - -export function getNpubStats(entries: UsageTrackingEntry[], clients: ClientInfo[]): NpubStats[] { - // Build client-id → ownerNpub lookup - const clientNpubMap = new Map(); - for (const c of clients) { - if (c.ownerNpub) { - clientNpubMap.set(c.clientId, c.ownerNpub); - } - } - - // Aggregate usage per npub - const npubs = new Map(); - for (const entry of entries) { - const npub = clientNpubMap.get(entry.client || ""); - if (!npub) continue; // skip entries whose client has no ownerNpub - - const existing = npubs.get(npub) || { - npub, - requests: 0, - satsCost: 0, - promptTokens: 0, - completionTokens: 0, - totalTokens: 0, - }; - npubs.set(npub, { - ...existing, - requests: existing.requests + 1, - satsCost: existing.satsCost + entry.satsCost, - promptTokens: existing.promptTokens + entry.promptTokens, - completionTokens: existing.completionTokens + entry.completionTokens, - totalTokens: existing.totalTokens + entry.totalTokens, - }); - } - - return Array.from(npubs.values()).sort((a, b) => b.satsCost - a.satsCost); -} - -// ─── Totals ─────────────────────────────────────────────────────────── - -export function getTotals(entries: UsageTrackingEntry[] | undefined) { - if (!entries || !Array.isArray(entries)) { - return { promptTokens: 0, completionTokens: 0, totalTokens: 0, satsCost: 0 }; - } - return entries.reduce( - (acc, entry) => ({ - promptTokens: acc.promptTokens + entry.promptTokens, - completionTokens: acc.completionTokens + entry.completionTokens, - totalTokens: acc.totalTokens + entry.totalTokens, - satsCost: acc.satsCost + entry.satsCost, - }), - { promptTokens: 0, completionTokens: 0, totalTokens: 0, satsCost: 0 } - ); -} diff --git a/src/tui/usage/render.ts b/src/tui/usage/render.ts index c5a6432..2fe7b41 100644 --- a/src/tui/usage/render.ts +++ b/src/tui/usage/render.ts @@ -275,8 +275,6 @@ export function renderToday(stats: UsageStats, width: number): string { const currentHour = new Date().getHours(); // Build today's date string from LOCAL date components so it matches the // server's tz-bucketed day keys (which use the client's tzOffsetMinutes). - // formatDate() returns a UTC date via toISOString() and would mismatch for - // non-UTC users. const _now = new Date(); const todayDateStr = `${_now.getFullYear()}-${String(_now.getMonth() + 1).padStart(2, "0")}-${String(_now.getDate()).padStart(2, "0")}`; diff --git a/src/tui/usage/types.ts b/src/tui/usage/types.ts index 3db517f..2c0345d 100644 --- a/src/tui/usage/types.ts +++ b/src/tui/usage/types.ts @@ -9,46 +9,9 @@ export interface UsageStats { totalSatsCost: number; recentSatsCost: number; limit: number; - /** Present when data was fetched via /usage/summary (server-aggregated path). */ summary: UsageSummary; } -export interface DayStats { - date: string; - requests: number; - satsCost: number; - promptTokens: number; - completionTokens: number; - totalTokens: number; -} - -export interface ModelStats { - modelId: string; - requests: number; - satsCost: number; - promptTokens: number; - completionTokens: number; - totalTokens: number; -} - -export interface ProviderStats { - baseUrl: string; - requests: number; - satsCost: number; - promptTokens: number; - completionTokens: number; - totalTokens: number; -} - -export interface ClientStats { - client: string; - requests: number; - satsCost: number; - promptTokens: number; - completionTokens: number; - totalTokens: number; -} - export type TabId = "overview" | "today" | "models" | "providers" | "tokens" | "clients" | "npubs" | "recent"; export interface Tab { @@ -57,15 +20,6 @@ export interface Tab { key: string; } -export interface NpubStats { - npub: string; - requests: number; - satsCost: number; - promptTokens: number; - completionTokens: number; - totalTokens: number; -} - export interface VimState { scrollPos: number; searchQuery: string;