laantungir/routstrd
1
0
Fork 0
mirror of https://github.com/Routstr/routstrd.git synced 2026-07-30 15:46:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

removed old functions with benchmark which is clearly not needed

Browse Source
This commit is contained in:
redshift
2026-06-11 08:49:15 +08:00
parent 829681ffca
commit 764d2dd826
4 changed files with 1 additions and 584 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench-usage.ts
-377
View File
@@ -1,377 +0,0 @@
/**
* bench-usage.ts — In-process, no-network benchmark (Option B).
*
* Compares the NEW server-aggregated summary path against the LEGACY
* "fetch-everything-then-re-aggregate" path, in terms of TIME, highlighting
* the daemon-side cache that lives in `getUsageSummary`.
*
* It deliberately SKIPS the HTTP layer. In production:
* - `fetchUsageSummary()` HTTP-calls the daemon, which runs `getUsageSummary()`.
* - `fetchUsage(N)` HTTP-calls the daemon, which runs `driver.list({limit:N})`.
* The real localhost socket round-trip (~1-3 ms) is roughly CONSTANT for both
* paths, so excluding it isolates the actual computational difference — which is
* the whole point. See the takeaway printed at the end.
*
* Run from the repo root: bun scripts/bench-usage.ts [--json]
*
*/
// @ts-ignore — bun:sqlite is a Bun built-in module (matches storage/bun.ts).
import { Database } from "bun:sqlite";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rmSync } from "node:fs";
import { createBunSqliteUsageTrackingDriver } from "@routstr/sdk/storage/bun";
import {
getUsageSummary,
__resetUsageSummaryCacheForTest,
} from "../src/daemon/http/usage-summary.ts";
import {
getTotals,
getModelStats,
getProviderStats,
getClientStats,
getDayStats,
getHourlyToday,
getNpubStats,
} from "../src/tui/usage/data.ts";
import type { UsageTrackingEntry } from "../src/daemon/types.ts";
// ─── Config ──────────────────────────────────────────────────────────────────
const SIZES = [100, 1000, 2500, 5000, 10000, 20000];
const SAMPLES = 25; // median of 25, after a warmup
const STEADY_REFRESHES = 10; // the monitor's ~2s refresh loop
const TZ_OFFSET = 300; // synthetic UTC-5; fixed so results are reproducible
const DAY_SPREAD = 45; // spread timestamps over ~45 days
const jsonMode = process.argv.includes("--json");
// ─── Synthetic, sanitized templates (no real DB, no real identifiers) ─────────
interface Template {
modelId: string;
baseUrl: string;
client: string;
cost: number;
satsCost: number;
promptTokens: number;
completionTokens: number;
}
const PROVIDER_A = "https://api.example-provider-a.com/";
const PROVIDER_B = "https://api.example-provider-b.com/";
const TEMPLATES: Template[] = [
{ modelId: "minimax-m3", baseUrl: PROVIDER_A, client: "client-a", cost: 0.0021, satsCost: 4, promptTokens: 320, completionTokens: 180 },
{ modelId: "kimi-k2.6", baseUrl: PROVIDER_A, client: "client-b", cost: 0.0140, satsCost: 27, promptTokens: 5200, completionTokens: 2400 },
{ modelId: "deepseek-v4-flash", baseUrl: PROVIDER_B, client: "client-c", cost: 0.0008, satsCost: 1, promptTokens: 90, completionTokens: 60 },
{ modelId: "deepseek-v4-flash", baseUrl: PROVIDER_B, client: "client-a", cost: 0.0420, satsCost: 81, promptTokens: 24000, completionTokens: 13000 },
{ modelId: "kimi-k2.6", baseUrl: PROVIDER_B, client: "client-b", cost: 0.0065, satsCost: 12, promptTokens: 1800, completionTokens: 950 },
];
// Clients array (shape works for both getUsageSummary's ClientEntry and
// data.ts's ClientInfo — both only read clientId / ownerNpub). ~half get a
// synthetic ownerNpub so the npub aggregation path is exercised.
const CLIENT_IDS = [...new Set(TEMPLATES.map((t) => t.client))];
const clients = CLIENT_IDS.map((id, i) => ({
clientId: id,
name: id,
apiKey: `apikey-${id}`,
createdAt: 0,
ownerNpub: i % 2 === 0 ? `npub1bench${id.replace(/[^a-z]/g, "")}` : undefined,
}));
// ─── Row generation ────────────────────────────────────────────────────────────
function generateRows(n: number, now: number): UsageTrackingEntry[] {
const rows: UsageTrackingEntry[] = new Array(n);
const spreadMs = DAY_SPREAD * 86_400_000;
for (let i = 0; i < n; i++) {
const t = TEMPLATES[i % TEMPLATES.length]!;
// Spread timestamps backwards from `now` over DAY_SPREAD days, with a
// little jitter so multiple buckets/hours are populated.
const jitter = (i * 97) % 86_400_000;
const timestamp = now - Math.floor((i / n) * spreadMs) - jitter;
const promptTokens = t.promptTokens + (i % 17);
const completionTokens = t.completionTokens + (i % 13);
rows[i] = {
id: `bench-${i}`,
timestamp,
modelId: t.modelId,
baseUrl: t.baseUrl,
requestId: `req-${i}`,
cost: t.cost,
satsCost: t.satsCost,
promptTokens,
completionTokens,
totalTokens: promptTokens + completionTokens,
client: t.client,
sessionId: `sess-${i % 50}`,
tags: [],
};
}
return rows;
}
// ─── Seeding (raw transactional insert — fast) ────────────────────────────────
const TABLE = "usage_tracking";
function seedRawDb(dbPath: string, rows: UsageTrackingEntry[]): void {
const db = new Database(dbPath);
// Match the SDK schema exactly (copied from storage/bun.ts).
db.run(`
CREATE TABLE IF NOT EXISTS ${TABLE} (
id TEXT PRIMARY KEY,
timestamp INTEGER NOT NULL,
model_id TEXT NOT NULL,
base_url TEXT NOT NULL,
request_id TEXT NOT NULL,
cost REAL NOT NULL,
sats_cost REAL NOT NULL,
prompt_tokens INTEGER NOT NULL,
completion_tokens INTEGER NOT NULL,
total_tokens INTEGER NOT NULL,
client TEXT,
session_id TEXT,
tags TEXT
)
`);
db.run(`CREATE INDEX IF NOT EXISTS idx_${TABLE}_timestamp ON ${TABLE}(timestamp)`);
db.run(`CREATE INDEX IF NOT EXISTS idx_${TABLE}_model_id ON ${TABLE}(model_id)`);
db.run(`CREATE INDEX IF NOT EXISTS idx_${TABLE}_base_url ON ${TABLE}(base_url)`);
const insert = db.query(`
INSERT OR REPLACE INTO ${TABLE} (
id, timestamp, model_id, base_url, request_id,
cost, sats_cost, prompt_tokens, completion_tokens, total_tokens,
client, session_id, tags
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const seed = db.transaction((batch: UsageTrackingEntry[]) => {
for (const r of batch) {
insert.run(
r.id, r.timestamp, r.modelId, r.baseUrl, r.requestId,
r.cost, r.satsCost, r.promptTokens, r.completionTokens, r.totalTokens,
r.client ?? null, r.sessionId ?? null, JSON.stringify(r.tags ?? []),
);
}
});
seed(rows);
db.close();
}
// ─── Timing helpers ────────────────────────────────────────────────────────────
async function timeMedian(fn: () => Promise<unknown>, samples = SAMPLES): Promise<number> {
await fn(); // warmup (discarded)
const times: number[] = [];
for (let i = 0; i < samples; i++) {
const start = performance.now();
await fn();
times.push(performance.now() - start);
}
times.sort((a, b) => a - b);
return times[Math.floor(times.length / 2)]!;
}
function runLegacyAggregation(rows: UsageTrackingEntry[]): void {
// Mirror exactly what the monitor's render path does on raw entries.
getTotals(rows);
getModelStats(rows);
getProviderStats(rows);
getClientStats(rows);
getDayStats(rows);
getHourlyToday(rows);
getNpubStats(rows, clients);
}
// ─── Bench per size ──────────────────────────────────────────────────────────
interface SizeResult {
size: number;
summaryColdMs: number;
summaryWarmMs: number;
legacyMs: number;
summaryKB: number;
legacyKB: number;
reductionX: number;
steadySummaryMs: number;
steadyLegacyMs: number;
speedupX: number;
}
async function benchSize(size: number): Promise<SizeResult> {
const now = Date.now();
const dbPath = join(tmpdir(), `routstrd-bench-${size}-${process.pid}-${Date.now()}.db`);
const rows = generateRows(size, now);
seedRawDb(dbPath, rows);
const driver = await createBunSqliteUsageTrackingDriver({ dbPath });
try {
// ── summary cold: reset cache before every timed call ──────────────────
const summaryColdMs = await timeMedian(async () => {
__resetUsageSummaryCacheForTest();
await getUsageSummary(driver, clients, TZ_OFFSET);
});
// ── summary warm: cache already populated (warmup call is itself a hit) ─
__resetUsageSummaryCacheForTest();
await getUsageSummary(driver, clients, TZ_OFFSET); // populate once
const summaryWarmMs = await timeMedian(async () => {
await getUsageSummary(driver, clients, TZ_OFFSET);
});
// ── legacy: list(limit:N) + full client-side re-aggregation, no cache ──
const legacyMs = await timeMedian(async () => {
const listed = await driver.list({ limit: size });
runLegacyAggregation(listed);
});
// ── payload bytes ──────────────────────────────────────────────────────
__resetUsageSummaryCacheForTest();
const summaryResult = await getUsageSummary(driver, clients, TZ_OFFSET);
const listedRows = await driver.list({ limit: size });
const summaryBytes = JSON.stringify(summaryResult).length;
const legacyBytes = JSON.stringify(listedRows).length;
// ── steady state (the 2s refresh loop, STEADY_REFRESHES iterations) ────
// summary: 1 cold + (N-1) warm; legacy: N × full cost (no cache).
const steadySummaryMs =
summaryColdMs + (STEADY_REFRESHES - 1) * summaryWarmMs;
const steadyLegacyMs = STEADY_REFRESHES * legacyMs;
return {
size,
summaryColdMs,
summaryWarmMs,
legacyMs,
summaryKB: summaryBytes / 1024,
legacyKB: legacyBytes / 1024,
reductionX: legacyBytes / summaryBytes,
steadySummaryMs,
steadyLegacyMs,
speedupX: steadyLegacyMs / steadySummaryMs,
};
} finally {
// driver holds the only open handle now; closing the file by removing it.
try {
rmSync(dbPath, { force: true });
rmSync(`${dbPath}-shm`, { force: true });
rmSync(`${dbPath}-wal`, { force: true });
} catch { /* ignore */ }
}
}
// ─── Output ──────────────────────────────────────────────────────────────────
function fmt(n: number, dp = 2): string {
return n.toFixed(dp);
}
function pad(s: string, width: number): string {
return s.padStart(width);
}
function printTable(results: SizeResult[]): void {
console.log("");
console.log("Routstr usage path benchmark — TIME (in-process, no network)");
console.log("Daemon-side summary cache vs. legacy fetch-all-and-re-aggregate.\n");
// ── Payload table (per request) ──────────────────────────────
console.log("Payload per request:");
const payHeader = [
pad("size", 7),
pad("summary", 11),
pad("legacy", 11),
pad("reduction", 11),
].join(" ");
console.log(payHeader);
console.log("-".repeat(payHeader.length));
for (const r of results) {
console.log([
pad(String(r.size), 7),
pad(fmt(r.summaryKB, 1) + " KB", 11),
pad(fmt(r.legacyKB, 1) + " KB", 11),
pad(fmt(r.reductionX, 1) + "x", 11),
].join(" "));
}
// ── Time table (ms) ──────────────────────────────────────────
console.log("");
console.log("Time (ms):");
const timeHeader = [
pad("size", 7),
pad("sum cold", 10),
pad("sum warm", 10),
pad("legacy", 10),
pad("steady sum", 12),
pad("steady leg", 12),
pad("speedup", 9),
].join(" ");
console.log(timeHeader);
console.log("-".repeat(timeHeader.length));
for (const r of results) {
console.log([
pad(String(r.size), 7),
pad(fmt(r.summaryColdMs) + "ms", 10),
pad(fmt(r.summaryWarmMs) + "ms", 10),
pad(fmt(r.legacyMs) + "ms", 10),
pad(fmt(r.steadySummaryMs) + "ms", 12),
pad(fmt(r.steadyLegacyMs) + "ms", 12),
pad(fmt(r.speedupX, 1) + "x", 9),
].join(" "));
}
console.log("");
console.log("Legend (in-process; times = medians of " + SAMPLES + " samples, warmup discarded;");
console.log("excludes the ~constant localhost HTTP round-trip both paths would add):");
console.log(" size seeded usage rows in the DB");
console.log("");
console.log(" Payload table:");
console.log(" summary JSON bytes of the /usage/summary response (aggregates + 50 recent rows)");
console.log(" legacy JSON bytes of the /usage response (all rows)");
console.log(" reduction legacy / summary");
console.log("");
console.log(" Time table:");
console.log(" sum cold summary build, COLD cache (cache miss: ~15-query rebuild + assemble)");
console.log(" sum warm summary from WARM cache (count() + return the memoized result)");
console.log(" legacy list all rows + the client-side re-aggregation the monitor does (no cache)");
console.log(" steady sum total ms the SUMMARY path spends over " + STEADY_REFRESHES + " refreshes of the monitor's ~2s loop,");
console.log(" assuming data is unchanged across the window (cache stays valid):");
console.log(" 1 cold build + " + (STEADY_REFRESHES - 1) + " warm cache hits = cold + " + (STEADY_REFRESHES - 1) + "xwarm. Since warm is ~0.01ms,");
console.log(" this is in practice ~one cold build.");
console.log(" steady leg same " + STEADY_REFRESHES + " refreshes for LEGACY = legacy x" + STEADY_REFRESHES + " (no cache, full cost each time)");
console.log(" speedup steady leg / steady sum");
}
// ─── Main ────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
const results: SizeResult[] = [];
for (const size of SIZES) {
results.push(await benchSize(size));
}
if (jsonMode) {
console.log(JSON.stringify(
{
config: { sizes: SIZES, samples: SAMPLES, steadyRefreshes: STEADY_REFRESHES, tzOffset: TZ_OFFSET },
note: "In-process; HTTP round-trip (~1-3ms localhost, ~constant for both) excluded.",
results,
},
null,
2,
));
} else {
printTable(results);
}
}
main().catch((err) => {
console.error(err);
process.exit(1);
});
src/tui/usage/data.ts
+1 -159
View File
@@ -1,6 +1,6 @@
import type { UsageTrackingEntry } from "../../daemon/types.ts";
import { callDaemon, isDaemonRunning } from "../../utils/daemon-client.ts";
import type { ClientStats, DayStats, ModelStats, NpubStats, ProviderStats, UsageStats, UsageSummary } from "./types.ts";
import type { UsageStats, UsageSummary } from "./types.ts";
export { isDaemonRunning };
@@ -101,17 +101,6 @@ export async function fetchUsageSummary(): Promise<UsageStats | null> {
}
}
export function getTodayStart(): number {
const now = new Date();
return new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime();
}
export function formatDate(timestamp: number): string {
const d = new Date(timestamp);
const p = (n: number) => String(n).padStart(2, "0");
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`;
}
export function formatTime(timestamp: number): string {
const d = new Date(timestamp);
const p = (n: number) => String(n).padStart(2, "0");
@@ -124,100 +113,6 @@ export function formatNumber(n: number): string {
return n.toString();
}
export function getDayStats(entries: UsageTrackingEntry[]): Map<string, DayStats> {
const days = new Map<string, DayStats>();
for (const entry of entries) {
const date = formatDate(entry.timestamp);
const existing = days.get(date) || { date, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 };
days.set(date, {
...existing,
requests: existing.requests + 1,
satsCost: existing.satsCost + entry.satsCost,
promptTokens: existing.promptTokens + entry.promptTokens,
completionTokens: existing.completionTokens + entry.completionTokens,
totalTokens: existing.totalTokens + entry.totalTokens,
});
}
return days;
}
export function getHourlyToday(entries: UsageTrackingEntry[]): Map<number, DayStats> {
const todayStart = getTodayStart();
const hours = new Map<number, DayStats>();
for (const entry of entries) {
if (entry.timestamp < todayStart) continue;
const hour = new Date(entry.timestamp).getHours();
const existing = hours.get(hour) || {
date: formatDate(entry.timestamp), requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0,
};
hours.set(hour, {
...existing,
requests: existing.requests + 1,
satsCost: existing.satsCost + entry.satsCost,
promptTokens: existing.promptTokens + entry.promptTokens,
completionTokens: existing.completionTokens + entry.completionTokens,
totalTokens: existing.totalTokens + entry.totalTokens,
});
}
return hours;
}
export function getModelStats(entries: UsageTrackingEntry[]): ModelStats[] {
const models = new Map<string, ModelStats>();
for (const entry of entries) {
const existing = models.get(entry.modelId) || {
modelId: entry.modelId, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0,
};
models.set(entry.modelId, {
...existing,
requests: existing.requests + 1,
satsCost: existing.satsCost + entry.satsCost,
promptTokens: existing.promptTokens + entry.promptTokens,
completionTokens: existing.completionTokens + entry.completionTokens,
totalTokens: existing.totalTokens + entry.totalTokens,
});
}
return Array.from(models.values()).sort((a, b) => b.satsCost - a.satsCost);
}
export function getProviderStats(entries: UsageTrackingEntry[]): ProviderStats[] {
const providers = new Map<string, ProviderStats>();
for (const entry of entries) {
const url = entry.baseUrl || "unknown";
const existing = providers.get(url) || {
baseUrl: url, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0,
};
providers.set(url, {
...existing,
requests: existing.requests + 1,
satsCost: existing.satsCost + entry.satsCost,
promptTokens: existing.promptTokens + entry.promptTokens,
completionTokens: existing.completionTokens + entry.completionTokens,
totalTokens: existing.totalTokens + entry.totalTokens,
});
}
return Array.from(providers.values()).sort((a, b) => b.satsCost - a.satsCost);
}
export function getClientStats(entries: UsageTrackingEntry[]): ClientStats[] {
const clients = new Map<string, ClientStats>();
for (const entry of entries) {
const client = entry.client || "unknown";
const existing = clients.get(client) || {
client, requests: 0, satsCost: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0,
};
clients.set(client, {
...existing,
requests: existing.requests + 1,
satsCost: existing.satsCost + entry.satsCost,
promptTokens: existing.promptTokens + entry.promptTokens,
completionTokens: existing.completionTokens + entry.completionTokens,
totalTokens: existing.totalTokens + entry.totalTokens,
});
}
return Array.from(clients.values()).sort((a, b) => b.satsCost - a.satsCost);
}
// ─── Client / Npub helpers ────────────────────────────────────────────
export interface ClientInfo {
@@ -251,56 +146,3 @@ export async function fetchClients(): Promise<ClientInfo[]> {
export function hasAnyNpubs(clients: ClientInfo[]): boolean {
return clients.some((c) => !!c.ownerNpub);
}
export function getNpubStats(entries: UsageTrackingEntry[], clients: ClientInfo[]): NpubStats[] {
// Build client-id → ownerNpub lookup
const clientNpubMap = new Map<string, string>();
for (const c of clients) {
if (c.ownerNpub) {
clientNpubMap.set(c.clientId, c.ownerNpub);
}
}
// Aggregate usage per npub
const npubs = new Map<string, NpubStats>();
for (const entry of entries) {
const npub = clientNpubMap.get(entry.client || "");
if (!npub) continue; // skip entries whose client has no ownerNpub
const existing = npubs.get(npub) || {
npub,
requests: 0,
satsCost: 0,
promptTokens: 0,
completionTokens: 0,
totalTokens: 0,
};
npubs.set(npub, {
...existing,
requests: existing.requests + 1,
satsCost: existing.satsCost + entry.satsCost,
promptTokens: existing.promptTokens + entry.promptTokens,
completionTokens: existing.completionTokens + entry.completionTokens,
totalTokens: existing.totalTokens + entry.totalTokens,
});
}
return Array.from(npubs.values()).sort((a, b) => b.satsCost - a.satsCost);
}
// ─── Totals ───────────────────────────────────────────────────────────
export function getTotals(entries: UsageTrackingEntry[] | undefined) {
if (!entries || !Array.isArray(entries)) {
return { promptTokens: 0, completionTokens: 0, totalTokens: 0, satsCost: 0 };
}
return entries.reduce(
(acc, entry) => ({
promptTokens: acc.promptTokens + entry.promptTokens,
completionTokens: acc.completionTokens + entry.completionTokens,
totalTokens: acc.totalTokens + entry.totalTokens,
satsCost: acc.satsCost + entry.satsCost,
}),
{ promptTokens: 0, completionTokens: 0, totalTokens: 0, satsCost: 0 }
);
}
src/tui/usage/render.ts
-2
View File
@@ -275,8 +275,6 @@ export function renderToday(stats: UsageStats, width: number): string {
const currentHour = new Date().getHours();
// Build today's date string from LOCAL date components so it matches the
// server's tz-bucketed day keys (which use the client's tzOffsetMinutes).
// formatDate() returns a UTC date via toISOString() and would mismatch for
// non-UTC users.
const _now = new Date();
const todayDateStr = `${_now.getFullYear()}-${String(_now.getMonth() + 1).padStart(2, "0")}-${String(_now.getDate()).padStart(2, "0")}`;
src/tui/usage/types.ts
-46
View File
@@ -9,46 +9,9 @@ export interface UsageStats {
totalSatsCost: number;
recentSatsCost: number;
limit: number;
/** Present when data was fetched via /usage/summary (server-aggregated path). */
summary: UsageSummary;
}
export interface DayStats {
date: string;
requests: number;
satsCost: number;
promptTokens: number;
completionTokens: number;
totalTokens: number;
}
export interface ModelStats {
modelId: string;
requests: number;
satsCost: number;
promptTokens: number;
completionTokens: number;
totalTokens: number;
}
export interface ProviderStats {
baseUrl: string;
requests: number;
satsCost: number;
promptTokens: number;
completionTokens: number;
totalTokens: number;
}
export interface ClientStats {
client: string;
requests: number;
satsCost: number;
promptTokens: number;
completionTokens: number;
totalTokens: number;
}
export type TabId = "overview" | "today" | "models" | "providers" | "tokens" | "clients" | "npubs" | "recent";
export interface Tab {
@@ -57,15 +20,6 @@ export interface Tab {
key: string;
}
export interface NpubStats {
npub: string;
requests: number;
satsCost: number;
promptTokens: number;
completionTokens: number;
totalTokens: number;
}
export interface VimState {
scrollPos: number;
searchQuery: string;