From 9bc3feff6221176a9a62afe07a4d7aca457fa14e Mon Sep 17 00:00:00 2001 From: Evan Yang Date: Tue, 24 Feb 2026 22:29:43 +0800 Subject: [PATCH] Enhance payment and analytics tracking by adding input and output token metrics across various components --- routstr/auth.py | 4 + routstr/core/logging.py | 4 +- routstr/core/usage_analytics_store.py | 358 +++++++++- routstr/payment/cost_calculation.py | 70 +- ui/components/top-models-usage-chart.tsx | 815 +++++++++++++++++++++++ ui/components/usage-summary-cards.tsx | 41 +- 6 files changed, 1217 insertions(+), 75 deletions(-) create mode 100644 ui/components/top-models-usage-chart.tsx diff --git a/routstr/auth.py b/routstr/auth.py index 07d83dff..62b6c5b8 100644 --- a/routstr/auth.py +++ b/routstr/auth.py @@ -776,6 +776,8 @@ async def adjust_payment_for_tokens( "key_hash": key.hashed_key[:8] + "...", "billing_key_hash": billing_key.hashed_key[:8] + "...", "charged_amount": cost.total_msats, + "input_tokens": cost.input_tokens, + "output_tokens": cost.output_tokens, "new_balance": billing_key.balance, "model": model, }, @@ -799,6 +801,8 @@ async def adjust_payment_for_tokens( "cost_difference": cost_difference, "input_msats": cost.input_msats, "output_msats": cost.output_msats, + "input_tokens": cost.input_tokens, + "output_tokens": cost.output_tokens, }, ) diff --git a/routstr/core/logging.py b/routstr/core/logging.py index d88debb8..14b1b1ff 100644 --- a/routstr/core/logging.py +++ b/routstr/core/logging.py @@ -13,11 +13,11 @@ DO NOT modify or remove these messages without updating the usage tracking logic 2. "Calculated token-based cost" (INFO) - routstr/auth.py - Used to track successful completions and revenue - - The 'token_cost' and 'model' fields are extracted for dashboard metrics + - The 'token_cost', 'model', 'input_tokens', and 'output_tokens' fields are extracted for dashboard metrics 3. "Max cost payment finalized" (INFO) - routstr/auth.py - Used as the successful completion fallback when token usage is unavailable - - The 'charged_amount' and 'model' fields are extracted for dashboard metrics + - The 'charged_amount', 'model', 'input_tokens', and 'output_tokens' fields are extracted for dashboard metrics 4. "Payment processed successfully" (INFO) - routstr/auth.py - Used to count successful payment processing events diff --git a/routstr/core/usage_analytics_store.py b/routstr/core/usage_analytics_store.py index c54c0d6d..2df1f440 100644 --- a/routstr/core/usage_analytics_store.py +++ b/routstr/core/usage_analytics_store.py @@ -21,7 +21,7 @@ class UsageAnalyticsStore: bytes. """ - SCHEMA_VERSION = "3" + SCHEMA_VERSION = "4" def __init__(self, logs_dir: Path, db_path: Path | None = None): self.logs_dir = logs_dir @@ -60,12 +60,20 @@ class UsageAnalyticsStore: cutoff_timestamp=cutoff_timestamp, limit=model_limit, ) + model_usage_mix = self._query_model_usage_mix_locked( + conn, + cutoff_timestamp=cutoff_timestamp, + interval_minutes=interval_minutes, + hours_back=hours_back, + limit=model_limit, + ) return { "metrics": metrics, "summary": summary, "error_details": error_details, "revenue_by_model": revenue_by_model, + "model_usage_mix": model_usage_mix, } def get_summary(self, *, hours_back: int) -> dict[str, Any]: @@ -147,15 +155,6 @@ class UsageAnalyticsStore: "SELECT value FROM analytics_meta WHERE key = 'schema_version'" ).fetchone() current_version = current_version_row[0] if current_version_row else None - if current_version != self.SCHEMA_VERSION: - self._drop_index_tables_locked(conn) - conn.execute( - """ - INSERT OR REPLACE INTO analytics_meta (key, value) - VALUES ('schema_version', ?) - """, - (self.SCHEMA_VERSION,), - ) conn.execute( """ @@ -181,7 +180,10 @@ class UsageAnalyticsStore: payment_processed INTEGER NOT NULL DEFAULT 0, upstream_errors INTEGER NOT NULL DEFAULT 0, revenue_msats REAL NOT NULL DEFAULT 0, - refunds_msats REAL NOT NULL DEFAULT 0 + refunds_msats REAL NOT NULL DEFAULT 0, + input_tokens INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0 ) """ ) @@ -195,6 +197,9 @@ class UsageAnalyticsStore: failed INTEGER NOT NULL DEFAULT 0, revenue_msats REAL NOT NULL DEFAULT 0, refunds_msats REAL NOT NULL DEFAULT 0, + input_tokens INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (minute_ts, model) ) """ @@ -234,6 +239,9 @@ class UsageAnalyticsStore: conn.execute( "CREATE INDEX IF NOT EXISTS idx_analytics_model_minute_ts ON analytics_model_minute (minute_ts)" ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_analytics_model_minute_model_ts ON analytics_model_minute (model, minute_ts)" + ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_analytics_model_presence_ts ON analytics_model_presence_minute (minute_ts)" ) @@ -243,8 +251,74 @@ class UsageAnalyticsStore: conn.execute( "CREATE INDEX IF NOT EXISTS idx_analytics_error_events_ts ON analytics_error_events (timestamp DESC)" ) + self._migrate_schema_locked(conn) + if current_version != self.SCHEMA_VERSION: + conn.execute( + """ + INSERT OR REPLACE INTO analytics_meta (key, value) + VALUES ('schema_version', ?) + """, + (self.SCHEMA_VERSION,), + ) conn.commit() + def _migrate_schema_locked(self, conn: sqlite3.Connection) -> None: + self._ensure_column_locked( + conn, + "analytics_minute", + "input_tokens", + "INTEGER NOT NULL DEFAULT 0", + ) + self._ensure_column_locked( + conn, + "analytics_minute", + "output_tokens", + "INTEGER NOT NULL DEFAULT 0", + ) + self._ensure_column_locked( + conn, + "analytics_minute", + "total_tokens", + "INTEGER NOT NULL DEFAULT 0", + ) + self._ensure_column_locked( + conn, + "analytics_model_minute", + "input_tokens", + "INTEGER NOT NULL DEFAULT 0", + ) + self._ensure_column_locked( + conn, + "analytics_model_minute", + "output_tokens", + "INTEGER NOT NULL DEFAULT 0", + ) + self._ensure_column_locked( + conn, + "analytics_model_minute", + "total_tokens", + "INTEGER NOT NULL DEFAULT 0", + ) + + def _ensure_column_locked( + self, + conn: sqlite3.Connection, + table: str, + column: str, + column_definition: str, + ) -> None: + existing_columns = { + str(row["name"]) + for row in conn.execute(f"PRAGMA table_info({table})").fetchall() + } + if column in existing_columns: + return + + conn.execute( + f"ALTER TABLE {table} ADD COLUMN {column} {column_definition}" + ) + logger.info(f"Migrated analytics schema: added {table}.{column}") + def _drop_index_tables_locked(self, conn: sqlite3.Connection) -> None: conn.execute("DROP TABLE IF EXISTS analytics_file_state") conn.execute("DROP TABLE IF EXISTS analytics_minute") @@ -473,13 +547,21 @@ class UsageAnalyticsStore: elif level == "WARNING": bucket["warnings"] += 1 - completed, revenue_msats = self._extract_success_metrics(entry, message) + completed, revenue_msats, input_tokens, output_tokens = ( + self._extract_success_metrics(entry, message) + ) if completed: bucket["total_requests"] += 1 bucket["successful_chat_completions"] += 1 model_bucket = model_updates[(minute_key, model)] model_bucket["requests"] += 1 model_bucket["successful"] += 1 + bucket["input_tokens"] += input_tokens + bucket["output_tokens"] += output_tokens + bucket["total_tokens"] += input_tokens + output_tokens + model_bucket["input_tokens"] += input_tokens + model_bucket["output_tokens"] += output_tokens + model_bucket["total_tokens"] += input_tokens + output_tokens if revenue_msats > 0: bucket["revenue_msats"] += revenue_msats @@ -547,6 +629,9 @@ class UsageAnalyticsStore: int(stats["upstream_errors"]), float(stats["revenue_msats"]), float(stats["refunds_msats"]), + int(stats["input_tokens"]), + int(stats["output_tokens"]), + int(stats["total_tokens"]), ) for minute_ts, stats in minute_updates.items() ] @@ -563,9 +648,12 @@ class UsageAnalyticsStore: payment_processed, upstream_errors, revenue_msats, - refunds_msats + refunds_msats, + input_tokens, + output_tokens, + total_tokens ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(minute_ts) DO UPDATE SET total_entries = total_entries + excluded.total_entries, total_requests = total_requests + excluded.total_requests, @@ -576,7 +664,10 @@ class UsageAnalyticsStore: payment_processed = payment_processed + excluded.payment_processed, upstream_errors = upstream_errors + excluded.upstream_errors, revenue_msats = revenue_msats + excluded.revenue_msats, - refunds_msats = refunds_msats + excluded.refunds_msats + refunds_msats = refunds_msats + excluded.refunds_msats, + input_tokens = input_tokens + excluded.input_tokens, + output_tokens = output_tokens + excluded.output_tokens, + total_tokens = total_tokens + excluded.total_tokens """, rows, ) @@ -591,6 +682,9 @@ class UsageAnalyticsStore: int(stats["failed"]), float(stats["revenue_msats"]), float(stats["refunds_msats"]), + int(stats["input_tokens"]), + int(stats["output_tokens"]), + int(stats["total_tokens"]), ) for (minute_ts, model), stats in model_updates.items() ] @@ -603,15 +697,21 @@ class UsageAnalyticsStore: successful, failed, revenue_msats, - refunds_msats + refunds_msats, + input_tokens, + output_tokens, + total_tokens ) - VALUES (?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(minute_ts, model) DO UPDATE SET requests = requests + excluded.requests, successful = successful + excluded.successful, failed = failed + excluded.failed, revenue_msats = revenue_msats + excluded.revenue_msats, - refunds_msats = refunds_msats + excluded.refunds_msats + refunds_msats = refunds_msats + excluded.refunds_msats, + input_tokens = input_tokens + excluded.input_tokens, + output_tokens = output_tokens + excluded.output_tokens, + total_tokens = total_tokens + excluded.total_tokens """, rows, ) @@ -694,7 +794,10 @@ class UsageAnalyticsStore: COALESCE(SUM(payment_processed), 0) AS payment_processed, COALESCE(SUM(upstream_errors), 0) AS upstream_errors, COALESCE(SUM(revenue_msats), 0) AS revenue_msats, - COALESCE(SUM(refunds_msats), 0) AS refunds_msats + COALESCE(SUM(refunds_msats), 0) AS refunds_msats, + COALESCE(SUM(input_tokens), 0) AS input_tokens, + COALESCE(SUM(output_tokens), 0) AS output_tokens, + COALESCE(SUM(total_tokens), 0) AS total_tokens FROM analytics_minute WHERE minute_ts >= ? GROUP BY bucket_ts @@ -713,6 +816,9 @@ class UsageAnalyticsStore: "upstream_errors": 0.0, "revenue_msats": 0.0, "refunds_msats": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0, } points: list[dict[str, Any]] = [] @@ -726,6 +832,9 @@ class UsageAnalyticsStore: upstream_errors = int(row["upstream_errors"]) revenue_msats = float(row["revenue_msats"]) refunds_msats = float(row["refunds_msats"]) + input_tokens = int(row["input_tokens"]) + output_tokens = int(row["output_tokens"]) + total_tokens = int(row["total_tokens"]) totals["total_requests"] += total_requests totals["successful_chat_completions"] += successful @@ -736,6 +845,9 @@ class UsageAnalyticsStore: totals["upstream_errors"] += upstream_errors totals["revenue_msats"] += revenue_msats totals["refunds_msats"] += refunds_msats + totals["input_tokens"] += input_tokens + totals["output_tokens"] += output_tokens + totals["total_tokens"] += total_tokens points.append( { @@ -749,6 +861,9 @@ class UsageAnalyticsStore: "upstream_errors": upstream_errors, "revenue_msats": revenue_msats, "refunds_msats": refunds_msats, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": total_tokens, "requests": total_requests, } ) @@ -763,6 +878,9 @@ class UsageAnalyticsStore: "upstream_errors": int(totals["upstream_errors"]), "revenue_msats": float(totals["revenue_msats"]), "refunds_msats": float(totals["refunds_msats"]), + "input_tokens": int(totals["input_tokens"]), + "output_tokens": int(totals["output_tokens"]), + "total_tokens": int(totals["total_tokens"]), } return { @@ -788,7 +906,10 @@ class UsageAnalyticsStore: COALESCE(SUM(payment_processed), 0) AS payment_processed, COALESCE(SUM(upstream_errors), 0) AS upstream_errors, COALESCE(SUM(revenue_msats), 0) AS revenue_msats, - COALESCE(SUM(refunds_msats), 0) AS refunds_msats + COALESCE(SUM(refunds_msats), 0) AS refunds_msats, + COALESCE(SUM(input_tokens), 0) AS input_tokens, + COALESCE(SUM(output_tokens), 0) AS output_tokens, + COALESCE(SUM(total_tokens), 0) AS total_tokens FROM analytics_minute WHERE minute_ts >= ? """, @@ -824,6 +945,9 @@ class UsageAnalyticsStore: total_requests = int(totals["total_requests"]) successful = int(totals["successful_chat_completions"]) failed_requests = int(totals["failed_requests"]) + input_tokens = int(totals["input_tokens"]) + output_tokens = int(totals["output_tokens"]) + total_tokens = int(totals["total_tokens"]) revenue_msats = float(totals["revenue_msats"]) refunds_msats = float(totals["refunds_msats"]) @@ -845,6 +969,18 @@ class UsageAnalyticsStore: "unique_models_count": len(unique_models), "unique_models": unique_models, "error_types": error_types, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": total_tokens, + "avg_input_tokens_per_completion": (input_tokens / successful) + if successful > 0 + else 0, + "avg_output_tokens_per_completion": (output_tokens / successful) + if successful > 0 + else 0, + "avg_total_tokens_per_completion": (total_tokens / successful) + if successful > 0 + else 0, "success_rate": (successful / total_requests * 100) if total_requests > 0 else 0, @@ -970,6 +1106,151 @@ class UsageAnalyticsStore: "total_models": len(models), } + def _query_model_usage_mix_locked( + self, + conn: sqlite3.Connection, + *, + cutoff_timestamp: str, + interval_minutes: int, + hours_back: int, + limit: int, + ) -> dict[str, Any]: + top_limit = max(1, min(int(limit), 10)) + top_rows = conn.execute( + """ + SELECT + model, + COALESCE(SUM(successful), 0) AS total_successful + FROM analytics_model_minute + WHERE minute_ts >= ? + AND model != 'unknown' + GROUP BY model + ORDER BY total_successful DESC + LIMIT ? + """, + (cutoff_timestamp, top_limit), + ).fetchall() + + top_models = [ + str(row["model"]) + for row in top_rows + if int(row["total_successful"] or 0) > 0 + ] + + bucket_seconds = max(60, int(interval_minutes) * 60) + total_rows = conn.execute( + """ + SELECT + datetime( + (CAST(strftime('%s', minute_ts) AS INTEGER) / ?) * ?, + 'unixepoch' + ) AS bucket_ts, + COALESCE(SUM(successful), 0) AS total_successful, + COALESCE(SUM(revenue_msats), 0) AS total_revenue_msats, + COALESCE(SUM(total_tokens), 0) AS total_tokens + FROM analytics_model_minute + WHERE minute_ts >= ? + GROUP BY bucket_ts + ORDER BY bucket_ts + """, + (bucket_seconds, bucket_seconds, cutoff_timestamp), + ).fetchall() + + bucket_index: dict[str, dict[str, Any]] = {} + for row in total_rows: + total_successful = int(row["total_successful"]) + total_revenue_msats = float(row["total_revenue_msats"]) + total_tokens = int(row["total_tokens"]) + if ( + total_successful <= 0 + and total_revenue_msats <= 0 + and total_tokens <= 0 + ): + continue + + bucket_ts = str(row["bucket_ts"]) + bucket = bucket_index.setdefault( + bucket_ts, + { + "timestamp": bucket_ts, + "total_successful": 0, + "total_revenue_msats": 0.0, + "total_tokens": 0, + "others": 0, + "others_revenue_msats": 0.0, + "others_tokens": 0, + "model_counts": {}, + "model_revenue_msats": {}, + "model_tokens": {}, + }, + ) + bucket["total_successful"] = total_successful + bucket["total_revenue_msats"] = total_revenue_msats + bucket["total_tokens"] = total_tokens + bucket["others"] = total_successful + bucket["others_revenue_msats"] = total_revenue_msats + bucket["others_tokens"] = total_tokens + + if top_models and bucket_index: + placeholders = ",".join("?" for _ in top_models) + top_model_rows = conn.execute( + f""" + SELECT + datetime( + (CAST(strftime('%s', minute_ts) AS INTEGER) / ?) * ?, + 'unixepoch' + ) AS bucket_ts, + model, + COALESCE(SUM(successful), 0) AS successful, + COALESCE(SUM(revenue_msats), 0) AS revenue_msats, + COALESCE(SUM(total_tokens), 0) AS total_tokens + FROM analytics_model_minute + WHERE minute_ts >= ? + AND model IN ({placeholders}) + GROUP BY bucket_ts, model + ORDER BY bucket_ts + """, + (bucket_seconds, bucket_seconds, cutoff_timestamp, *top_models), + ).fetchall() + + for row in top_model_rows: + bucket_ts = str(row["bucket_ts"]) + bucket = bucket_index.get(bucket_ts) + if bucket is None: + continue + + model = str(row["model"]) + successful = int(row["successful"]) + revenue_msats = float(row["revenue_msats"]) + total_tokens = int(row["total_tokens"]) + + model_counts = bucket["model_counts"] + model_counts[model] = successful + model_revenue_msats = bucket["model_revenue_msats"] + model_revenue_msats[model] = revenue_msats + model_tokens = bucket["model_tokens"] + model_tokens[model] = total_tokens + + bucket["others"] = max(0, int(bucket["others"]) - successful) + bucket["others_revenue_msats"] = max( + 0.0, + float(bucket["others_revenue_msats"]) - revenue_msats, + ) + bucket["others_tokens"] = max( + 0, + int(bucket["others_tokens"]) - total_tokens, + ) + + metrics = sorted(bucket_index.values(), key=lambda item: str(item["timestamp"])) + + return { + "top_models": top_models, + "metrics": metrics, + "hours_back": hours_back, + "interval_minutes": interval_minutes, + "total_buckets": len(metrics), + } + def _cutoff_timestamp(self, hours_back: int) -> str: cutoff = datetime.now(timezone.utc) - timedelta(hours=hours_back) return cutoff.strftime("%Y-%m-%d %H:%M:%S") @@ -983,26 +1264,43 @@ class UsageAnalyticsStore: def _extract_success_metrics( self, entry: dict[str, Any], message: str - ) -> tuple[bool, float]: + ) -> tuple[bool, float, int, int]: # These auth logs are emitted once per successful settlement across providers # and avoid duplicate counting from provider-specific completion logs. logger_name = str(entry.get("name", "")) if not logger_name.startswith("routstr.auth"): - return False, 0.0 + return False, 0.0, 0, 0 + + input_tokens = self._parse_token_count(entry.get("input_tokens", 0)) + output_tokens = self._parse_token_count(entry.get("output_tokens", 0)) if "calculated token-based cost" in message: token_cost = entry.get("token_cost", 0) if isinstance(token_cost, (int, float)) and token_cost > 0: - return True, float(token_cost) - return True, 0.0 + return True, float(token_cost), input_tokens, output_tokens + return True, 0.0, input_tokens, output_tokens if "max cost payment finalized" in message: charged_amount = entry.get("charged_amount", 0) if isinstance(charged_amount, (int, float)) and charged_amount > 0: - return True, float(charged_amount) - return True, 0.0 + return True, float(charged_amount), input_tokens, output_tokens + return True, 0.0, input_tokens, output_tokens - return False, 0.0 + return False, 0.0, 0, 0 + + def _parse_token_count(self, value: Any) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int): + return max(0, value) + if isinstance(value, float): + return max(0, int(value)) + if isinstance(value, str): + try: + return max(0, int(float(value))) + except ValueError: + return 0 + return 0 def _new_minute_stats(self) -> dict[str, float]: return { @@ -1016,6 +1314,9 @@ class UsageAnalyticsStore: "upstream_errors": 0.0, "revenue_msats": 0.0, "refunds_msats": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0, } def _new_model_stats(self) -> dict[str, float]: @@ -1025,4 +1326,7 @@ class UsageAnalyticsStore: "failed": 0.0, "revenue_msats": 0.0, "refunds_msats": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0, } diff --git a/routstr/payment/cost_calculation.py b/routstr/payment/cost_calculation.py index 1df4caf8..2ed7e4b8 100644 --- a/routstr/payment/cost_calculation.py +++ b/routstr/payment/cost_calculation.py @@ -16,6 +16,8 @@ class CostData(BaseModel): output_msats: int total_msats: int total_usd: float = 0.0 + input_tokens: int = 0 + output_tokens: int = 0 class MaxCostData(CostData): @@ -63,10 +65,49 @@ async def calculate_cost( # todo: can be sync output_msats=0, total_msats=0, total_usd=0.0, + input_tokens=0, + output_tokens=0, ) usage_data = response_data["usage"] + def parse_token_count(value: object) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int): + return max(0, value) + if isinstance(value, float): + return max(0, int(value)) + if isinstance(value, str): + try: + return max(0, int(float(value))) + except ValueError: + return 0 + return 0 + + input_tokens = parse_token_count(usage_data.get("prompt_tokens", 0)) + output_tokens = parse_token_count(usage_data.get("completion_tokens", 0)) + input_tokens = ( + input_tokens + if input_tokens != 0 + else parse_token_count(usage_data.get("input_tokens", 0)) + ) + output_tokens = ( + output_tokens + if output_tokens != 0 + else parse_token_count(usage_data.get("output_tokens", 0)) + ) + input_tokens = ( + input_tokens + if input_tokens != 0 + else parse_token_count(response_data.get("usage", {}).get("input_tokens", 0)) + ) + output_tokens = ( + output_tokens + if output_tokens != 0 + else parse_token_count(response_data.get("usage", {}).get("output_tokens", 0)) + ) + usd_cost = 0.0 # Prioritize cost_details.upstream_inference_cost @@ -104,6 +145,8 @@ async def calculate_cost( # todo: can be sync output_msats=-1, total_msats=cost_in_msats, total_usd=usd_cost, + input_tokens=input_tokens, + output_tokens=output_tokens, ) except Exception as e: logger.warning( @@ -184,31 +227,10 @@ async def calculate_cost( # todo: can be sync input_msats=0, output_msats=0, total_msats=max_cost, + input_tokens=input_tokens, + output_tokens=output_tokens, ) - input_tokens = usage_data.get("prompt_tokens", 0) - output_tokens = usage_data.get("completion_tokens", 0) - - # added for response api - input_tokens = ( - input_tokens if input_tokens != 0 else usage_data.get("input_tokens", 0) - ) - output_tokens = ( - output_tokens if output_tokens != 0 else usage_data.get("output_tokens", 0) - ) - - # added for response api - input_tokens = ( - input_tokens - if input_tokens != 0 - else response_data.get("usage", {}).get("input_tokens", 0) - ) - output_tokens = ( - output_tokens - if output_tokens != 0 - else response_data.get("usage", {}).get("output_tokens", 0) - ) - input_msats = round(input_tokens / 1000 * MSATS_PER_1K_INPUT_TOKENS, 3) output_msats = round(output_tokens / 1000 * MSATS_PER_1K_OUTPUT_TOKENS, 3) @@ -234,4 +256,6 @@ async def calculate_cost( # todo: can be sync output_msats=int(output_msats), total_msats=token_based_cost, total_usd=total_usd, + input_tokens=input_tokens, + output_tokens=output_tokens, ) diff --git a/ui/components/top-models-usage-chart.tsx b/ui/components/top-models-usage-chart.tsx new file mode 100644 index 00000000..87474803 --- /dev/null +++ b/ui/components/top-models-usage-chart.tsx @@ -0,0 +1,815 @@ +'use client'; + +import { useEffect, useMemo, useRef, useState } from 'react'; +import { ExpandIcon, Minimize2Icon } from 'lucide-react'; +import { Bar, BarChart, CartesianGrid, XAxis, YAxis } from 'recharts'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Button } from '@/components/ui/button'; +import { + ChartConfig, + ChartContainer, + ChartTooltip, +} from '@/components/ui/chart'; +import { useIsMobile } from '@/hooks/use-mobile'; +import { type ModelUsageMix } from '@/lib/api/services/admin'; +import type { DisplayUnit } from '@/lib/types/units'; +import { cn } from '@/lib/utils'; + +interface TopModelsUsageChartProps { + mix: ModelUsageMix; + displayUnit: DisplayUnit; + usdPerSat: number | null; +} + +type ChartMode = 'requests' | 'revenue' | 'tokens'; + +interface TooltipRow { + color: string; + dataKey: string; + label: string; + value: number; +} + +type LeaderboardTrend = 'up' | 'down' | 'flat' | 'new'; + +interface LeaderboardRow { + chartDataKey: string | null; + displayName: string; + model: string; + provider: string; + rank: number; + totalRaw: number; + trend: LeaderboardTrend; + trendPercent: number | null; +} + +function parseBucketDate(value: string): Date | null { + const normalized = value.includes('T') + ? value + : `${value.replace(' ', 'T')}Z`; + const parsed = new Date(normalized); + if (!Number.isNaN(parsed.getTime())) { + return parsed; + } + const fallback = new Date(value); + return Number.isNaN(fallback.getTime()) ? null : fallback; +} + +function hueFromString(input: string): number { + let hash = 0; + for (let i = 0; i < input.length; i += 1) { + hash = (hash << 5) - hash + input.charCodeAt(i); + hash |= 0; + } + return Math.abs(hash) % 360; +} + +function getSeriesColor(model: string, index: number): string { + const palette = [ + 'var(--chart-1)', + 'var(--chart-2)', + 'var(--chart-3)', + 'var(--chart-4)', + 'var(--chart-5)', + '#f59e0b', + '#06b6d4', + '#8b5cf6', + '#f97316', + '#34d399', + ]; + + if (index < palette.length) { + return palette[index]; + } + + const hue = (hueFromString(model) + index * 23) % 360; + return `hsl(${hue} 70% 56%)`; +} + +function formatTooltipTimestamp(label: string): string { + const date = parseBucketDate(label); + if (!date) { + return label; + } + return date.toLocaleString([], { + month: 'long', + day: 'numeric', + year: 'numeric', + }); +} + +function formatAxisTimestamp( + timestamp: string, + hasMultipleDays: boolean +): string { + const date = parseBucketDate(timestamp); + if (!date) { + return ''; + } + + if (hasMultipleDays) { + return date.toLocaleDateString([], { + month: 'short', + day: 'numeric', + }); + } + + return date.toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + }); +} + +function convertRevenueMsats( + amountMsats: number, + displayUnit: DisplayUnit, + usdPerSat: number | null +): number { + if (displayUnit === 'msat') { + return amountMsats; + } + + const sats = amountMsats / 1000; + if (displayUnit === 'usd') { + return sats * (usdPerSat ?? 0); + } + + return sats; +} + +function prettifyProvider(provider: string): string { + const normalized = provider.trim().toLowerCase(); + const aliasMap: Record = { + 'x ai': 'x-ai', + xai: 'x-ai', + 'z ai': 'z-ai', + zai: 'z-ai', + open_ai: 'openai', + openai: 'openai', + }; + if (aliasMap[normalized]) { + return aliasMap[normalized]; + } + return normalized.replace(/[_-]+/g, ' '); +} + +function detectProviderFromModel(model: string): string { + const value = model.toLowerCase(); + if (value.includes('claude')) return 'anthropic'; + if (value.includes('gpt') || value.includes('openai')) return 'openai'; + if (value.includes('gemini')) return 'google'; + if (value.includes('grok') || value.includes('x-ai') || value.includes('xai')) { + return 'x-ai'; + } + if (value.includes('deepseek')) return 'deepseek'; + if (value.includes('minimax')) return 'minimax'; + if (value.includes('kimi') || value.includes('moonshot')) return 'moonshot'; + if (value.includes('mistral')) return 'mistral'; + if (value.includes('qwen') || value.includes('alibaba')) return 'alibaba'; + if (value.includes('glm') || value.includes('z-ai') || value.includes('z ai')) { + return 'z-ai'; + } + return 'unknown'; +} + +function getModelPresentation( + model: string +): { displayName: string; provider: string } { + const trimmed = model.trim(); + const slashIndex = trimmed.indexOf('/'); + if (slashIndex > 0 && slashIndex < trimmed.length - 1) { + const provider = prettifyProvider(trimmed.slice(0, slashIndex)); + const displayName = trimmed.slice(slashIndex + 1); + return { displayName, provider }; + } + + return { + displayName: trimmed, + provider: detectProviderFromModel(trimmed), + }; +} + +export function TopModelsUsageChart({ + mix, + displayUnit, + usdPerSat, +}: TopModelsUsageChartProps) { + const [mode, setMode] = useState('requests'); + const [hoveredSeriesKey, setHoveredSeriesKey] = useState(null); + const [isChartPointerInside, setIsChartPointerInside] = useState(false); + const [isFullscreen, setIsFullscreen] = useState(false); + const isMobile = useIsMobile(); + const containerRef = useRef(null); + const compactNumber = useMemo( + () => + new Intl.NumberFormat('en-US', { + notation: 'compact', + maximumFractionDigits: 2, + }), + [] + ); + const mixTopModels = useMemo( + () => (Array.isArray(mix.top_models) ? mix.top_models : []), + [mix.top_models] + ); + const mixMetrics = useMemo( + () => (Array.isArray(mix.metrics) ? mix.metrics : []), + [mix.metrics] + ); + + const chartModels = useMemo( + () => mixTopModels.slice(0, 10), + [mixTopModels] + ); + const leaderboardModels = useMemo( + () => mixTopModels.slice(0, 10), + [mixTopModels] + ); + const revenueDisplayUnit: DisplayUnit = useMemo(() => { + if (displayUnit === 'usd' && usdPerSat === null) { + return 'sat'; + } + return displayUnit; + }, [displayUnit, usdPerSat]); + const revenueUnitLabel = + revenueDisplayUnit === 'usd' + ? 'USD' + : revenueDisplayUnit === 'sat' + ? 'sats' + : revenueDisplayUnit === 'msat' + ? 'msats' + : revenueDisplayUnit; + + const series = useMemo( + () => + chartModels.map((model, index) => ({ + requestsKey: `model_req_${index}`, + revenueKey: `model_rev_${index}`, + tokensKey: `model_tok_${index}`, + label: model, + color: getSeriesColor(model, index), + })), + [chartModels] + ); + + const chartData = useMemo( + () => + mixMetrics.map((metric) => { + const modelCounts = metric.model_counts ?? {}; + const modelRevenue = metric.model_revenue_msats ?? {}; + const modelTokens = metric.model_tokens ?? {}; + const point: Record = { + timestamp: metric.timestamp, + total_successful: metric.total_successful, + total_revenue_msats: metric.total_revenue_msats, + total_tokens: metric.total_tokens, + others_requests: metric.others, + others_revenue_msats: metric.others_revenue_msats, + others_tokens: metric.others_tokens, + }; + + for (const item of series) { + point[item.requestsKey] = modelCounts[item.label] ?? 0; + point[item.revenueKey] = modelRevenue[item.label] ?? 0; + point[item.tokensKey] = modelTokens[item.label] ?? 0; + } + + return point; + }), + [mixMetrics, series] + ); + + const hasMultipleDays = useMemo(() => { + const daySet = new Set( + chartData.map((item) => + parseBucketDate(String(item.timestamp))?.toDateString() + ) + ); + return daySet.size > 1; + }, [chartData]); + + const chartConfig = useMemo(() => { + const config: ChartConfig = {}; + for (const item of series) { + config[item.requestsKey] = { + label: item.label, + color: item.color, + }; + config[item.revenueKey] = { + label: item.label, + color: item.color, + }; + config[item.tokensKey] = { + label: item.label, + color: item.color, + }; + } + config.others_requests = { + label: 'Others', + color: '#6b7280', + }; + config.others_revenue_msats = { + label: 'Others', + color: '#6b7280', + }; + config.others_tokens = { + label: 'Others', + color: '#6b7280', + }; + return config; + }, [series]); + + useEffect(() => { + setHoveredSeriesKey(null); + setIsChartPointerInside(false); + }, [mode]); + + useEffect(() => { + const handleFullscreenChange = () => { + setIsFullscreen(document.fullscreenElement === containerRef.current); + }; + + document.addEventListener('fullscreenchange', handleFullscreenChange); + + return () => { + document.removeEventListener('fullscreenchange', handleFullscreenChange); + }; + }, []); + + const toggleFullscreen = async () => { + if (!containerRef.current) { + return; + } + + try { + if (document.fullscreenElement === containerRef.current) { + await document.exitFullscreen(); + } else { + await containerRef.current.requestFullscreen(); + } + } catch (error) { + console.error('Failed to toggle top models chart fullscreen', error); + } + }; + + const formatValue = (rawValue: number): string => { + if (mode === 'requests') { + return compactNumber.format(rawValue); + } + + if (mode === 'tokens') { + return compactNumber.format(rawValue); + } + + const converted = convertRevenueMsats( + rawValue, + revenueDisplayUnit, + usdPerSat + ); + const compact = compactNumber.format(converted); + if (revenueDisplayUnit === 'usd') { + return `$${compact}`; + } + return `${compact} ${revenueUnitLabel}`; + }; + + const activeSeries = series.map((item) => ({ + dataKey: + mode === 'requests' + ? item.requestsKey + : mode === 'revenue' + ? item.revenueKey + : item.tokensKey, + name: item.label, + color: item.color, + })); + const othersKey = ( + mode === 'requests' + ? 'others_requests' + : mode === 'revenue' + ? 'others_revenue_msats' + : 'others_tokens' + ) as 'others_requests' | 'others_revenue_msats' | 'others_tokens'; + const activeSeriesKeys = [ + ...activeSeries.map((item) => item.dataKey), + othersKey, + ]; + const activeHoverSeriesKey = + hoveredSeriesKey && activeSeriesKeys.includes(hoveredSeriesKey) + ? hoveredSeriesKey + : null; + const getSeriesOpacity = (dataKey: string): number => + activeHoverSeriesKey && activeHoverSeriesKey !== dataKey ? 0.18 : 1; + const formatLeaderboardTotal = (rawValue: number): string => { + if (mode === 'requests') { + return `${compactNumber.format(rawValue)} requests`; + } + + if (mode === 'tokens') { + return `${compactNumber.format(rawValue)} tokens`; + } + + const converted = convertRevenueMsats( + rawValue, + revenueDisplayUnit, + usdPerSat + ); + const compact = compactNumber.format(converted); + if (revenueDisplayUnit === 'usd') { + return `$${compact}`; + } + return `${compact} ${revenueUnitLabel}`; + }; + const formatTrendPercent = (value: number): string => { + const abs = Math.abs(value); + const rounded = abs >= 10 ? abs.toFixed(0) : abs.toFixed(1); + return rounded.replace(/\.0$/, ''); + }; + const leaderboardRows = useMemo(() => { + if (leaderboardModels.length === 0 || mixMetrics.length === 0) { + return []; + } + + const windowSize = Math.floor(mixMetrics.length / 2); + const previousMetrics = + windowSize > 0 + ? mixMetrics.slice(-windowSize * 2, -windowSize) + : []; + const currentMetrics = + windowSize > 0 ? mixMetrics.slice(-windowSize) : mixMetrics; + + const rows = leaderboardModels + .map((model) => { + const readMetric = (metric: (typeof mixMetrics)[number]): number => + mode === 'requests' + ? (metric.model_counts ?? {})[model] ?? 0 + : mode === 'revenue' + ? (metric.model_revenue_msats ?? {})[model] ?? 0 + : (metric.model_tokens ?? {})[model] ?? 0; + + const totalRaw = mixMetrics.reduce( + (sum, metric) => sum + readMetric(metric), + 0 + ); + const previousRaw = previousMetrics.reduce( + (sum, metric) => sum + readMetric(metric), + 0 + ); + const currentRaw = currentMetrics.reduce( + (sum, metric) => sum + readMetric(metric), + 0 + ); + const trendPercent = + previousRaw > 0 + ? ((currentRaw - previousRaw) / previousRaw) * 100 + : null; + + let trend: LeaderboardTrend = 'flat'; + if (previousRaw <= 0 && currentRaw > 0) { + trend = 'new'; + } else if (trendPercent !== null && trendPercent > 0.5) { + trend = 'up'; + } else if (trendPercent !== null && trendPercent < -0.5) { + trend = 'down'; + } + + const presentation = getModelPresentation(model); + const matchingSeries = series.find((item) => item.label === model); + const chartDataKey = matchingSeries + ? mode === 'requests' + ? matchingSeries.requestsKey + : mode === 'revenue' + ? matchingSeries.revenueKey + : matchingSeries.tokensKey + : null; + + return { + chartDataKey, + displayName: presentation.displayName, + model, + provider: presentation.provider, + rank: 0, + totalRaw, + trend, + trendPercent, + } satisfies LeaderboardRow; + }) + .filter((row) => row.totalRaw > 0) + .sort((a, b) => b.totalRaw - a.totalRaw) + .slice(0, 10) + .map((row, index) => ({ + ...row, + rank: index + 1, + })); + + return rows; + }, [leaderboardModels, mixMetrics, mode, series]); + + if (chartData.length === 0) { + return null; + } + + return ( +
+ + +
+
+ Model Usage +

+ Stacked requests, revenue, or tokens by model ( + {mix.interval_minutes}m buckets). +

+
+
+
+ + + +
+ +
+
+ + + { + setHoveredSeriesKey(null); + setIsChartPointerInside(false); + }} + > + setIsChartPointerInside(true)} + onMouseMove={() => setIsChartPointerInside(true)} + onMouseLeave={() => { + setHoveredSeriesKey(null); + setIsChartPointerInside(false); + }} + margin={{ + top: 12, + right: isMobile ? 8 : 18, + left: isMobile ? 0 : 8, + bottom: 0, + }} + > + + + formatAxisTimestamp(String(value), hasMultipleDays) + } + /> + + formatValue( + typeof value === 'number' ? value : Number(value || 0) + ) + } + /> + { + if (!isChartPointerInside || !active || !payload?.length) { + return null; + } + + const rows = payload + .map((entry) => { + const value = + typeof entry.value === 'number' + ? entry.value + : Number(entry.value || 0); + + return { + color: String(entry.color || '#6b7280'), + dataKey: String(entry.dataKey || ''), + label: String(entry.name || ''), + value, + } satisfies TooltipRow; + }) + .filter((row) => Number.isFinite(row.value) && row.value > 0) + .sort((a, b) => b.value - a.value); + + const total = rows.reduce((sum, row) => sum + row.value, 0); + if (rows.length === 0) { + return null; + } + + return ( +
+

+ {formatTooltipTimestamp(String(label || ''))} +

+
+ {rows.map((row) => ( +
+ + + {row.label} + + + {formatValue(row.value)} + +
+ ))} +
+
+
+ Total + + {formatValue(total)} + +
+
+
+ ); + }} + /> + {activeSeries.map((item) => ( + setHoveredSeriesKey(item.dataKey)} + onMouseLeave={() => setHoveredSeriesKey(null)} + /> + ))} + setHoveredSeriesKey(othersKey)} + onMouseLeave={() => setHoveredSeriesKey(null)} + /> + + + +
+
+

+ Top models +

+

+ Change vs prior period +

+
+ + {leaderboardRows.length > 0 ? ( +
+ {leaderboardRows.map((row) => { + const rowIsLinked = Boolean(row.chartDataKey); + const rowIsActive = + row.chartDataKey !== null && + activeHoverSeriesKey === row.chartDataKey; + const rowIsDimmed = + Boolean(activeHoverSeriesKey) && + row.chartDataKey !== null && + row.chartDataKey !== activeHoverSeriesKey; + + let trendLabel = '0%'; + let trendClass = 'text-muted-foreground'; + if (row.trend === 'new') { + trendLabel = 'new'; + trendClass = 'text-blue-500'; + } else if (row.trend === 'up' && row.trendPercent !== null) { + trendLabel = `↑${formatTrendPercent(row.trendPercent)}%`; + trendClass = 'text-emerald-500'; + } else if ( + row.trend === 'down' && + row.trendPercent !== null + ) { + trendLabel = `↓${formatTrendPercent(row.trendPercent)}%`; + trendClass = 'text-red-500'; + } else if (row.trendPercent !== null) { + trendLabel = `${formatTrendPercent(row.trendPercent)}%`; + } + + return ( +
{ + if (row.chartDataKey) { + setHoveredSeriesKey(row.chartDataKey); + } + }} + onMouseLeave={() => { + if (row.chartDataKey) { + setHoveredSeriesKey(null); + } + }} + > + + {row.rank}. + +
+ + {row.displayName} + {' '} + + by {row.provider} + +
+ + {formatLeaderboardTotal(row.totalRaw)} + + + {trendLabel} + +
+ ); + })} +
+ ) : ( +

+ No model totals available for this range. +

+ )} +
+ + +
+ ); +} diff --git a/ui/components/usage-summary-cards.tsx b/ui/components/usage-summary-cards.tsx index e578748a..7db46735 100644 --- a/ui/components/usage-summary-cards.tsx +++ b/ui/components/usage-summary-cards.tsx @@ -35,9 +35,10 @@ export function UsageSummaryCards({ summary }: UsageSummaryCardsProps) { const formatAmount = (msat: number) => formatFromMsat(msat, displayUnit, usdPerSat); - const hasTokenStats = - typeof summary.total_tokens === 'number' || - typeof summary.avg_total_tokens_per_completion === 'number'; + const totalTokens = Number(summary.total_tokens ?? 0); + const avgTotalTokensPerCompletion = Number( + summary.avg_total_tokens_per_completion ?? 0 + ); const cards = [ { @@ -52,26 +53,20 @@ export function UsageSummaryCards({ summary }: UsageSummaryCardsProps) { icon: CheckCircle2, iconClassName: 'text-emerald-600 dark:text-emerald-300', }, - ...(hasTokenStats - ? [ - { - title: 'Total Tokens', - value: Number(summary.total_tokens ?? 0).toLocaleString(), - icon: Database, - iconClassName: 'text-cyan-600 dark:text-cyan-300', - }, - { - title: 'Avg Tokens/Completion', - value: Number( - summary.avg_total_tokens_per_completion ?? 0 - ).toLocaleString(undefined, { - maximumFractionDigits: 1, - }), - icon: Activity, - iconClassName: 'text-indigo-600 dark:text-indigo-300', - }, - ] - : []), + { + title: 'Total Tokens', + value: totalTokens.toLocaleString(), + icon: Database, + iconClassName: 'text-cyan-600 dark:text-cyan-300', + }, + { + title: 'Avg Tokens/Completion', + value: avgTotalTokensPerCompletion.toLocaleString(undefined, { + maximumFractionDigits: 1, + }), + icon: Activity, + iconClassName: 'text-indigo-600 dark:text-indigo-300', + }, { title: 'Revenue', value: formatAmount(summary.revenue_msats),