From a290cc33affc2c8139934c7f745af6b3790ecae5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 15 Nov 2025 23:37:00 +0000 Subject: [PATCH] feat: Add usage tracking and analytics dashboard Co-authored-by: db2002dominic --- routstr/core/admin.py | 323 +++++++++++++++++++++++++- routstr/core/main.py | 9 + ui/app/usage/page.tsx | 253 ++++++++++++++++++++ ui/components/app-sidebar.tsx | 6 + ui/components/error-details-table.tsx | 78 +++++++ ui/components/usage-metrics-chart.tsx | 78 +++++++ ui/components/usage-summary-cards.tsx | 86 +++++++ ui/lib/api/services/admin.ts | 71 ++++++ 8 files changed, 902 insertions(+), 2 deletions(-) create mode 100644 ui/app/usage/page.tsx create mode 100644 ui/components/error-details-table.tsx create mode 100644 ui/components/usage-metrics-chart.tsx create mode 100644 ui/components/usage-summary-cards.tsx diff --git a/routstr/core/admin.py b/routstr/core/admin.py index 91899aaf..51db1a4e 100644 --- a/routstr/core/admin.py +++ b/routstr/core/admin.py @@ -1,9 +1,10 @@ import json import secrets -from datetime import datetime, timezone +from collections import defaultdict +from datetime import datetime, timedelta, timezone from pathlib import Path -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import HTMLResponse, RedirectResponse from pydantic import BaseModel from sqlmodel import select @@ -2958,3 +2959,321 @@ h1 { color: #333; } .no-logs { text-align: center; color: #666; padding: 40px; } .request-id-display { background-color: #e9ecef; padding: 10px; border-radius: 4px; margin-bottom: 20px; font-family: monospace; } """ + + +def _parse_log_file(file_path: Path) -> list[dict]: + """Parse JSON log file and return list of log entries.""" + entries: list[dict] = [] + try: + with open(file_path) as f: + for line in f: + try: + entry = json.loads(line.strip()) + entries.append(entry) + except json.JSONDecodeError: + continue + except Exception as e: + logger.error(f"Error reading log file {file_path}: {e}") + return entries + + +def _aggregate_metrics_by_time( + entries: list[dict], interval_minutes: int, hours_back: int = 24 +) -> dict[str, list[dict]]: + """Aggregate log metrics into time buckets.""" + now = datetime.now(timezone.utc) + cutoff = now - timedelta(hours=hours_back) + + time_buckets: dict[str, dict[str, int]] = defaultdict( + lambda: { + "total_requests": 0, + "successful_chat_completions": 0, + "failed_requests": 0, + "errors": 0, + "warnings": 0, + "payment_processed": 0, + "upstream_errors": 0, + } + ) + + for entry in entries: + try: + timestamp_str = entry.get("asctime", "") + if not timestamp_str: + continue + + log_time = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") + log_time = log_time.replace(tzinfo=timezone.utc) + + if log_time < cutoff: + continue + + bucket_time = log_time.replace( + minute=(log_time.minute // interval_minutes) * interval_minutes, + second=0, + microsecond=0, + ) + bucket_key = bucket_time.isoformat() + + message = entry.get("message", "").lower() + level = entry.get("levelname", "").upper() + + if level == "ERROR": + time_buckets[bucket_key]["errors"] += 1 + elif level == "WARNING": + time_buckets[bucket_key]["warnings"] += 1 + + if "received proxy request" in message: + time_buckets[bucket_key]["total_requests"] += 1 + + if "token adjustment completed for non-streaming" in message: + time_buckets[bucket_key]["successful_chat_completions"] += 1 + elif "token adjustment completed for streaming" in message: + time_buckets[bucket_key]["successful_chat_completions"] += 1 + + if "upstream request failed" in message or "revert payment" in message: + time_buckets[bucket_key]["failed_requests"] += 1 + + if "payment processed successfully" in message: + time_buckets[bucket_key]["payment_processed"] += 1 + + if "upstream" in message and level == "ERROR": + time_buckets[bucket_key]["upstream_errors"] += 1 + + except Exception: + continue + + result = [] + for bucket_key in sorted(time_buckets.keys()): + result.append({"timestamp": bucket_key, **time_buckets[bucket_key]}) + + return { + "metrics": result, + "interval_minutes": interval_minutes, + "hours_back": hours_back, + "total_buckets": len(result), + } + + +def _get_summary_stats(entries: list[dict], hours_back: int = 24) -> dict: + """Calculate summary statistics from log entries.""" + now = datetime.now(timezone.utc) + cutoff = now - timedelta(hours=hours_back) + + stats = { + "total_entries": 0, + "total_requests": 0, + "successful_chat_completions": 0, + "failed_requests": 0, + "total_errors": 0, + "total_warnings": 0, + "payment_processed": 0, + "upstream_errors": 0, + "unique_models": set(), + "error_types": defaultdict(int), + } + + for entry in entries: + try: + timestamp_str = entry.get("asctime", "") + if not timestamp_str: + continue + + log_time = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") + log_time = log_time.replace(tzinfo=timezone.utc) + + if log_time < cutoff: + continue + + stats["total_entries"] += 1 + + message = entry.get("message", "").lower() + level = entry.get("levelname", "").upper() + + if level == "ERROR": + stats["total_errors"] += 1 + if "error_type" in entry: + error_type = str(entry["error_type"]) + stats["error_types"][error_type] += 1 + elif level == "WARNING": + stats["total_warnings"] += 1 + + if "received proxy request" in message: + stats["total_requests"] += 1 + + if "token adjustment completed" in message: + stats["successful_chat_completions"] += 1 + + if "upstream request failed" in message or "revert payment" in message: + stats["failed_requests"] += 1 + + if "payment processed successfully" in message: + stats["payment_processed"] += 1 + + if "upstream" in message and level == "ERROR": + stats["upstream_errors"] += 1 + + if "model" in entry: + model = entry["model"] + if isinstance(model, str) and model != "unknown": + stats["unique_models"].add(model) + + except Exception: + continue + + return { + "total_entries": stats["total_entries"], + "total_requests": stats["total_requests"], + "successful_chat_completions": stats["successful_chat_completions"], + "failed_requests": stats["failed_requests"], + "total_errors": stats["total_errors"], + "total_warnings": stats["total_warnings"], + "payment_processed": stats["payment_processed"], + "upstream_errors": stats["upstream_errors"], + "unique_models_count": len(stats["unique_models"]), + "unique_models": sorted(list(stats["unique_models"])), + "error_types": dict(stats["error_types"]), + "success_rate": ( + (stats["successful_chat_completions"] / stats["total_requests"] * 100) + if stats["total_requests"] > 0 + else 0 + ), + } + + +@admin_router.get("/api/usage/metrics", dependencies=[Depends(require_admin_api)]) +async def get_usage_metrics( + request: Request, + interval: int = Query(default=15, ge=1, le=1440, description="Time interval in minutes"), + hours: int = Query(default=24, ge=1, le=168, description="Hours of history to analyze"), +) -> dict: + """Get usage metrics aggregated by time interval.""" + logs_dir = Path("logs") + all_entries: list[dict] = [] + + if not logs_dir.exists(): + return { + "metrics": [], + "interval_minutes": interval, + "hours_back": hours, + "total_buckets": 0, + } + + cutoff_date = datetime.now(timezone.utc) - timedelta(hours=hours) + + for log_file in sorted(logs_dir.glob("app_*.log")): + try: + file_date_str = log_file.stem.split("_")[1] + file_date = datetime.strptime(file_date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc) + + if file_date < cutoff_date.replace(hour=0, minute=0, second=0, microsecond=0): + continue + + entries = _parse_log_file(log_file) + all_entries.extend(entries) + except Exception as e: + logger.error(f"Error processing log file {log_file}: {e}") + continue + + return _aggregate_metrics_by_time(all_entries, interval, hours) + + +@admin_router.get("/api/usage/summary", dependencies=[Depends(require_admin_api)]) +async def get_usage_summary( + request: Request, + hours: int = Query(default=24, ge=1, le=168, description="Hours of history to analyze"), +) -> dict: + """Get summary statistics for the specified time period.""" + logs_dir = Path("logs") + all_entries: list[dict] = [] + + if not logs_dir.exists(): + return { + "total_entries": 0, + "total_requests": 0, + "successful_chat_completions": 0, + "failed_requests": 0, + "total_errors": 0, + "total_warnings": 0, + "payment_processed": 0, + "upstream_errors": 0, + "unique_models_count": 0, + "unique_models": [], + "error_types": {}, + "success_rate": 0, + } + + cutoff_date = datetime.now(timezone.utc) - timedelta(hours=hours) + + for log_file in sorted(logs_dir.glob("app_*.log")): + try: + file_date_str = log_file.stem.split("_")[1] + file_date = datetime.strptime(file_date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc) + + if file_date < cutoff_date.replace(hour=0, minute=0, second=0, microsecond=0): + continue + + entries = _parse_log_file(log_file) + all_entries.extend(entries) + except Exception as e: + logger.error(f"Error processing log file {log_file}: {e}") + continue + + return _get_summary_stats(all_entries, hours) + + +@admin_router.get("/api/usage/error-details", dependencies=[Depends(require_admin_api)]) +async def get_error_details( + request: Request, + hours: int = Query(default=24, ge=1, le=168, description="Hours of history to analyze"), + limit: int = Query(default=100, ge=1, le=1000, description="Maximum number of errors to return"), +) -> dict: + """Get detailed error information.""" + logs_dir = Path("logs") + errors: list[dict] = [] + + if not logs_dir.exists(): + return {"errors": [], "total_count": 0} + + cutoff_date = datetime.now(timezone.utc) - timedelta(hours=hours) + + for log_file in sorted(logs_dir.glob("app_*.log"), reverse=True): + try: + file_date_str = log_file.stem.split("_")[1] + file_date = datetime.strptime(file_date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc) + + if file_date < cutoff_date.replace(hour=0, minute=0, second=0, microsecond=0): + continue + + entries = _parse_log_file(log_file) + + for entry in entries: + if entry.get("levelname", "").upper() == "ERROR": + timestamp_str = entry.get("asctime", "") + if timestamp_str: + log_time = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") + log_time = log_time.replace(tzinfo=timezone.utc) + + if log_time >= cutoff_date: + errors.append({ + "timestamp": timestamp_str, + "message": entry.get("message", ""), + "error_type": entry.get("error_type", "unknown"), + "pathname": entry.get("pathname", ""), + "lineno": entry.get("lineno", 0), + "request_id": entry.get("request_id", ""), + }) + + if len(errors) >= limit: + break + + except Exception as e: + logger.error(f"Error processing log file {log_file}: {e}") + continue + + if len(errors) >= limit: + break + + errors.sort(key=lambda x: x["timestamp"], reverse=True) + + return {"errors": errors[:limit], "total_count": len(errors)} diff --git a/routstr/core/main.py b/routstr/core/main.py index b88be3cc..49d6e645 100644 --- a/routstr/core/main.py +++ b/routstr/core/main.py @@ -282,6 +282,15 @@ if UI_DIST_PATH.exists() and UI_DIST_PATH.is_dir(): async def redirect_unauthorized_index_txt() -> RedirectResponse: return RedirectResponse("/unauthorized") + @app.get("/usage", include_in_schema=False) + async def serve_usage_ui() -> FileResponse: + return FileResponse(UI_DIST_PATH / "usage" / "index.html") + + # Add explicit route for /usage/index.txt to redirect to /usage + @app.get("/usage/index.txt", include_in_schema=False) + async def redirect_usage_index_txt() -> RedirectResponse: + return RedirectResponse("/usage") + @app.get("/favicon.ico", include_in_schema=False) async def serve_favicon() -> FileResponse: icon_path = UI_DIST_PATH / "icon.ico" diff --git a/ui/app/usage/page.tsx b/ui/app/usage/page.tsx new file mode 100644 index 00000000..3e0341d3 --- /dev/null +++ b/ui/app/usage/page.tsx @@ -0,0 +1,253 @@ +'use client'; + +import { useState } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { AppSidebar } from '@/components/app-sidebar'; +import { SiteHeader } from '@/components/site-header'; +import { SidebarInset, SidebarProvider } from '@/components/ui/sidebar'; +import { UsageMetricsChart } from '@/components/usage-metrics-chart'; +import { UsageSummaryCards } from '@/components/usage-summary-cards'; +import { ErrorDetailsTable } from '@/components/error-details-table'; +import { AdminService } from '@/lib/api/services/admin'; +import { Button } from '@/components/ui/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { RefreshCw } from 'lucide-react'; + +export default function UsagePage() { + const [timeRange, setTimeRange] = useState('24'); + const [interval, setInterval] = useState('15'); + + const { + data: metricsData, + isLoading: metricsLoading, + refetch: refetchMetrics, + } = useQuery({ + queryKey: ['usage-metrics', interval, timeRange], + queryFn: () => + AdminService.getUsageMetrics(parseInt(interval), parseInt(timeRange)), + refetchInterval: 60_000, + staleTime: 30_000, + }); + + const { + data: summaryData, + isLoading: summaryLoading, + refetch: refetchSummary, + } = useQuery({ + queryKey: ['usage-summary', timeRange], + queryFn: () => AdminService.getUsageSummary(parseInt(timeRange)), + refetchInterval: 60_000, + staleTime: 30_000, + }); + + const { + data: errorData, + isLoading: errorLoading, + refetch: refetchErrors, + } = useQuery({ + queryKey: ['usage-errors', timeRange], + queryFn: () => AdminService.getErrorDetails(parseInt(timeRange), 100), + refetchInterval: 60_000, + staleTime: 30_000, + }); + + const handleRefresh = () => { + refetchMetrics(); + refetchSummary(); + refetchErrors(); + }; + + return ( + + + + +
+
+
+

+ Usage Tracking +

+

+ Monitor system usage, requests, and errors over time +

+
+
+ + + +
+
+ +
+ {summaryLoading ? ( +
Loading summary...
+ ) : summaryData ? ( + + ) : null} + +
+ {metricsLoading ? ( +
+ Loading metrics... +
+ ) : metricsData && metricsData.metrics.length > 0 ? ( + <> + + + + + ) : ( + + + No Data Available + + +

+ No metrics data found for the selected time range. This + could be because no requests have been logged yet or the + log files are not available. +

+ + + )} +
+ + {errorLoading ? ( +
Loading errors...
+ ) : errorData ? ( + + ) : null} + + {summaryData && summaryData.unique_models.length > 0 && ( + + + Active Models + + +
+ {summaryData.unique_models.map((model) => ( + + {model} + + ))} +
+ + + )} + + {summaryData && + summaryData.error_types && + Object.keys(summaryData.error_types).length > 0 && ( + + + Error Types Distribution + + +
+ {Object.entries(summaryData.error_types) + .sort(([, a], [, b]) => b - a) + .map(([type, count]) => ( +
+ {type} + + {count} + +
+ ))} +
+ + + )} +
+
+ + + ); +} diff --git a/ui/components/app-sidebar.tsx b/ui/components/app-sidebar.tsx index 70771d1f..5dc9879d 100644 --- a/ui/components/app-sidebar.tsx +++ b/ui/components/app-sidebar.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { + ActivityIcon, DatabaseIcon, FileTextIcon, LayoutDashboardIcon, @@ -35,6 +36,11 @@ const data = { url: '/', icon: LayoutDashboardIcon, }, + { + title: 'Usage', + url: '/usage', + icon: ActivityIcon, + }, { title: 'Models', url: '/model', diff --git a/ui/components/error-details-table.tsx b/ui/components/error-details-table.tsx new file mode 100644 index 00000000..5e51e810 --- /dev/null +++ b/ui/components/error-details-table.tsx @@ -0,0 +1,78 @@ +'use client'; + +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { Badge } from '@/components/ui/badge'; +import { ErrorDetail } from '@/lib/api/services/admin'; + +interface ErrorDetailsTableProps { + errors: ErrorDetail[]; +} + +export function ErrorDetailsTable({ errors }: ErrorDetailsTableProps) { + if (errors.length === 0) { + return ( + + + Recent Errors + + +

+ No errors found in the selected time period +

+ + + ); + } + + return ( + + + Recent Errors ({errors.length}) + + +
+ + + + Timestamp + Type + Message + Location + Request ID + + + + {errors.map((error, index) => ( + + + {new Date(error.timestamp).toLocaleString()} + + + {error.error_type} + + + {error.message} + + + {error.pathname}:{error.lineno} + + + {error.request_id || '-'} + + + ))} + +
+
+ + + ); +} diff --git a/ui/components/usage-metrics-chart.tsx b/ui/components/usage-metrics-chart.tsx new file mode 100644 index 00000000..29059af6 --- /dev/null +++ b/ui/components/usage-metrics-chart.tsx @@ -0,0 +1,78 @@ +'use client'; + +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { + Line, + LineChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, + Legend, + CartesianGrid, +} from 'recharts'; +import { UsageMetricData } from '@/lib/api/services/admin'; + +interface UsageMetricsChartProps { + data: UsageMetricData[]; + title: string; + dataKeys: Array<{ + key: keyof UsageMetricData; + name: string; + color: string; + }>; +} + +export function UsageMetricsChart({ + data, + title, + dataKeys, +}: UsageMetricsChartProps) { + const formattedData = data.map((item) => ({ + ...item, + time: new Date(item.timestamp).toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + }), + })); + + return ( + + + {title} + + + + + + + + + + {dataKeys.map((dataKey) => ( + + ))} + + + + + ); +} diff --git a/ui/components/usage-summary-cards.tsx b/ui/components/usage-summary-cards.tsx new file mode 100644 index 00000000..d12224c2 --- /dev/null +++ b/ui/components/usage-summary-cards.tsx @@ -0,0 +1,86 @@ +'use client'; + +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { UsageSummary } from '@/lib/api/services/admin'; +import { + CheckCircle2, + XCircle, + AlertTriangle, + Activity, + Database, + CreditCard, + TrendingUp, +} from 'lucide-react'; + +interface UsageSummaryCardsProps { + summary: UsageSummary; +} + +export function UsageSummaryCards({ summary }: UsageSummaryCardsProps) { + const cards = [ + { + title: 'Total Requests', + value: summary.total_requests.toLocaleString(), + icon: Activity, + color: 'text-blue-500', + }, + { + title: 'Successful Completions', + value: summary.successful_chat_completions.toLocaleString(), + icon: CheckCircle2, + color: 'text-green-500', + }, + { + title: 'Failed Requests', + value: summary.failed_requests.toLocaleString(), + icon: XCircle, + color: 'text-red-500', + }, + { + title: 'Errors', + value: summary.total_errors.toLocaleString(), + icon: AlertTriangle, + color: 'text-orange-500', + }, + { + title: 'Success Rate', + value: `${summary.success_rate.toFixed(1)}%`, + icon: TrendingUp, + color: 'text-emerald-500', + }, + { + title: 'Unique Models', + value: summary.unique_models_count.toLocaleString(), + icon: Database, + color: 'text-purple-500', + }, + { + title: 'Payments Processed', + value: summary.payment_processed.toLocaleString(), + icon: CreditCard, + color: 'text-indigo-500', + }, + { + title: 'Upstream Errors', + value: summary.upstream_errors.toLocaleString(), + icon: AlertTriangle, + color: 'text-yellow-500', + }, + ]; + + return ( +
+ {cards.map((card) => ( + + + {card.title} + + + +
{card.value}
+ + + ))} +
+ ); +} diff --git a/ui/lib/api/services/admin.ts b/ui/lib/api/services/admin.ts index 9645a6a1..b0dc0cad 100644 --- a/ui/lib/api/services/admin.ts +++ b/ui/lib/api/services/admin.ts @@ -802,6 +802,30 @@ export class AdminService { '/admin/api/temporary-balances' ); } + + static async getUsageMetrics( + interval: number = 15, + hours: number = 24 + ): Promise { + return await apiClient.get( + `/admin/api/usage/metrics?interval=${interval}&hours=${hours}` + ); + } + + static async getUsageSummary(hours: number = 24): Promise { + return await apiClient.get( + `/admin/api/usage/summary?hours=${hours}` + ); + } + + static async getErrorDetails( + hours: number = 24, + limit: number = 100 + ): Promise { + return await apiClient.get( + `/admin/api/usage/error-details?hours=${hours}&limit=${limit}` + ); + } } export const TemporaryBalanceSchema = z.object({ @@ -814,3 +838,50 @@ export const TemporaryBalanceSchema = z.object({ }); export type TemporaryBalance = z.infer; + +export interface UsageMetricData { + timestamp: string; + total_requests: number; + successful_chat_completions: number; + failed_requests: number; + errors: number; + warnings: number; + payment_processed: number; + upstream_errors: number; +} + +export interface UsageMetrics { + metrics: UsageMetricData[]; + interval_minutes: number; + hours_back: number; + total_buckets: number; +} + +export interface UsageSummary { + total_entries: number; + total_requests: number; + successful_chat_completions: number; + failed_requests: number; + total_errors: number; + total_warnings: number; + payment_processed: number; + upstream_errors: number; + unique_models_count: number; + unique_models: string[]; + error_types: Record; + success_rate: number; +} + +export interface ErrorDetail { + timestamp: string; + message: string; + error_type: string; + pathname: string; + lineno: number; + request_id: string; +} + +export interface ErrorDetails { + errors: ErrorDetail[]; + total_count: number; +}