Compare commits

...
Author SHA1 Message Date
Cursor Agentanddb2002dominic b25abbfe9a Refactor: Use total_msats for revenue calculation
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-16 20:12:34 +00:00
Cursor Agentanddb2002dominic fbeb10ee49 Refactor: Improve type safety for UsageMetricsChart data
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-16 20:01:58 +00:00
Cursor Agentanddb2002dominic d7e78b66d5 Refactor: Add type hints and improve sorting in get_revenue_by_model
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-16 19:53:03 +00:00
Cursor Agentanddb2002dominic 757637d86e Refactor: Add type assertions and improve stats calculation
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-16 19:51:33 +00:00
Cursor Agentanddb2002dominic e9c8a0c03b Refactor: Improve type hints and assertions in metrics aggregation
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-16 19:49:02 +00:00
Shroominic ca2e442656 ruff fmt+lint 2025-11-16 11:46:15 -08:00
Cursor Agentanddb2002dominic 340391ee1f Remove unused USAGE_TRACKING.md documentation
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-16 19:44:03 +00:00
Cursor Agentanddb2002dominic 7e1bd2125c Refactor: Document and enforce critical log messages for usage tracking
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-16 19:40:16 +00:00
Cursor Agentanddb2002dominic 9921f3a3b0 feat: Add revenue tracking and reporting by model
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-16 19:36:52 +00:00
Cursor Agentanddb2002dominic 765b3e6fd8 feat: Add usage tracking and analytics dashboard
Co-authored-by: db2002dominic <db2002dominic@gmail.com>
2025-11-15 23:37:00 +00:00
13 changed files with 1444 additions and 2 deletions
+4
View File
@@ -370,6 +370,8 @@ async def pay_for_request(
await session.refresh(key)
# IMPORTANT: Do not modify this log message - used for usage statistics tracking
# This log is parsed to count successful payment processing events
logger.info(
"Payment processed successfully",
extra={
@@ -399,6 +401,8 @@ async def revert_pay_for_request(
result = await session.exec(stmt) # type: ignore[call-overload]
await session.commit()
if result.rowcount == 0:
# IMPORTANT: Do not modify this log message - used for usage statistics tracking
# This error log indicates a refund failure (tracked separately from successful refunds)
logger.error(
"Failed to revert payment - insufficient reserved balance",
extra={
+614 -2
View File
@@ -1,9 +1,10 @@
import json
import secrets
from datetime import datetime, timezone
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from pydantic import BaseModel
from sqlmodel import select
@@ -2865,3 +2866,614 @@ h1 { color: #333; }
.no-logs { text-align: center; color: #666; padding: 40px; }
.request-id-display { background-color: #e9ecef; padding: 10px; border-radius: 4px; margin-bottom: 20px; font-family: monospace; }
"""
def _parse_log_file(file_path: Path) -> list[dict]:
"""Parse JSON log file and return list of log entries."""
entries: list[dict] = []
try:
with open(file_path) as f:
for line in f:
try:
entry = json.loads(line.strip())
entries.append(entry)
except json.JSONDecodeError:
continue
except Exception as e:
logger.error(f"Error reading log file {file_path}: {e}")
return entries
def _aggregate_metrics_by_time(
entries: list[dict], interval_minutes: int, hours_back: int = 24
) -> dict:
"""
Aggregate log metrics into time buckets.
CRITICAL: This function parses specific log messages for usage tracking.
The following log messages must not be modified without updating this function:
- "received proxy request" -> counts total_requests
- "token adjustment completed" -> counts successful_chat_completions, extracts revenue_msats from cost_data.total_msats
- "upstream request failed" OR "revert payment" -> counts failed_requests
- "payment processed successfully" -> counts payment_processed
- "revert payment" -> extracts refunds_msats from max_cost_for_model
- ERROR level with "upstream" -> counts upstream_errors
See routstr/core/logging.py for full documentation of critical log messages.
"""
now = datetime.now(timezone.utc)
cutoff = now - timedelta(hours=hours_back)
time_buckets: dict[str, dict[str, int | float]] = defaultdict(
lambda: {
"total_requests": 0,
"successful_chat_completions": 0,
"failed_requests": 0,
"errors": 0,
"warnings": 0,
"payment_processed": 0,
"upstream_errors": 0,
"revenue_msats": 0,
"refunds_msats": 0,
}
)
for entry in entries:
try:
timestamp_str = entry.get("asctime", "")
if not timestamp_str:
continue
log_time = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S")
log_time = log_time.replace(tzinfo=timezone.utc)
if log_time < cutoff:
continue
bucket_time = log_time.replace(
minute=(log_time.minute // interval_minutes) * interval_minutes,
second=0,
microsecond=0,
)
bucket_key = bucket_time.isoformat()
message = entry.get("message", "").lower()
level = entry.get("levelname", "").upper()
if level == "ERROR":
time_buckets[bucket_key]["errors"] += 1
elif level == "WARNING":
time_buckets[bucket_key]["warnings"] += 1
if "received proxy request" in message:
time_buckets[bucket_key]["total_requests"] += 1
if "token adjustment completed for non-streaming" in message:
time_buckets[bucket_key]["successful_chat_completions"] += 1
elif "token adjustment completed for streaming" in message:
time_buckets[bucket_key]["successful_chat_completions"] += 1
if "upstream request failed" in message or "revert payment" in message:
time_buckets[bucket_key]["failed_requests"] += 1
if "payment processed successfully" in message:
time_buckets[bucket_key]["payment_processed"] += 1
if "upstream" in message and level == "ERROR":
time_buckets[bucket_key]["upstream_errors"] += 1
if "token adjustment completed" in message:
cost_data = entry.get("cost_data")
if isinstance(cost_data, dict):
actual_cost = cost_data.get("total_msats", 0)
if isinstance(actual_cost, (int, float)) and actual_cost > 0:
time_buckets[bucket_key]["revenue_msats"] += actual_cost
if "revert payment" in message:
max_cost = entry.get("max_cost_for_model", 0)
if isinstance(max_cost, (int, float)) and max_cost > 0:
time_buckets[bucket_key]["refunds_msats"] += max_cost
except Exception:
continue
result = []
for bucket_key in sorted(time_buckets.keys()):
result.append({"timestamp": bucket_key, **time_buckets[bucket_key]})
return {
"metrics": result,
"interval_minutes": interval_minutes,
"hours_back": hours_back,
"total_buckets": len(result),
}
def _get_summary_stats(entries: list[dict], hours_back: int = 24) -> dict:
"""
Calculate summary statistics from log entries.
CRITICAL: This function parses specific log messages for usage tracking.
See routstr/core/logging.py for documentation of critical log messages.
Changes to log messages in proxy.py, auth.py, or upstream/base.py may break this function.
"""
now = datetime.now(timezone.utc)
cutoff = now - timedelta(hours=hours_back)
stats: dict[str, int | float | set[str] | defaultdict[str, int]] = {
"total_entries": 0,
"total_requests": 0,
"successful_chat_completions": 0,
"failed_requests": 0,
"total_errors": 0,
"total_warnings": 0,
"payment_processed": 0,
"upstream_errors": 0,
"unique_models": set(),
"error_types": defaultdict(int),
"revenue_msats": 0,
"refunds_msats": 0,
"revenue_sats": 0.0,
"refunds_sats": 0.0,
"net_revenue_msats": 0,
"net_revenue_sats": 0.0,
}
for entry in entries:
try:
timestamp_str = entry.get("asctime", "")
if not timestamp_str:
continue
log_time = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S")
log_time = log_time.replace(tzinfo=timezone.utc)
if log_time < cutoff:
continue
total_entries = stats["total_entries"]
assert isinstance(total_entries, int)
stats["total_entries"] = total_entries + 1
message = entry.get("message", "").lower()
level = entry.get("levelname", "").upper()
if level == "ERROR":
assert isinstance(stats["total_errors"], int)
stats["total_errors"] += 1
if "error_type" in entry:
error_type = str(entry["error_type"])
error_types = stats["error_types"]
assert isinstance(error_types, defaultdict)
error_types[error_type] += 1
elif level == "WARNING":
assert isinstance(stats["total_warnings"], int)
stats["total_warnings"] += 1
if "received proxy request" in message:
assert isinstance(stats["total_requests"], int)
stats["total_requests"] += 1
if "token adjustment completed" in message:
assert isinstance(stats["successful_chat_completions"], int)
stats["successful_chat_completions"] += 1
if "upstream request failed" in message or "revert payment" in message:
assert isinstance(stats["failed_requests"], int)
stats["failed_requests"] += 1
if "payment processed successfully" in message:
assert isinstance(stats["payment_processed"], int)
stats["payment_processed"] += 1
if "upstream" in message and level == "ERROR":
assert isinstance(stats["upstream_errors"], int)
stats["upstream_errors"] += 1
if "model" in entry:
model = entry["model"]
if isinstance(model, str) and model != "unknown":
unique_models = stats["unique_models"]
assert isinstance(unique_models, set)
unique_models.add(model)
if "token adjustment completed" in message:
cost_data = entry.get("cost_data")
if isinstance(cost_data, dict):
actual_cost = cost_data.get("total_msats", 0)
if isinstance(actual_cost, (int, float)) and actual_cost > 0:
assert isinstance(stats["revenue_msats"], (int, float))
stats["revenue_msats"] = float(stats["revenue_msats"]) + float(actual_cost)
if "revert payment" in message:
max_cost = entry.get("max_cost_for_model", 0)
if isinstance(max_cost, (int, float)) and max_cost > 0:
assert isinstance(stats["refunds_msats"], (int, float))
stats["refunds_msats"] = float(stats["refunds_msats"]) + float(max_cost)
except Exception:
continue
revenue_msats_val = stats["revenue_msats"]
assert isinstance(revenue_msats_val, (int, float))
revenue_msats = float(revenue_msats_val)
refunds_msats_val = stats["refunds_msats"]
assert isinstance(refunds_msats_val, (int, float))
refunds_msats = float(refunds_msats_val)
revenue_sats = revenue_msats / 1000
refunds_sats = refunds_msats / 1000
net_revenue_msats = revenue_msats - refunds_msats
net_revenue_sats = net_revenue_msats / 1000
unique_models = stats["unique_models"]
assert isinstance(unique_models, set)
error_types = stats["error_types"]
assert isinstance(error_types, defaultdict)
total_entries_val = stats["total_entries"]
assert isinstance(total_entries_val, int)
total_requests_val = stats["total_requests"]
assert isinstance(total_requests_val, int)
successful_completions_val = stats["successful_chat_completions"]
assert isinstance(successful_completions_val, int)
failed_requests_val = stats["failed_requests"]
assert isinstance(failed_requests_val, int)
total_errors_val = stats["total_errors"]
assert isinstance(total_errors_val, int)
total_warnings_val = stats["total_warnings"]
assert isinstance(total_warnings_val, int)
payment_processed_val = stats["payment_processed"]
assert isinstance(payment_processed_val, int)
upstream_errors_val = stats["upstream_errors"]
assert isinstance(upstream_errors_val, int)
return {
"total_entries": total_entries_val,
"total_requests": total_requests_val,
"successful_chat_completions": successful_completions_val,
"failed_requests": failed_requests_val,
"total_errors": total_errors_val,
"total_warnings": total_warnings_val,
"payment_processed": payment_processed_val,
"upstream_errors": upstream_errors_val,
"unique_models_count": len(unique_models),
"unique_models": sorted(list(unique_models)),
"error_types": dict(error_types),
"success_rate": (
(successful_completions_val / total_requests_val * 100)
if total_requests_val > 0
else 0
),
"revenue_msats": revenue_msats,
"refunds_msats": refunds_msats,
"revenue_sats": revenue_sats,
"refunds_sats": refunds_sats,
"net_revenue_msats": net_revenue_msats,
"net_revenue_sats": net_revenue_sats,
"avg_revenue_per_request_msats": (
revenue_msats / successful_completions_val
if successful_completions_val > 0
else 0
),
"refund_rate": (
(failed_requests_val / total_requests_val * 100)
if total_requests_val > 0
else 0
),
}
@admin_router.get("/api/usage/metrics", dependencies=[Depends(require_admin_api)])
async def get_usage_metrics(
request: Request,
interval: int = Query(
default=15, ge=1, le=1440, description="Time interval in minutes"
),
hours: int = Query(
default=24, ge=1, le=168, description="Hours of history to analyze"
),
) -> dict:
"""Get usage metrics aggregated by time interval."""
logs_dir = Path("logs")
all_entries: list[dict] = []
if not logs_dir.exists():
return {
"metrics": [],
"interval_minutes": interval,
"hours_back": hours,
"total_buckets": 0,
}
cutoff_date = datetime.now(timezone.utc) - timedelta(hours=hours)
for log_file in sorted(logs_dir.glob("app_*.log")):
try:
file_date_str = log_file.stem.split("_")[1]
file_date = datetime.strptime(file_date_str, "%Y-%m-%d").replace(
tzinfo=timezone.utc
)
if file_date < cutoff_date.replace(
hour=0, minute=0, second=0, microsecond=0
):
continue
entries = _parse_log_file(log_file)
all_entries.extend(entries)
except Exception as e:
logger.error(f"Error processing log file {log_file}: {e}")
continue
return _aggregate_metrics_by_time(all_entries, interval, hours)
@admin_router.get("/api/usage/summary", dependencies=[Depends(require_admin_api)])
async def get_usage_summary(
request: Request,
hours: int = Query(
default=24, ge=1, le=168, description="Hours of history to analyze"
),
) -> dict:
"""Get summary statistics for the specified time period."""
logs_dir = Path("logs")
all_entries: list[dict] = []
if not logs_dir.exists():
return {
"total_entries": 0,
"total_requests": 0,
"successful_chat_completions": 0,
"failed_requests": 0,
"total_errors": 0,
"total_warnings": 0,
"payment_processed": 0,
"upstream_errors": 0,
"unique_models_count": 0,
"unique_models": [],
"error_types": {},
"success_rate": 0,
}
cutoff_date = datetime.now(timezone.utc) - timedelta(hours=hours)
for log_file in sorted(logs_dir.glob("app_*.log")):
try:
file_date_str = log_file.stem.split("_")[1]
file_date = datetime.strptime(file_date_str, "%Y-%m-%d").replace(
tzinfo=timezone.utc
)
if file_date < cutoff_date.replace(
hour=0, minute=0, second=0, microsecond=0
):
continue
entries = _parse_log_file(log_file)
all_entries.extend(entries)
except Exception as e:
logger.error(f"Error processing log file {log_file}: {e}")
continue
return _get_summary_stats(all_entries, hours)
@admin_router.get("/api/usage/error-details", dependencies=[Depends(require_admin_api)])
async def get_error_details(
request: Request,
hours: int = Query(
default=24, ge=1, le=168, description="Hours of history to analyze"
),
limit: int = Query(
default=100, ge=1, le=1000, description="Maximum number of errors to return"
),
) -> dict:
"""Get detailed error information."""
logs_dir = Path("logs")
errors: list[dict] = []
if not logs_dir.exists():
return {"errors": [], "total_count": 0}
cutoff_date = datetime.now(timezone.utc) - timedelta(hours=hours)
for log_file in sorted(logs_dir.glob("app_*.log"), reverse=True):
try:
file_date_str = log_file.stem.split("_")[1]
file_date = datetime.strptime(file_date_str, "%Y-%m-%d").replace(
tzinfo=timezone.utc
)
if file_date < cutoff_date.replace(
hour=0, minute=0, second=0, microsecond=0
):
continue
entries = _parse_log_file(log_file)
for entry in entries:
if entry.get("levelname", "").upper() == "ERROR":
timestamp_str = entry.get("asctime", "")
if timestamp_str:
log_time = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S")
log_time = log_time.replace(tzinfo=timezone.utc)
if log_time >= cutoff_date:
errors.append(
{
"timestamp": timestamp_str,
"message": entry.get("message", ""),
"error_type": entry.get("error_type", "unknown"),
"pathname": entry.get("pathname", ""),
"lineno": entry.get("lineno", 0),
"request_id": entry.get("request_id", ""),
}
)
if len(errors) >= limit:
break
except Exception as e:
logger.error(f"Error processing log file {log_file}: {e}")
continue
if len(errors) >= limit:
break
errors.sort(key=lambda x: x["timestamp"], reverse=True)
return {"errors": errors[:limit], "total_count": len(errors)}
@admin_router.get(
"/api/usage/revenue-by-model", dependencies=[Depends(require_admin_api)]
)
async def get_revenue_by_model(
request: Request,
hours: int = Query(
default=24, ge=1, le=168, description="Hours of history to analyze"
),
limit: int = Query(
default=20, ge=1, le=100, description="Maximum number of models to return"
),
) -> dict:
"""
Get revenue breakdown by model.
CRITICAL: This function parses specific log messages for revenue tracking.
See routstr/core/logging.py for documentation of critical log messages.
Changes to log messages may break revenue calculations per model.
"""
logs_dir = Path("logs")
all_entries: list[dict] = []
if not logs_dir.exists():
return {"models": [], "total_revenue_sats": 0}
cutoff_date = datetime.now(timezone.utc) - timedelta(hours=hours)
for log_file in sorted(logs_dir.glob("app_*.log")):
try:
file_date_str = log_file.stem.split("_")[1]
file_date = datetime.strptime(file_date_str, "%Y-%m-%d").replace(
tzinfo=timezone.utc
)
if file_date < cutoff_date.replace(
hour=0, minute=0, second=0, microsecond=0
):
continue
entries = _parse_log_file(log_file)
all_entries.extend(entries)
except Exception as e:
logger.error(f"Error processing log file {log_file}: {e}")
continue
model_stats: dict[str, dict[str, int | float]] = defaultdict(
lambda: {
"revenue_msats": 0,
"refunds_msats": 0,
"requests": 0,
"successful": 0,
"failed": 0,
}
)
for entry in all_entries:
try:
timestamp_str = entry.get("asctime", "")
if not timestamp_str:
continue
log_time = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S")
log_time = log_time.replace(tzinfo=timezone.utc)
if log_time < cutoff_date:
continue
model = entry.get("model", "unknown")
if not isinstance(model, str):
model = "unknown"
message = entry.get("message", "").lower()
if "received proxy request" in message:
model_stats[model]["requests"] += 1
if "token adjustment completed" in message:
model_stats[model]["successful"] += 1
cost_data = entry.get("cost_data")
if isinstance(cost_data, dict):
actual_cost = cost_data.get("total_msats", 0)
if isinstance(actual_cost, (int, float)) and actual_cost > 0:
model_stats[model]["revenue_msats"] += actual_cost
if "revert payment" in message or "upstream request failed" in message:
model_stats[model]["failed"] += 1
if "revert payment" in message:
max_cost = entry.get("max_cost_for_model", 0)
if isinstance(max_cost, (int, float)) and max_cost > 0:
model_stats[model]["refunds_msats"] += max_cost
except Exception:
continue
models: list[dict[str, str | int | float]] = []
total_revenue = 0.0
for model, stats in model_stats.items():
revenue_msats_raw = stats["revenue_msats"]
assert isinstance(revenue_msats_raw, (int, float))
revenue_msats_val = float(revenue_msats_raw)
refunds_msats_raw = stats["refunds_msats"]
assert isinstance(refunds_msats_raw, (int, float))
refunds_msats_val = float(refunds_msats_raw)
revenue_sats = revenue_msats_val / 1000
refunds_sats = refunds_msats_val / 1000
net_revenue_sats = revenue_sats - refunds_sats
total_revenue += net_revenue_sats
requests_raw = stats["requests"]
assert isinstance(requests_raw, int)
requests_val = requests_raw
successful_raw = stats["successful"]
assert isinstance(successful_raw, int)
successful_val = successful_raw
failed_raw = stats["failed"]
assert isinstance(failed_raw, int)
failed_val = failed_raw
model_data: dict[str, str | int | float] = {
"model": model,
"revenue_sats": revenue_sats,
"refunds_sats": refunds_sats,
"net_revenue_sats": net_revenue_sats,
"requests": requests_val,
"successful": successful_val,
"failed": failed_val,
"avg_revenue_per_request": (
revenue_sats / successful_val if successful_val > 0 else 0
),
}
models.append(model_data)
def _sort_key(x: dict[str, str | int | float]) -> float:
val = x["net_revenue_sats"]
assert isinstance(val, (int, float))
return float(val)
models.sort(key=_sort_key, reverse=True)
return {
"models": models[:limit],
"total_revenue_sats": total_revenue,
"total_models": len(models),
}
+37
View File
@@ -1,3 +1,40 @@
"""
Logging configuration for Routstr.
CRITICAL LOG MESSAGES FOR USAGE STATISTICS:
===========================================
The following log messages are parsed by the usage tracking system (routstr/core/admin.py).
DO NOT modify or remove these messages without updating the usage tracking logic:
1. "Received proxy request" (INFO) - routstr/proxy.py
- Used to count total incoming requests
- Includes model information in context
2. "Token adjustment completed for streaming" (INFO) - routstr/upstream/base.py
"Token adjustment completed for non-streaming" (INFO) - routstr/upstream/base.py
- Used to track successful completions and revenue
- The 'cost_data.total_msats' field is extracted for revenue calculation
- Must include 'cost_data' in extra dict
3. "Payment processed successfully" (INFO) - routstr/auth.py
- Used to count successful payment processing events
- Tracks payment-related metrics
4. "Upstream request failed, revert payment" (WARNING) - routstr/proxy.py
- Used to track failed requests and refunds
- The 'max_cost_for_model' field is extracted for refund calculation
- Must include 'max_cost_for_model' in extra dict
5. Any ERROR level logs with "upstream" in the message
- Used to count upstream provider errors
- Helps identify service reliability issues
If you need to modify these messages, ensure you also update the parsing logic in:
- routstr/core/admin.py:_aggregate_metrics_by_time()
- routstr/core/admin.py:_get_summary_stats()
- routstr/core/admin.py:get_revenue_by_model()
"""
import logging.config
import logging.handlers
import os
+9
View File
@@ -273,6 +273,15 @@ if UI_DIST_PATH.exists() and UI_DIST_PATH.is_dir():
async def redirect_unauthorized_index_txt() -> RedirectResponse:
return RedirectResponse("/unauthorized")
@app.get("/usage", include_in_schema=False)
async def serve_usage_ui() -> FileResponse:
return FileResponse(UI_DIST_PATH / "usage" / "index.html")
# Add explicit route for /usage/index.txt to redirect to /usage
@app.get("/usage/index.txt", include_in_schema=False)
async def redirect_usage_index_txt() -> RedirectResponse:
return RedirectResponse("/usage")
@app.get("/favicon.ico", include_in_schema=False)
async def serve_favicon() -> FileResponse:
icon_path = UI_DIST_PATH / "icon.ico"
+5
View File
@@ -141,6 +141,8 @@ async def proxy(
"unauthorized", "Unauthorized", 401, request=request
)
# IMPORTANT: Do not modify this log message - used for usage statistics tracking
# This log is parsed by the usage tracking system to count total requests
logger.info( # TODO: move to middleware, async
"Received proxy request",
extra={
@@ -222,6 +224,9 @@ async def proxy(
if response.status_code != 200:
await revert_pay_for_request(key, session, max_cost_for_model)
# IMPORTANT: Do not modify this log message - used for usage statistics tracking
# This log is parsed to track refunds and failed requests
# The 'max_cost_for_model' field is used to calculate total refunds
logger.warning(
"Upstream request failed, revert payment",
extra={
+6
View File
@@ -452,6 +452,9 @@ class BaseUpstreamProvider:
)
)
usage_finalized = True
# IMPORTANT: Do not modify this log message - used for usage statistics tracking
# This log is parsed to track revenue from successful requests
# The 'cost_data.total_msats' field is used to calculate total revenue
logger.info(
"Token adjustment completed for streaming",
extra={
@@ -555,6 +558,9 @@ class BaseUpstreamProvider:
)
response_json["cost"] = cost_data
# IMPORTANT: Do not modify this log message - used for usage statistics tracking
# This log is parsed to track revenue from successful requests
# The 'cost_data.total_msats' field is used to calculate total revenue
logger.info(
"Token adjustment completed for non-streaming",
extra={
+302
View File
@@ -0,0 +1,302 @@
'use client';
import { useState } from 'react';
import { useQuery } from '@tanstack/react-query';
import { AppSidebar } from '@/components/app-sidebar';
import { SiteHeader } from '@/components/site-header';
import { SidebarInset, SidebarProvider } from '@/components/ui/sidebar';
import { UsageMetricsChart } from '@/components/usage-metrics-chart';
import { UsageSummaryCards } from '@/components/usage-summary-cards';
import { ErrorDetailsTable } from '@/components/error-details-table';
import { RevenueByModelTable } from '@/components/revenue-by-model-table';
import { AdminService } from '@/lib/api/services/admin';
import { Button } from '@/components/ui/button';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import { RefreshCw } from 'lucide-react';
export default function UsagePage() {
const [timeRange, setTimeRange] = useState('24');
const [interval, setInterval] = useState('15');
const {
data: metricsData,
isLoading: metricsLoading,
refetch: refetchMetrics,
} = useQuery({
queryKey: ['usage-metrics', interval, timeRange],
queryFn: () =>
AdminService.getUsageMetrics(parseInt(interval), parseInt(timeRange)),
refetchInterval: 60_000,
staleTime: 30_000,
});
const {
data: summaryData,
isLoading: summaryLoading,
refetch: refetchSummary,
} = useQuery({
queryKey: ['usage-summary', timeRange],
queryFn: () => AdminService.getUsageSummary(parseInt(timeRange)),
refetchInterval: 60_000,
staleTime: 30_000,
});
const {
data: errorData,
isLoading: errorLoading,
refetch: refetchErrors,
} = useQuery({
queryKey: ['usage-errors', timeRange],
queryFn: () => AdminService.getErrorDetails(parseInt(timeRange), 100),
refetchInterval: 60_000,
staleTime: 30_000,
});
const {
data: revenueByModelData,
isLoading: revenueByModelLoading,
refetch: refetchRevenueByModel,
} = useQuery({
queryKey: ['revenue-by-model', timeRange],
queryFn: () => AdminService.getRevenueByModel(parseInt(timeRange), 20),
refetchInterval: 60_000,
staleTime: 30_000,
});
const handleRefresh = () => {
refetchMetrics();
refetchSummary();
refetchErrors();
refetchRevenueByModel();
};
return (
<SidebarProvider>
<AppSidebar variant='inset' />
<SidebarInset className='p-0'>
<SiteHeader />
<div className='container max-w-7xl px-4 py-8 md:px-6 lg:px-8'>
<div className='mb-8 flex flex-col gap-4 lg:flex-row lg:items-start lg:justify-between'>
<div>
<h1 className='text-3xl font-bold tracking-tight'>
Usage Tracking
</h1>
<p className='text-muted-foreground mt-2'>
Monitor system usage, requests, and errors over time
</p>
</div>
<div className='flex items-center gap-4'>
<Select value={timeRange} onValueChange={setTimeRange}>
<SelectTrigger className='w-[180px]'>
<SelectValue placeholder='Select time range' />
</SelectTrigger>
<SelectContent>
<SelectItem value='1'>Last Hour</SelectItem>
<SelectItem value='6'>Last 6 Hours</SelectItem>
<SelectItem value='24'>Last 24 Hours</SelectItem>
<SelectItem value='72'>Last 3 Days</SelectItem>
<SelectItem value='168'>Last Week</SelectItem>
</SelectContent>
</Select>
<Select value={interval} onValueChange={setInterval}>
<SelectTrigger className='w-[180px]'>
<SelectValue placeholder='Select interval' />
</SelectTrigger>
<SelectContent>
<SelectItem value='5'>5 Minutes</SelectItem>
<SelectItem value='15'>15 Minutes</SelectItem>
<SelectItem value='30'>30 Minutes</SelectItem>
<SelectItem value='60'>1 Hour</SelectItem>
</SelectContent>
</Select>
<Button onClick={handleRefresh} variant='outline' size='icon'>
<RefreshCw className='h-4 w-4' />
</Button>
</div>
</div>
<div className='space-y-6'>
{summaryLoading ? (
<div className='text-center py-8'>Loading summary...</div>
) : summaryData ? (
<UsageSummaryCards summary={summaryData} />
) : null}
<div className='grid gap-6 lg:grid-cols-2'>
{metricsLoading ? (
<div className='text-center py-8 col-span-2'>
Loading metrics...
</div>
) : metricsData && metricsData.metrics.length > 0 ? (
<>
<UsageMetricsChart
data={metricsData.metrics as Array<Record<string, unknown> & { timestamp: string }>}
title='Request Volume'
dataKeys={[
{
key: 'total_requests',
name: 'Total Requests',
color: '#3b82f6',
},
{
key: 'successful_chat_completions',
name: 'Successful',
color: '#10b981',
},
{
key: 'failed_requests',
name: 'Failed',
color: '#ef4444',
},
]}
/>
<UsageMetricsChart
data={metricsData.metrics.map((m) => ({
...m,
revenue_sats: m.revenue_msats / 1000,
refunds_sats: m.refunds_msats / 1000,
net_revenue_sats:
(m.revenue_msats - m.refunds_msats) / 1000,
})) as Array<Record<string, unknown> & { timestamp: string }>}
title='Revenue Over Time (sats)'
dataKeys={[
{
key: 'revenue_sats',
name: 'Revenue',
color: '#10b981',
},
{
key: 'refunds_sats',
name: 'Refunds',
color: '#ef4444',
},
{
key: 'net_revenue_sats',
name: 'Net Revenue',
color: '#059669',
},
]}
/>
<UsageMetricsChart
data={metricsData.metrics as Array<Record<string, unknown> & { timestamp: string }>}
title='Error Tracking'
dataKeys={[
{
key: 'errors',
name: 'Errors',
color: '#f97316',
},
{
key: 'warnings',
name: 'Warnings',
color: '#eab308',
},
{
key: 'upstream_errors',
name: 'Upstream Errors',
color: '#ef4444',
},
]}
/>
<UsageMetricsChart
data={metricsData.metrics as Array<Record<string, unknown> & { timestamp: string }>}
title='Payment Activity'
dataKeys={[
{
key: 'payment_processed',
name: 'Payments Processed',
color: '#6366f1',
},
]}
/>
</>
) : (
<Card className='col-span-2'>
<CardHeader>
<CardTitle>No Data Available</CardTitle>
</CardHeader>
<CardContent>
<p className='text-muted-foreground'>
No metrics data found for the selected time range. This
could be because no requests have been logged yet or the
log files are not available.
</p>
</CardContent>
</Card>
)}
</div>
{revenueByModelLoading ? (
<div className='text-center py-8'>Loading revenue by model...</div>
) : revenueByModelData && revenueByModelData.models.length > 0 ? (
<RevenueByModelTable
models={revenueByModelData.models}
totalRevenue={revenueByModelData.total_revenue_sats}
/>
) : null}
{errorLoading ? (
<div className='text-center py-8'>Loading errors...</div>
) : errorData ? (
<ErrorDetailsTable errors={errorData.errors} />
) : null}
{summaryData && summaryData.unique_models.length > 0 && (
<Card>
<CardHeader>
<CardTitle>Active Models</CardTitle>
</CardHeader>
<CardContent>
<div className='flex flex-wrap gap-2'>
{summaryData.unique_models.map((model) => (
<span
key={model}
className='bg-secondary text-secondary-foreground inline-flex items-center rounded-md px-2.5 py-0.5 text-xs font-semibold'
>
{model}
</span>
))}
</div>
</CardContent>
</Card>
)}
{summaryData &&
summaryData.error_types &&
Object.keys(summaryData.error_types).length > 0 && (
<Card>
<CardHeader>
<CardTitle>Error Types Distribution</CardTitle>
</CardHeader>
<CardContent>
<div className='space-y-2'>
{Object.entries(summaryData.error_types)
.sort(([, a], [, b]) => b - a)
.map(([type, count]) => (
<div
key={type}
className='flex items-center justify-between'
>
<span className='text-sm font-medium'>{type}</span>
<span className='text-muted-foreground text-sm'>
{count}
</span>
</div>
))}
</div>
</CardContent>
</Card>
)}
</div>
</div>
</SidebarInset>
</SidebarProvider>
);
}
+6
View File
@@ -2,6 +2,7 @@
import * as React from 'react';
import {
ActivityIcon,
DatabaseIcon,
LayoutDashboardIcon,
ServerIcon,
@@ -34,6 +35,11 @@ const data = {
url: '/',
icon: LayoutDashboardIcon,
},
{
title: 'Usage',
url: '/usage',
icon: ActivityIcon,
},
{
title: 'Models',
url: '/model',
+78
View File
@@ -0,0 +1,78 @@
'use client';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import {
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
} from '@/components/ui/table';
import { Badge } from '@/components/ui/badge';
import { ErrorDetail } from '@/lib/api/services/admin';
interface ErrorDetailsTableProps {
errors: ErrorDetail[];
}
export function ErrorDetailsTable({ errors }: ErrorDetailsTableProps) {
if (errors.length === 0) {
return (
<Card>
<CardHeader>
<CardTitle>Recent Errors</CardTitle>
</CardHeader>
<CardContent>
<p className='text-muted-foreground text-center py-8'>
No errors found in the selected time period
</p>
</CardContent>
</Card>
);
}
return (
<Card>
<CardHeader>
<CardTitle>Recent Errors ({errors.length})</CardTitle>
</CardHeader>
<CardContent>
<div className='max-h-[400px] overflow-y-auto'>
<Table>
<TableHeader>
<TableRow>
<TableHead>Timestamp</TableHead>
<TableHead>Type</TableHead>
<TableHead>Message</TableHead>
<TableHead>Location</TableHead>
<TableHead>Request ID</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{errors.map((error, index) => (
<TableRow key={index}>
<TableCell className='font-mono text-xs'>
{new Date(error.timestamp).toLocaleString()}
</TableCell>
<TableCell>
<Badge variant='destructive'>{error.error_type}</Badge>
</TableCell>
<TableCell className='max-w-md truncate'>
{error.message}
</TableCell>
<TableCell className='font-mono text-xs'>
{error.pathname}:{error.lineno}
</TableCell>
<TableCell className='font-mono text-xs'>
{error.request_id || '-'}
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
</CardContent>
</Card>
);
}
+79
View File
@@ -0,0 +1,79 @@
'use client';
import {
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
} from '@/components/ui/table';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import { ModelRevenueData } from '@/lib/api/services/admin';
interface RevenueByModelTableProps {
models: ModelRevenueData[];
totalRevenue: number;
}
export function RevenueByModelTable({
models,
totalRevenue,
}: RevenueByModelTableProps) {
return (
<Card>
<CardHeader>
<CardTitle>Revenue by Model</CardTitle>
<p className="text-sm text-muted-foreground">
Total Revenue: {totalRevenue.toLocaleString(undefined, { maximumFractionDigits: 2 })} sats
</p>
</CardHeader>
<CardContent>
<Table>
<TableHeader>
<TableRow>
<TableHead>Model</TableHead>
<TableHead className="text-right">Requests</TableHead>
<TableHead className="text-right">Successful</TableHead>
<TableHead className="text-right">Failed</TableHead>
<TableHead className="text-right">Revenue (sats)</TableHead>
<TableHead className="text-right">Refunds (sats)</TableHead>
<TableHead className="text-right">Net Revenue (sats)</TableHead>
<TableHead className="text-right">Avg/Request</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{models.length === 0 ? (
<TableRow>
<TableCell colSpan={8} className="text-center text-muted-foreground">
No model data available
</TableCell>
</TableRow>
) : (
models.map((model) => (
<TableRow key={model.model}>
<TableCell className="font-medium">{model.model}</TableCell>
<TableCell className="text-right">{model.requests}</TableCell>
<TableCell className="text-right text-green-600">{model.successful}</TableCell>
<TableCell className="text-right text-red-600">{model.failed}</TableCell>
<TableCell className="text-right">
{model.revenue_sats.toLocaleString(undefined, { maximumFractionDigits: 2 })}
</TableCell>
<TableCell className="text-right text-red-500">
{model.refunds_sats.toLocaleString(undefined, { maximumFractionDigits: 2 })}
</TableCell>
<TableCell className="text-right font-semibold">
{model.net_revenue_sats.toLocaleString(undefined, { maximumFractionDigits: 2 })}
</TableCell>
<TableCell className="text-right text-muted-foreground">
{model.avg_revenue_per_request.toLocaleString(undefined, { maximumFractionDigits: 3 })}
</TableCell>
</TableRow>
))
)}
</TableBody>
</Table>
</CardContent>
</Card>
);
}
+77
View File
@@ -0,0 +1,77 @@
'use client';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import {
Line,
LineChart,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
Legend,
CartesianGrid,
} from 'recharts';
interface UsageMetricsChartProps {
data: Array<Record<string, unknown> & { timestamp: string }>;
title: string;
dataKeys: Array<{
key: string;
name: string;
color: string;
}>;
}
export function UsageMetricsChart({
data,
title,
dataKeys,
}: UsageMetricsChartProps) {
const formattedData = data.map((item) => ({
...item,
time: new Date(item.timestamp).toLocaleTimeString([], {
hour: '2-digit',
minute: '2-digit',
}),
}));
return (
<Card>
<CardHeader>
<CardTitle>{title}</CardTitle>
</CardHeader>
<CardContent>
<ResponsiveContainer width='100%' height={300}>
<LineChart data={formattedData}>
<CartesianGrid strokeDasharray='3 3' className='stroke-muted' />
<XAxis
dataKey='time'
className='text-xs'
tick={{ fill: 'currentColor' }}
/>
<YAxis className='text-xs' tick={{ fill: 'currentColor' }} />
<Tooltip
contentStyle={{
backgroundColor: 'hsl(var(--background))',
border: '1px solid hsl(var(--border))',
borderRadius: '6px',
}}
/>
<Legend />
{dataKeys.map((dataKey) => (
<Line
key={dataKey.key}
type='monotone'
dataKey={dataKey.key}
stroke={dataKey.color}
name={dataKey.name}
strokeWidth={2}
dot={false}
/>
))}
</LineChart>
</ResponsiveContainer>
</CardContent>
</Card>
);
}
+119
View File
@@ -0,0 +1,119 @@
'use client';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import { UsageSummary } from '@/lib/api/services/admin';
import {
CheckCircle2,
XCircle,
AlertTriangle,
Activity,
Database,
CreditCard,
TrendingUp,
DollarSign,
TrendingDown,
Coins,
} from 'lucide-react';
interface UsageSummaryCardsProps {
summary: UsageSummary;
}
export function UsageSummaryCards({ summary }: UsageSummaryCardsProps) {
const cards = [
{
title: 'Total Requests',
value: summary.total_requests.toLocaleString(),
icon: Activity,
color: 'text-blue-500',
},
{
title: 'Successful Completions',
value: summary.successful_chat_completions.toLocaleString(),
icon: CheckCircle2,
color: 'text-green-500',
},
{
title: 'Revenue (sats)',
value: summary.revenue_sats.toLocaleString(undefined, {
maximumFractionDigits: 2,
}),
icon: Coins,
color: 'text-green-600',
},
{
title: 'Net Revenue (sats)',
value: summary.net_revenue_sats.toLocaleString(undefined, {
maximumFractionDigits: 2,
}),
icon: DollarSign,
color: 'text-emerald-600',
},
{
title: 'Refunds (sats)',
value: summary.refunds_sats.toLocaleString(undefined, {
maximumFractionDigits: 2,
}),
icon: TrendingDown,
color: 'text-red-500',
},
{
title: 'Avg Revenue/Request',
value: `${(summary.avg_revenue_per_request_msats / 1000).toLocaleString(undefined, { maximumFractionDigits: 3 })} sats`,
icon: CreditCard,
color: 'text-cyan-500',
},
{
title: 'Success Rate',
value: `${summary.success_rate.toFixed(1)}%`,
icon: TrendingUp,
color: 'text-emerald-500',
},
{
title: 'Refund Rate',
value: `${summary.refund_rate.toFixed(1)}%`,
icon: XCircle,
color: 'text-orange-500',
},
{
title: 'Failed Requests',
value: summary.failed_requests.toLocaleString(),
icon: XCircle,
color: 'text-red-400',
},
{
title: 'Errors',
value: summary.total_errors.toLocaleString(),
icon: AlertTriangle,
color: 'text-orange-500',
},
{
title: 'Unique Models',
value: summary.unique_models_count.toLocaleString(),
icon: Database,
color: 'text-purple-500',
},
{
title: 'Upstream Errors',
value: summary.upstream_errors.toLocaleString(),
icon: AlertTriangle,
color: 'text-yellow-500',
},
];
return (
<div className='grid gap-4 md:grid-cols-2 lg:grid-cols-4'>
{cards.map((card) => (
<Card key={card.title}>
<CardHeader className='flex flex-row items-center justify-between space-y-0 pb-2'>
<CardTitle className='text-sm font-medium'>{card.title}</CardTitle>
<card.icon className={`h-4 w-4 ${card.color}`} />
</CardHeader>
<CardContent>
<div className='text-2xl font-bold'>{card.value}</div>
</CardContent>
</Card>
))}
</div>
);
}
+108
View File
@@ -802,6 +802,39 @@ export class AdminService {
'/admin/api/temporary-balances'
);
}
static async getUsageMetrics(
interval: number = 15,
hours: number = 24
): Promise<UsageMetrics> {
return await apiClient.get<UsageMetrics>(
`/admin/api/usage/metrics?interval=${interval}&hours=${hours}`
);
}
static async getUsageSummary(hours: number = 24): Promise<UsageSummary> {
return await apiClient.get<UsageSummary>(
`/admin/api/usage/summary?hours=${hours}`
);
}
static async getErrorDetails(
hours: number = 24,
limit: number = 100
): Promise<ErrorDetails> {
return await apiClient.get<ErrorDetails>(
`/admin/api/usage/error-details?hours=${hours}&limit=${limit}`
);
}
static async getRevenueByModel(
hours: number = 24,
limit: number = 20
): Promise<RevenueByModel> {
return await apiClient.get<RevenueByModel>(
`/admin/api/usage/revenue-by-model?hours=${hours}&limit=${limit}`
);
}
}
export const TemporaryBalanceSchema = z.object({
@@ -814,3 +847,78 @@ export const TemporaryBalanceSchema = z.object({
});
export type TemporaryBalance = z.infer<typeof TemporaryBalanceSchema>;
export interface UsageMetricData {
timestamp: string;
total_requests: number;
successful_chat_completions: number;
failed_requests: number;
errors: number;
warnings: number;
payment_processed: number;
upstream_errors: number;
revenue_msats: number;
refunds_msats: number;
[key: string]: unknown;
}
export interface UsageMetrics {
metrics: UsageMetricData[];
interval_minutes: number;
hours_back: number;
total_buckets: number;
}
export interface UsageSummary {
total_entries: number;
total_requests: number;
successful_chat_completions: number;
failed_requests: number;
total_errors: number;
total_warnings: number;
payment_processed: number;
upstream_errors: number;
unique_models_count: number;
unique_models: string[];
error_types: Record<string, number>;
success_rate: number;
revenue_msats: number;
refunds_msats: number;
revenue_sats: number;
refunds_sats: number;
net_revenue_msats: number;
net_revenue_sats: number;
avg_revenue_per_request_msats: number;
refund_rate: number;
}
export interface ErrorDetail {
timestamp: string;
message: string;
error_type: string;
pathname: string;
lineno: number;
request_id: string;
}
export interface ErrorDetails {
errors: ErrorDetail[];
total_count: number;
}
export interface ModelRevenueData {
model: string;
revenue_sats: number;
refunds_sats: number;
net_revenue_sats: number;
requests: number;
successful: number;
failed: number;
avg_revenue_per_request: number;
}
export interface RevenueByModel {
models: ModelRevenueData[];
total_revenue_sats: number;
total_models: number;
}