mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-07-30 23:36:15 +00:00
Compare commits
16
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7ffebe6d91 | ||
|
|
38d7075652 | ||
|
|
ed27ad0dea | ||
|
|
6b1b4285fa | ||
|
|
2131cfc7f9 | ||
|
|
d6d9af91e4 | ||
|
|
cbd97e8e3a | ||
|
|
e4b26dd27b | ||
|
|
4d6e2cdf59 | ||
|
|
d079f45ba3 | ||
|
|
d08f07d6fb | ||
|
|
af0796689f | ||
|
|
f408785409 | ||
|
|
eb6dc5189c | ||
|
|
cbdbe4b899 | ||
|
|
7870db9af1 |
@@ -34,6 +34,14 @@ UPSTREAM_API_KEY=your-upstream-api-key
|
||||
# LOG_LEVEL=INFO
|
||||
# ENABLE_CONSOLE_LOGGING=true
|
||||
|
||||
# SentryStr: mirror logs to Nostr relays (requires `pip install sentrystr`).
|
||||
# Announcements run on a background worker thread so they never block requests.
|
||||
# ENABLE_SENTRYSTR=false
|
||||
# SENTRYSTR_LOG_LEVEL=ERROR # defaults to ERROR; keep >= LOG_LEVEL
|
||||
# SENTRYSTR_RELAYS="wss://relay.damus.io,wss://nos.lol" # defaults to RELAYS
|
||||
# SENTRYSTR_NSEC=nsec1... # defaults to NSEC; ephemeral key if unset
|
||||
# SENTRYSTR_RECIPIENT_NPUB=npub1... # if set, log events are NIP-44 encrypted
|
||||
|
||||
# Custom Model Management
|
||||
# BASE_URL=https://openrouter.ai/api/v1
|
||||
# MODELS_PATH=models.json
|
||||
|
||||
@@ -23,6 +23,11 @@ dependencies = [
|
||||
"litellm>=1.55.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
# Decentralized log mirroring to Nostr relays. Enable at runtime with
|
||||
# ENABLE_SENTRYSTR=true (see routstr/core/logging.py).
|
||||
sentrystr = ["sentrystr>=0.2.0"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"mypy>=1.15.0",
|
||||
|
||||
+44
-52
@@ -1021,32 +1021,37 @@ async def adjust_payment_for_tokens(
|
||||
# actual cost exceeded discounted reservation (due to tolerance_percentage)
|
||||
if cost_difference > 0:
|
||||
# Always release the reservation and charge min(actual_cost, balance).
|
||||
# Using a CASE expression makes this a single atomic UPDATE — no
|
||||
# multi-level fallback needed and balance can never go negative.
|
||||
# CASE expressions keep this atomic and safe even when the
|
||||
# stale-reservation sweeper has already released the reservation.
|
||||
chargeable = case(
|
||||
(col(ApiKey.balance) >= total_cost_msats, total_cost_msats),
|
||||
else_=col(ApiKey.balance),
|
||||
)
|
||||
overrun_safe_reserved = case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
finalize_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
|
||||
.values(
|
||||
reserved_balance=col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
reserved_balance=overrun_safe_reserved,
|
||||
balance=col(ApiKey.balance) - chargeable,
|
||||
total_spent=col(ApiKey.total_spent) + chargeable,
|
||||
)
|
||||
)
|
||||
result = await session.exec(finalize_stmt) # type: ignore[call-overload]
|
||||
await session.exec(finalize_stmt) # type: ignore[call-overload]
|
||||
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
|
||||
.values(
|
||||
reserved_balance=col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
reserved_balance=overrun_safe_reserved,
|
||||
total_spent=col(ApiKey.total_spent) + min(billing_key.balance, total_cost_msats),
|
||||
)
|
||||
)
|
||||
@@ -1054,51 +1059,38 @@ async def adjust_payment_for_tokens(
|
||||
|
||||
await session.commit()
|
||||
|
||||
if result.rowcount:
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
cost.total_msats = total_cost_msats
|
||||
logger.info(
|
||||
"Finalized payment with additional charge",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"charged_amount": total_cost_msats,
|
||||
"new_balance": billing_key.balance,
|
||||
"model": model,
|
||||
},
|
||||
)
|
||||
await _accumulate_fee(total_cost_msats)
|
||||
payments_logger.info(
|
||||
"FINALIZE",
|
||||
extra={
|
||||
"event": "finalize",
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"model": model,
|
||||
"cost_reserved": deducted_max_cost,
|
||||
"cost_charged": total_cost_msats,
|
||||
"input_tokens": cost.input_tokens,
|
||||
"output_tokens": cost.output_tokens,
|
||||
"balance": billing_key.balance,
|
||||
"reserved_balance": billing_key.reserved_balance,
|
||||
"total_spent": billing_key.total_spent,
|
||||
"finalize_type": "overrun",
|
||||
},
|
||||
)
|
||||
else:
|
||||
# Guard fired: reservation was already released by a concurrent
|
||||
# finalization for this key. Nothing left to do.
|
||||
logger.warning(
|
||||
"Finalization skipped - reservation already released",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"attempted_charge": total_cost_msats,
|
||||
"model": model,
|
||||
},
|
||||
)
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
cost.total_msats = total_cost_msats
|
||||
logger.info(
|
||||
"Finalized payment with additional charge",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"charged_amount": total_cost_msats,
|
||||
"new_balance": billing_key.balance,
|
||||
"model": model,
|
||||
},
|
||||
)
|
||||
await _accumulate_fee(total_cost_msats)
|
||||
payments_logger.info(
|
||||
"FINALIZE",
|
||||
extra={
|
||||
"event": "finalize",
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"model": model,
|
||||
"cost_reserved": deducted_max_cost,
|
||||
"cost_charged": total_cost_msats,
|
||||
"input_tokens": cost.input_tokens,
|
||||
"output_tokens": cost.output_tokens,
|
||||
"balance": billing_key.balance,
|
||||
"reserved_balance": billing_key.reserved_balance,
|
||||
"total_spent": billing_key.total_spent,
|
||||
"finalize_type": "overrun",
|
||||
},
|
||||
)
|
||||
else:
|
||||
# Refund some of the base cost
|
||||
refund = abs(cost_difference)
|
||||
|
||||
@@ -7,11 +7,26 @@ logger = get_logger(__name__)
|
||||
|
||||
|
||||
class UpstreamError(Exception):
|
||||
"""Exception raised when an upstream provider fails."""
|
||||
"""Exception raised when an upstream provider fails.
|
||||
|
||||
def __init__(self, message: str, status_code: int = 502):
|
||||
``code`` carries a stable, machine-readable classification (e.g.
|
||||
``UPSTREAM_RATE_LIMIT``) so callers can distinguish failure kinds without
|
||||
string-matching the message. ``details`` holds optional structured,
|
||||
redaction-safe context. Both default to ``None`` for backwards
|
||||
compatibility.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
status_code: int = 502,
|
||||
code: str | None = None,
|
||||
details: dict[str, object] | None = None,
|
||||
):
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.details = details
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
|
||||
+193
-20
@@ -51,6 +51,8 @@ from pythonjsonlogger import jsonlogger
|
||||
from rich.console import Console
|
||||
from rich.logging import RichHandler
|
||||
|
||||
from .redaction import redact_obj, redact_org_ids
|
||||
|
||||
# Only use RichHandler when stdout is a real TTY. In non-TTY contexts
|
||||
# (docker logs, pipes, CI) Rich pads every line to width and wraps long
|
||||
# records, producing visually-empty trailing whitespace and split records.
|
||||
@@ -180,6 +182,37 @@ class RequestIdFilter(logging.Filter):
|
||||
return True
|
||||
|
||||
|
||||
# Standard ``LogRecord`` attributes that are never user-supplied ``extra``
|
||||
# fields; skipped when redacting structured extras (``msg``/``message`` are
|
||||
# handled separately above).
|
||||
_NON_EXTRA_RECORD_ATTRS = frozenset(
|
||||
{
|
||||
"name",
|
||||
"msg",
|
||||
"args",
|
||||
"levelname",
|
||||
"levelno",
|
||||
"pathname",
|
||||
"filename",
|
||||
"module",
|
||||
"exc_info",
|
||||
"exc_text",
|
||||
"stack_info",
|
||||
"lineno",
|
||||
"funcName",
|
||||
"created",
|
||||
"msecs",
|
||||
"relativeCreated",
|
||||
"thread",
|
||||
"threadName",
|
||||
"processName",
|
||||
"process",
|
||||
"taskName",
|
||||
"message",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class SecurityFilter(logging.Filter):
|
||||
"""Filter to remove sensitive information from logs."""
|
||||
|
||||
@@ -199,31 +232,57 @@ class SecurityFilter(logging.Filter):
|
||||
"refund_address",
|
||||
}
|
||||
|
||||
def _scrub(self, message: str) -> str:
|
||||
"""Redact secrets from a single string."""
|
||||
standalone_patterns = [
|
||||
r"Bearer\s+([a-zA-Z0-9_\-\.]{10,})", # Bearer token (must be 10 characters or more to reduce false-positives)
|
||||
r"cashu[A-Z]+([a-zA-Z0-9_\-\.=/+]+)", # Cashu tokens
|
||||
r"nsec[a-z0-9]+", # Nostr Public / Private Key
|
||||
]
|
||||
for pattern in standalone_patterns:
|
||||
message = re.sub(pattern, "[REDACTED]", message, flags=re.IGNORECASE)
|
||||
|
||||
for key in self.SENSITIVE_KEYS:
|
||||
if key in message.lower():
|
||||
key_patterns = [
|
||||
rf"{key}\s*[:=]\s*([a-zA-Z0-9_\-\.=/+]+)", # key:value or key=value (including any variant with spaces)
|
||||
rf'{key}\s*[:=]\s*["\']([^"\']+)["\']', # key:"value" or key='value' (including any variant with spaces)
|
||||
]
|
||||
for pattern in key_patterns:
|
||||
message = re.sub(
|
||||
pattern, f"{key}: [REDACTED]", message, flags=re.IGNORECASE
|
||||
)
|
||||
return message
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
"""Filter out sensitive information from log records."""
|
||||
try:
|
||||
message = record.getMessage()
|
||||
standalone_patterns = [
|
||||
r"Bearer\s+([a-zA-Z0-9_\-\.]{10,})", # Bearer token (must be 10 characters or more to reduce false-positives)
|
||||
r"cashu[A-Z]+([a-zA-Z0-9_\-\.=/+]+)", # Cashu tokens
|
||||
r"nsec[a-z0-9]+", # Nostr Public / Private Key
|
||||
]
|
||||
for pattern in standalone_patterns:
|
||||
message = re.sub(pattern, "[REDACTED]", message, flags=re.IGNORECASE)
|
||||
|
||||
for key in self.SENSITIVE_KEYS:
|
||||
if key in message.lower():
|
||||
key_patterns = [
|
||||
rf"{key}\s*[:=]\s*([a-zA-Z0-9_\-\.=/+]+)", # key:value or key=value (including any variant with spaces)
|
||||
rf'{key}\s*[:=]\s*["\']([^"\']+)["\']', # key:"value" or key='value' (including any variant with spaces)
|
||||
]
|
||||
for pattern in key_patterns:
|
||||
message = re.sub(
|
||||
pattern, f"{key}: [REDACTED]", message, flags=re.IGNORECASE
|
||||
)
|
||||
record.msg = message
|
||||
message = redact_org_ids(record.getMessage())
|
||||
record.msg = self._scrub(message)
|
||||
record.args = ()
|
||||
|
||||
# The exception traceback is appended by the *formatter* (e.g. when a
|
||||
# QueueHandler flattens the record before it is announced to the
|
||||
# public, permanent SentryStr relays), so the message scrub above
|
||||
# never sees it. Pre-render and redact it here and cache the result
|
||||
# in `exc_text` so every formatter reuses the scrubbed version.
|
||||
if record.exc_info and record.exc_info[0] is not None and not record.exc_text:
|
||||
record.exc_text = logging.Formatter().formatException(record.exc_info)
|
||||
if record.exc_text:
|
||||
record.exc_text = self._scrub(redact_org_ids(record.exc_text))
|
||||
if record.stack_info:
|
||||
record.stack_info = self._scrub(redact_org_ids(record.stack_info))
|
||||
|
||||
# Structured `extra={...}` fields are emitted by the JSON formatter
|
||||
# straight from the record dict and never pass through the message
|
||||
# formatting above. Redact organization IDs from any string-valued
|
||||
# extra so they cannot leak via structured logs.
|
||||
for attr, value in list(record.__dict__.items()):
|
||||
if attr in _NON_EXTRA_RECORD_ATTRS:
|
||||
continue
|
||||
if isinstance(value, (str, dict, list, tuple)):
|
||||
record.__dict__[attr] = redact_obj(value)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -408,6 +467,120 @@ def setup_logging() -> None:
|
||||
|
||||
logging.config.dictConfig(LOGGING_CONFIG)
|
||||
|
||||
_install_sentrystr_logging(LOGGING_CONFIG)
|
||||
|
||||
|
||||
# Handle for the background SentryStr logging worker (None when disabled).
|
||||
_sentrystr_handle: Any = None
|
||||
|
||||
# Bound on the in-memory queue feeding the background SentryStr worker. Records
|
||||
# beyond this are dropped rather than blocking the caller or growing memory
|
||||
# without limit when a relay is slow or unreachable.
|
||||
_SENTRYSTR_QUEUE_SIZE = 10_000
|
||||
|
||||
# Max seconds to wait for the queued backlog to flush on shutdown before
|
||||
# abandoning it, so shutdown cannot hang on a slow or unreachable relay.
|
||||
_SENTRYSTR_SHUTDOWN_TIMEOUT = 10.0
|
||||
|
||||
|
||||
def _install_sentrystr_logging(logging_config: dict[str, Any]) -> None:
|
||||
"""Mirror routstr application logs to Nostr relays via SentryStr.
|
||||
|
||||
Announcing to relays blocks, so SentryStr's installer runs the publish on a
|
||||
dedicated background worker thread fed by an in-memory queue; the request /
|
||||
event-loop threads only pay the cost of enqueueing a record. Disabled unless
|
||||
`ENABLE_SENTRYSTR` is set. Any failure here is swallowed so optional logging
|
||||
can never take down startup.
|
||||
"""
|
||||
global _sentrystr_handle
|
||||
|
||||
if _sentrystr_handle is not None:
|
||||
return
|
||||
|
||||
try:
|
||||
from .settings import settings
|
||||
except Exception:
|
||||
return
|
||||
|
||||
if not getattr(settings, "enable_sentrystr", False):
|
||||
return
|
||||
|
||||
routstr_logger = logging.getLogger("routstr")
|
||||
|
||||
relays = list(settings.sentrystr_relays or settings.relays or [])
|
||||
if not relays:
|
||||
routstr_logger.warning(
|
||||
"SentryStr logging is enabled but no relays are configured; set "
|
||||
"SENTRYSTR_RELAYS or RELAYS. Skipping SentryStr logging."
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
from sentrystr import install_sentrystr_logging
|
||||
except Exception:
|
||||
routstr_logger.warning(
|
||||
"SentryStr logging is enabled but the `sentrystr` package is not "
|
||||
"installed. Install it with `pip install sentrystr`. Skipping "
|
||||
"SentryStr logging."
|
||||
)
|
||||
return
|
||||
|
||||
# Default to ERROR: these events are announced to public, permanent relays,
|
||||
# so only genuine problems should be mirrored unless the operator opts into
|
||||
# something more verbose via SENTRYSTR_LOG_LEVEL. (Keep this at or above
|
||||
# LOG_LEVEL; a lower value is gated out by the routstr loggers' own level.)
|
||||
level = (settings.sentrystr_log_level or "ERROR").upper()
|
||||
|
||||
# routstr application loggers set propagate=False, so the handler must be
|
||||
# attached to each of them rather than relying on propagation to the root.
|
||||
routstr_loggers = [
|
||||
name
|
||||
for name in logging_config.get("loggers", {})
|
||||
if name == "routstr" or name.startswith("routstr.")
|
||||
] or ["routstr"]
|
||||
|
||||
try:
|
||||
_sentrystr_handle = install_sentrystr_logging(
|
||||
relays=relays,
|
||||
private_key=(settings.sentrystr_nsec or settings.nsec or None),
|
||||
level=level,
|
||||
loggers=routstr_loggers,
|
||||
recipient_pubkey=(settings.sentrystr_recipient_npub or None),
|
||||
platform="routstr",
|
||||
formatter=logging.Formatter("%(message)s"),
|
||||
# Run the same scrubbing/enrichment filters used by the other
|
||||
# handlers, on the calling thread, before records are handed to the
|
||||
# background worker. RequestIdFilter must run here because it reads a
|
||||
# ContextVar that is only set on the request thread; SecurityFilter
|
||||
# must run here so secrets are redacted before anything is announced
|
||||
# to the (public, permanent) relays. Passing them to the installer
|
||||
# attaches them before the handler goes live, so no record can slip
|
||||
# through unfiltered during startup.
|
||||
filters=[RequestIdFilter(), SecurityFilter()],
|
||||
# Bound the queue so a slow/unreachable relay can't grow memory.
|
||||
queue_size=_SENTRYSTR_QUEUE_SIZE,
|
||||
)
|
||||
except Exception as exc:
|
||||
routstr_logger.warning("Failed to initialize SentryStr logging: %s", exc)
|
||||
return
|
||||
|
||||
routstr_logger.info(
|
||||
"SentryStr logging enabled",
|
||||
extra={"sentrystr_level": level, "sentrystr_relay_count": len(relays)},
|
||||
)
|
||||
|
||||
|
||||
def shutdown_sentrystr_logging() -> None:
|
||||
"""Flush and stop the background SentryStr logging worker, if running."""
|
||||
global _sentrystr_handle
|
||||
handle = _sentrystr_handle
|
||||
_sentrystr_handle = None
|
||||
if handle is not None:
|
||||
try:
|
||||
handle.close(timeout=_SENTRYSTR_SHUTDOWN_TIMEOUT)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""Get a logger instance for the given module name."""
|
||||
|
||||
+11
-1
@@ -230,6 +230,13 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
)
|
||||
|
||||
# Flush and stop the background SentryStr logging worker (no-op when
|
||||
# SentryStr logging is disabled). Run off the event loop: the drain is
|
||||
# blocking (though bounded) and must not stall the loop during shutdown.
|
||||
from .logging import shutdown_sentrystr_logging
|
||||
|
||||
await asyncio.to_thread(shutdown_sentrystr_logging)
|
||||
|
||||
|
||||
class _ImmutableStaticFiles(StaticFiles):
|
||||
"""Static files with long Cache-Control for content-hashed Next.js assets.
|
||||
@@ -379,7 +386,10 @@ if UI_DIST_PATH.exists() and UI_DIST_PATH.is_dir():
|
||||
|
||||
else:
|
||||
logger.warning(
|
||||
f"UI dist directory not found at {UI_DIST_PATH}, skipping static file serving"
|
||||
"UI dist directory not found at %s; serving API only. Run `make ui-build` "
|
||||
"to build the static UI served from here, or `make ui-dev` for the Next.js "
|
||||
"dev server with hot reload on :3000 (it targets this backend on :8000).",
|
||||
UI_DIST_PATH,
|
||||
)
|
||||
|
||||
@app.get("/", include_in_schema=False)
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Redaction helpers for sensitive provider identifiers.
|
||||
|
||||
Single source of truth for stripping account-scoped identifiers (e.g. OpenAI
|
||||
organization IDs) from any text before it is logged, returned to a caller, or
|
||||
written to an audit entry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# OpenAI-style organization identifiers look like ``org-<base62>``. Require at
|
||||
# least 6 trailing chars so the already-redacted literal ``org-[REDACTED]`` is
|
||||
# never re-matched (``[`` is not in the character class).
|
||||
_ORG_ID_PATTERN = re.compile(r"\borg-[A-Za-z0-9]{6,}\b")
|
||||
|
||||
ORG_ID_PLACEHOLDER = "org-[REDACTED]"
|
||||
|
||||
|
||||
def redact_org_ids(text: str) -> str:
|
||||
"""Replace OpenAI-style organization IDs with ``org-[REDACTED]``.
|
||||
|
||||
Args:
|
||||
text: Arbitrary text that may embed an ``org-*`` identifier.
|
||||
|
||||
Returns:
|
||||
The text with every organization ID replaced. Non-string input is
|
||||
returned unchanged after coercion to ``str``.
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
return _ORG_ID_PATTERN.sub(ORG_ID_PLACEHOLDER, text)
|
||||
|
||||
|
||||
def redact_obj(obj: Any) -> Any:
|
||||
"""Recursively redact organization IDs in arbitrary nested structures.
|
||||
|
||||
Strings are redacted in place; dicts and lists/tuples are walked so that
|
||||
identifiers nested inside structured payloads (e.g. log ``extra`` fields or
|
||||
error ``details``) are also stripped. Other types are returned unchanged.
|
||||
"""
|
||||
if isinstance(obj, str):
|
||||
return redact_org_ids(obj)
|
||||
if isinstance(obj, dict):
|
||||
return {key: redact_obj(value) for key, value in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [redact_obj(value) for value in obj]
|
||||
if isinstance(obj, tuple):
|
||||
return tuple(redact_obj(value) for value in obj)
|
||||
return obj
|
||||
@@ -16,7 +16,12 @@ class Settings(BaseSettings):
|
||||
|
||||
@classmethod
|
||||
def parse_env_var(cls, field_name: str, raw_value: str) -> Any: # type: ignore[override]
|
||||
if field_name in {"cashu_mints", "cors_origins", "relays"}:
|
||||
if field_name in {
|
||||
"cashu_mints",
|
||||
"cors_origins",
|
||||
"relays",
|
||||
"sentrystr_relays",
|
||||
}:
|
||||
v = str(raw_value).strip()
|
||||
if v == "":
|
||||
return []
|
||||
@@ -103,6 +108,21 @@ class Settings(BaseSettings):
|
||||
log_level: str = Field(default="INFO", env="LOG_LEVEL")
|
||||
enable_console_logging: bool = Field(default=True, env="ENABLE_CONSOLE_LOGGING")
|
||||
|
||||
# SentryStr (decentralized logging to Nostr relays). When enabled, routstr
|
||||
# application logs at `sentrystr_log_level` and above are mirrored to Nostr
|
||||
# relays on a background worker thread (see routstr/core/logging.py).
|
||||
enable_sentrystr: bool = Field(default=False, env="ENABLE_SENTRYSTR")
|
||||
# Minimum level forwarded to relays. Empty -> defaults to ERROR. Should be
|
||||
# at or above `log_level`, since the routstr loggers gate lower records.
|
||||
sentrystr_log_level: str = Field(default="", env="SENTRYSTR_LOG_LEVEL")
|
||||
# Relays to announce to. Empty -> fall back to the discovery `relays`.
|
||||
sentrystr_relays: list[str] = Field(default_factory=list, env="SENTRYSTR_RELAYS")
|
||||
# Secret key (nsec or hex) used to sign log events. Empty -> fall back to
|
||||
# `nsec`; if that is also empty an ephemeral key is generated per process.
|
||||
sentrystr_nsec: str = Field(default="", env="SENTRYSTR_NSEC")
|
||||
# Optional recipient npub. When set, log events are NIP-44 encrypted to it.
|
||||
sentrystr_recipient_npub: str = Field(default="", env="SENTRYSTR_RECIPIENT_NPUB")
|
||||
|
||||
# Other
|
||||
chat_completions_api_version: str = Field(
|
||||
default="", env="CHAT_COMPLETIONS_API_VERSION"
|
||||
|
||||
@@ -11,6 +11,8 @@ from PIL import Image
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..core import get_logger
|
||||
from ..core.exceptions import UpstreamError
|
||||
from ..core.redaction import redact_org_ids
|
||||
from ..core.settings import settings
|
||||
from ..wallet import deserialize_token_from_string
|
||||
|
||||
@@ -418,16 +420,27 @@ def create_error_response(
|
||||
status_code: int,
|
||||
request: Request,
|
||||
token: str | None = None,
|
||||
code: str | int | None = None,
|
||||
details: dict[str, object] | None = None,
|
||||
) -> Response:
|
||||
"""Create a standardized error response."""
|
||||
"""Create a standardized error response.
|
||||
|
||||
``code`` is a stable, machine-readable classification (e.g.
|
||||
``UPSTREAM_RATE_LIMIT``); when omitted it defaults to the HTTP status code
|
||||
for backwards compatibility. ``details`` carries optional structured,
|
||||
redaction-safe context.
|
||||
"""
|
||||
error_obj: dict[str, object] = {
|
||||
"message": redact_org_ids(message),
|
||||
"type": error_type,
|
||||
"code": code if code is not None else status_code,
|
||||
}
|
||||
if details is not None:
|
||||
error_obj["details"] = details
|
||||
return Response(
|
||||
content=json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"message": message,
|
||||
"type": error_type,
|
||||
"code": status_code,
|
||||
},
|
||||
"error": error_obj,
|
||||
"request_id": getattr(request.state, "request_id", "unknown"),
|
||||
}
|
||||
),
|
||||
@@ -435,3 +448,20 @@ def create_error_response(
|
||||
media_type="application/json",
|
||||
headers={"X-Cashu": token} if token else {},
|
||||
)
|
||||
|
||||
|
||||
def create_upstream_error_response(
|
||||
error: UpstreamError,
|
||||
request: Request,
|
||||
fallback_status: int = 502,
|
||||
) -> Response:
|
||||
"""Build an error response from an :class:`UpstreamError`, preserving its
|
||||
structured ``code``, ``details``, and original ``status_code``."""
|
||||
return create_error_response(
|
||||
"upstream_error",
|
||||
str(error),
|
||||
error.status_code or fallback_status,
|
||||
request=request,
|
||||
code=getattr(error, "code", None),
|
||||
details=getattr(error, "details", None),
|
||||
)
|
||||
|
||||
+7
-13
@@ -24,6 +24,7 @@ from .payment.helpers import (
|
||||
calculate_discounted_max_cost,
|
||||
check_token_balance,
|
||||
create_error_response,
|
||||
create_upstream_error_response,
|
||||
get_max_cost_for_model,
|
||||
)
|
||||
from .payment.models import Model
|
||||
@@ -211,9 +212,7 @@ async def proxy(
|
||||
e,
|
||||
)
|
||||
if i == len(all_upstreams) - 1:
|
||||
last_error_response = create_error_response(
|
||||
"upstream_error", str(e), 502, request=request
|
||||
)
|
||||
last_error_response = create_upstream_error_response(e, request)
|
||||
continue
|
||||
return last_error_response or create_error_response(
|
||||
"upstream_error", "All upstreams failed", 502, request=request
|
||||
@@ -283,11 +282,10 @@ async def proxy(
|
||||
last_error = e
|
||||
continue
|
||||
|
||||
if last_error is not None:
|
||||
return create_upstream_error_response(last_error, request)
|
||||
return create_error_response(
|
||||
"upstream_error",
|
||||
str(last_error) if last_error else "All upstreams failed",
|
||||
502,
|
||||
request=request,
|
||||
"upstream_error", "All upstreams failed", 502, request=request
|
||||
)
|
||||
|
||||
elif auth := headers.get("authorization", None):
|
||||
@@ -343,9 +341,7 @@ async def proxy(
|
||||
except UpstreamError as e:
|
||||
logger.warning(f"Upstream {upstream.provider_type} failed (GET): {e}")
|
||||
if i == len(upstreams) - 1:
|
||||
last_error_response = create_error_response(
|
||||
"upstream_error", str(e), 502, request=request
|
||||
)
|
||||
last_error_response = create_upstream_error_response(e, request)
|
||||
continue
|
||||
return last_error_response or create_error_response(
|
||||
"upstream_error", "All upstreams failed", 502, request=request
|
||||
@@ -525,9 +521,7 @@ async def proxy(
|
||||
# If this was the last provider
|
||||
if i == len(upstreams) - 1:
|
||||
await revert_pay_for_request(key, session, max_cost_for_model)
|
||||
return create_error_response(
|
||||
"upstream_error", str(e), 502, request=request
|
||||
)
|
||||
return create_upstream_error_response(e, request)
|
||||
|
||||
# Otherwise loop continues to next provider
|
||||
continue
|
||||
|
||||
+85
-17
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import traceback
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Iterator
|
||||
@@ -23,6 +24,7 @@ from ..core.db import (
|
||||
store_cashu_transaction,
|
||||
)
|
||||
from ..core.exceptions import UpstreamError
|
||||
from ..core.redaction import redact_org_ids
|
||||
from ..payment.cost_calculation import (
|
||||
CostData,
|
||||
CostDataError,
|
||||
@@ -47,6 +49,7 @@ from .cache_breakpoints import (
|
||||
)
|
||||
from .count_tokens import count_tokens_locally
|
||||
from .litellm_routing import detect_litellm_prefix
|
||||
from .rate_limit import UPSTREAM_RATE_LIMIT, classify_rate_limit
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -582,7 +585,7 @@ class BaseUpstreamProvider:
|
||||
preview = body_bytes.decode("utf-8", errors="ignore").strip()
|
||||
if preview:
|
||||
message = preview[:500]
|
||||
return message, upstream_code
|
||||
return redact_org_ids(message), upstream_code
|
||||
|
||||
async def on_upstream_error_redirect(
|
||||
self, status_code: int, error_message: str
|
||||
@@ -625,10 +628,23 @@ class BaseUpstreamProvider:
|
||||
body_bytes = b""
|
||||
body_read_error = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
# ``message`` is already redacted by ``_extract_upstream_error_message``;
|
||||
# the raw body preview is redacted here before it reaches logs or the
|
||||
# forwarded envelope so provider account identifiers never leak.
|
||||
message, upstream_code = self._extract_upstream_error_message(body_bytes)
|
||||
body_preview = body_bytes.decode("utf-8", errors="ignore").strip()[:500]
|
||||
body_preview = redact_org_ids(
|
||||
body_bytes.decode("utf-8", errors="ignore").strip()[:500]
|
||||
)
|
||||
is_json_body = _is_json_content_type(content_type)
|
||||
|
||||
# Classify upstream rate-limit failures into a stable, structured error.
|
||||
rate_limit = classify_rate_limit(status_code, message, headers)
|
||||
error_code: str | int = upstream_code or status_code
|
||||
error_details: dict[str, object] | None = None
|
||||
if rate_limit is not None:
|
||||
error_code = UPSTREAM_RATE_LIMIT
|
||||
error_details = rate_limit.as_details()
|
||||
|
||||
logger.warning(
|
||||
"Upstream %s returned %s for model=%s path=%s: %s",
|
||||
self.provider_type,
|
||||
@@ -642,6 +658,7 @@ class BaseUpstreamProvider:
|
||||
"model": model_id or "unknown",
|
||||
"upstream_status": status_code,
|
||||
"upstream_code": upstream_code,
|
||||
"error_code": error_code,
|
||||
"upstream_content_type": content_type,
|
||||
"upstream_request_id": upstream_request_id,
|
||||
"message_preview": message[:200],
|
||||
@@ -676,13 +693,41 @@ class BaseUpstreamProvider:
|
||||
):
|
||||
headers.pop(header_name, None)
|
||||
|
||||
# Propagate a usable retry hint to the caller when the upstream supplied
|
||||
# one but did not echo a ``Retry-After`` header. RFC 7231 delta-seconds
|
||||
# is an integer, so round sub-second hints up to a usable ``1``.
|
||||
if (
|
||||
rate_limit is not None
|
||||
and rate_limit.retry_after_seconds is not None
|
||||
and "retry-after" not in {k.lower() for k in headers}
|
||||
):
|
||||
headers["Retry-After"] = str(max(1, math.ceil(rate_limit.retry_after_seconds)))
|
||||
|
||||
if is_json_body:
|
||||
if not content_type:
|
||||
headers.pop("content-type", None)
|
||||
headers.pop("Content-Type", None)
|
||||
media_type = content_type or None
|
||||
# Re-serialise the body with organization IDs stripped. The narrow
|
||||
# ``org-*`` regex preserves the surrounding JSON structure.
|
||||
redacted_text = redact_org_ids(body_bytes.decode("utf-8", errors="ignore"))
|
||||
redacted_body = redacted_text.encode()
|
||||
# Surface the stable rate-limit classification on the forwarded
|
||||
# body so callers can switch on ``error.code`` without parsing the
|
||||
# provider-specific message. Fall back to the redacted bytes if the
|
||||
# body is not a JSON object with an ``error`` mapping.
|
||||
if rate_limit is not None:
|
||||
try:
|
||||
parsed = json.loads(redacted_text)
|
||||
err = parsed.get("error") if isinstance(parsed, dict) else None
|
||||
if isinstance(err, dict):
|
||||
err["code"] = UPSTREAM_RATE_LIMIT
|
||||
err["details"] = error_details
|
||||
redacted_body = json.dumps(parsed).encode()
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
return Response(
|
||||
content=body_bytes,
|
||||
content=redacted_body,
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
media_type=media_type,
|
||||
@@ -693,15 +738,18 @@ class BaseUpstreamProvider:
|
||||
for header_name in ("content-type", "Content-Type"):
|
||||
headers.pop(header_name, None)
|
||||
|
||||
error_obj: dict[str, object] = {
|
||||
"message": message or "Upstream returned a non-JSON error response",
|
||||
"type": "upstream_error",
|
||||
"code": error_code,
|
||||
"upstream_status": status_code,
|
||||
"upstream_content_type": content_type or None,
|
||||
"upstream_body_preview": body_preview or None,
|
||||
}
|
||||
if error_details is not None:
|
||||
error_obj["details"] = error_details
|
||||
envelope = {
|
||||
"error": {
|
||||
"message": message or "Upstream returned a non-JSON error response",
|
||||
"type": "upstream_error",
|
||||
"code": upstream_code or status_code,
|
||||
"upstream_status": status_code,
|
||||
"upstream_content_type": content_type or None,
|
||||
"upstream_body_preview": body_preview or None,
|
||||
},
|
||||
"error": error_obj,
|
||||
"request_id": getattr(request.state, "request_id", None),
|
||||
}
|
||||
|
||||
@@ -2521,9 +2569,16 @@ class BaseUpstreamProvider:
|
||||
body_bytes = await response.aread()
|
||||
except Exception:
|
||||
body_bytes = b""
|
||||
body_preview = body_bytes.decode(
|
||||
"utf-8", errors="ignore"
|
||||
).strip()[:500]
|
||||
# Redact provider account identifiers before the body text
|
||||
# reaches logs or the raised error.
|
||||
body_preview = redact_org_ids(
|
||||
body_bytes.decode("utf-8", errors="ignore").strip()[:500]
|
||||
)
|
||||
rate_limit = classify_rate_limit(
|
||||
response.status_code,
|
||||
body_preview,
|
||||
dict(response.headers),
|
||||
)
|
||||
logger.error(
|
||||
"Upstream %s returned %s for model=%s path=%s: %s",
|
||||
self.provider_type,
|
||||
@@ -2535,6 +2590,7 @@ class BaseUpstreamProvider:
|
||||
"provider": self.provider_type,
|
||||
"model": original_model_id or "unknown",
|
||||
"status_code": response.status_code,
|
||||
"error_code": rate_limit.code if rate_limit else None,
|
||||
"reason_phrase": response.reason_phrase,
|
||||
"path": path,
|
||||
"body_preview": body_preview,
|
||||
@@ -2547,6 +2603,8 @@ class BaseUpstreamProvider:
|
||||
f"for model {original_model_id or 'unknown'}: "
|
||||
f"{body_preview[:200] or '<empty>'}",
|
||||
status_code=response.status_code,
|
||||
code=rate_limit.code if rate_limit else None,
|
||||
details=rate_limit.as_details() if rate_limit else None,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -2843,9 +2901,16 @@ class BaseUpstreamProvider:
|
||||
body_bytes = await response.aread()
|
||||
except Exception:
|
||||
body_bytes = b""
|
||||
body_preview = body_bytes.decode(
|
||||
"utf-8", errors="ignore"
|
||||
).strip()[:500]
|
||||
# Redact provider account identifiers before the body text
|
||||
# reaches logs or the raised error.
|
||||
body_preview = redact_org_ids(
|
||||
body_bytes.decode("utf-8", errors="ignore").strip()[:500]
|
||||
)
|
||||
rate_limit = classify_rate_limit(
|
||||
response.status_code,
|
||||
body_preview,
|
||||
dict(response.headers),
|
||||
)
|
||||
logger.error(
|
||||
"Upstream %s returned %s for model=%s path=%s: %s",
|
||||
self.provider_type,
|
||||
@@ -2857,6 +2922,7 @@ class BaseUpstreamProvider:
|
||||
"provider": self.provider_type,
|
||||
"model": original_model_id or "unknown",
|
||||
"status_code": response.status_code,
|
||||
"error_code": rate_limit.code if rate_limit else None,
|
||||
"path": path,
|
||||
"body_preview": body_preview,
|
||||
},
|
||||
@@ -2868,6 +2934,8 @@ class BaseUpstreamProvider:
|
||||
f"for model {original_model_id or 'unknown'}: "
|
||||
f"{body_preview[:200] or '<empty>'}",
|
||||
status_code=response.status_code,
|
||||
code=rate_limit.code if rate_limit else None,
|
||||
details=rate_limit.as_details() if rate_limit else None,
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
@@ -29,7 +29,9 @@ import litellm
|
||||
|
||||
from ..core import get_logger
|
||||
from ..core.exceptions import UpstreamError
|
||||
from ..core.redaction import redact_org_ids
|
||||
from ..payment.models import Model
|
||||
from .rate_limit import classify_rate_limit
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -505,23 +507,33 @@ async def dispatch_anthropic_messages(
|
||||
try:
|
||||
result = await litellm.anthropic.messages.acreate(**kwargs)
|
||||
except Exception as exc:
|
||||
exc_message = getattr(exc, "message", None) or str(exc) or repr(exc)
|
||||
raw_message = getattr(exc, "message", None) or str(exc) or repr(exc)
|
||||
# Redact provider account identifiers before the message reaches logs
|
||||
# or the surfaced error.
|
||||
exc_message = redact_org_ids(raw_message)
|
||||
exc_status = getattr(exc, "status_code", None)
|
||||
exc_response = getattr(exc, "response", None)
|
||||
response_text = None
|
||||
if exc_response is not None:
|
||||
try:
|
||||
response_text = getattr(exc_response, "text", str(exc_response))
|
||||
response_text = redact_org_ids(
|
||||
getattr(exc_response, "text", str(exc_response))
|
||||
)
|
||||
except Exception:
|
||||
response_text = "<unreadable>"
|
||||
status_for_classify = exc_status if isinstance(exc_status, int) else 502
|
||||
rate_limit = classify_rate_limit(
|
||||
status_for_classify, exc_message, getattr(exc, "headers", None)
|
||||
)
|
||||
logger.error(
|
||||
"litellm dispatch failed",
|
||||
extra={
|
||||
"error": exc_message,
|
||||
"error_type": type(exc).__name__,
|
||||
"status_code": exc_status,
|
||||
"error_code": rate_limit.code if rate_limit else None,
|
||||
"llm_provider": getattr(exc, "llm_provider", None),
|
||||
"body": getattr(exc, "body", None),
|
||||
"body": redact_org_ids(str(getattr(exc, "body", "") or "")) or None,
|
||||
"response_text": response_text,
|
||||
"model": litellm_model,
|
||||
"api_base": base_url,
|
||||
@@ -529,7 +541,9 @@ async def dispatch_anthropic_messages(
|
||||
)
|
||||
raise UpstreamError(
|
||||
f"Upstream error via litellm: {exc_message}",
|
||||
status_code=exc_status if isinstance(exc_status, int) else 502,
|
||||
status_code=status_for_classify,
|
||||
code=rate_limit.code if rate_limit else None,
|
||||
details=rate_limit.as_details() if rate_limit else None,
|
||||
) from exc
|
||||
|
||||
if not client_stream and hasattr(result, "__aiter__"):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import httpx
|
||||
@@ -18,6 +19,38 @@ class OpenRouterUpstreamProvider(BaseUpstreamProvider):
|
||||
supports_anthropic_messages = True
|
||||
litellm_provider_prefix = "openrouter/"
|
||||
|
||||
def prepare_request_body(
|
||||
self, body: bytes | None, model_obj: Model
|
||||
) -> bytes | None:
|
||||
"""Set provider.require_parameters on tool-use requests.
|
||||
|
||||
Without it OpenRouter can route a tool call to an endpoint that doesn't
|
||||
support function calling and 404 with "No endpoints found that support
|
||||
tool use". We leave a client-supplied value untouched.
|
||||
"""
|
||||
body = super().prepare_request_body(body, model_obj)
|
||||
if not body:
|
||||
return body
|
||||
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return body
|
||||
|
||||
if not isinstance(data, dict) or not data.get("tools"):
|
||||
return body
|
||||
|
||||
provider = data.get("provider")
|
||||
if not isinstance(provider, dict):
|
||||
provider = {}
|
||||
|
||||
if "require_parameters" in provider:
|
||||
return body
|
||||
|
||||
provider["require_parameters"] = True
|
||||
data["provider"] = provider
|
||||
return json.dumps(data).encode()
|
||||
|
||||
def _apply_provider_field(self, response_json: object) -> None:
|
||||
"""Stamp the ``provider`` field for OpenRouter responses.
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Detection and parsing of upstream provider rate-limit errors.
|
||||
|
||||
Upstream OpenAI-compatible providers signal rate limits via HTTP 429 and/or a
|
||||
human-readable message such as::
|
||||
|
||||
Rate limit reached for gpt-5.5-2026-04-23 (for limit gpt-5.5) in organization
|
||||
org-XXXX on tokens per min (TPM): Limit 180000000, Used 180000000,
|
||||
Requested 8929. Please try again in 2ms.
|
||||
|
||||
This module classifies those failures into a stable :data:`UPSTREAM_RATE_LIMIT`
|
||||
code and extracts useful debugging fields. All retained text is redacted of
|
||||
organization IDs first.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
from ..core.redaction import redact_org_ids
|
||||
|
||||
# Stable error code callers can switch on to distinguish upstream rate limits
|
||||
# from generic request failures. The literal value matches the identifier named
|
||||
# in issue #555 ("UPSTREAM_RATE_LIMIT") so the public API contract is exact.
|
||||
UPSTREAM_RATE_LIMIT = "UPSTREAM_RATE_LIMIT"
|
||||
|
||||
# Message fragments that indicate a rate-limit even when the status code is not
|
||||
# 429 (some providers wrap it in a 400/500 envelope).
|
||||
_RATE_LIMIT_MARKERS = (
|
||||
"rate limit reached",
|
||||
"rate_limit_exceeded",
|
||||
"rate limit exceeded",
|
||||
"too many requests",
|
||||
)
|
||||
|
||||
_MODEL_RE = re.compile(r"Rate limit reached for ([^\s(]+)", re.IGNORECASE)
|
||||
_LIMIT_NAME_RE = re.compile(r"\(for limit ([^)]+)\)", re.IGNORECASE)
|
||||
_METRIC_RE = re.compile(r"on ([a-z ]+\((?:TPM|RPM|TPD|RPD|IPM)\))", re.IGNORECASE)
|
||||
_LIMIT_RE = re.compile(r"Limit (\d+)", re.IGNORECASE)
|
||||
_USED_RE = re.compile(r"Used (\d+)", re.IGNORECASE)
|
||||
_REQUESTED_RE = re.compile(r"Requested (\d+)", re.IGNORECASE)
|
||||
_RETRY_RE = re.compile(r"try again in ([\d.]+)\s*(ms|s)", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RateLimitInfo:
|
||||
"""Structured, redaction-safe view of an upstream rate-limit error."""
|
||||
|
||||
code: str
|
||||
message: str
|
||||
model: str | None = None
|
||||
limit_name: str | None = None
|
||||
metric: str | None = None
|
||||
limit: int | None = None
|
||||
used: int | None = None
|
||||
requested: int | None = None
|
||||
retry_after_seconds: float | None = None
|
||||
|
||||
def as_details(self) -> dict[str, object]:
|
||||
"""Return a JSON-serialisable dict for embedding in an error envelope."""
|
||||
return {k: v for k, v in asdict(self).items() if v is not None}
|
||||
|
||||
|
||||
def _looks_like_rate_limit(status_code: int, message: str) -> bool:
|
||||
if status_code == 429:
|
||||
return True
|
||||
lowered = message.lower()
|
||||
return any(marker in lowered for marker in _RATE_LIMIT_MARKERS)
|
||||
|
||||
|
||||
def _parse_retry_after_header(headers: dict[str, str] | None) -> float | None:
|
||||
"""Parse a ``Retry-After`` header (delta-seconds form) into seconds."""
|
||||
if not headers:
|
||||
return None
|
||||
raw = headers.get("retry-after") or headers.get("Retry-After")
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return float(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _int_or_none(match: re.Match[str] | None) -> int | None:
|
||||
if match is None:
|
||||
return None
|
||||
try:
|
||||
return int(match.group(1))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def classify_rate_limit(
|
||||
status_code: int,
|
||||
message: str,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> RateLimitInfo | None:
|
||||
"""Classify an upstream error as a rate-limit and extract its fields.
|
||||
|
||||
Args:
|
||||
status_code: HTTP status code from the upstream response.
|
||||
message: Upstream error message (may contain sensitive identifiers).
|
||||
headers: Optional upstream response headers, used for ``Retry-After``.
|
||||
|
||||
Returns:
|
||||
A :class:`RateLimitInfo` when the error is a rate-limit, else ``None``.
|
||||
"""
|
||||
message = message or ""
|
||||
if not _looks_like_rate_limit(status_code, message):
|
||||
return None
|
||||
|
||||
redacted = redact_org_ids(message)
|
||||
|
||||
model_match = _MODEL_RE.search(redacted)
|
||||
metric_match = _METRIC_RE.search(redacted)
|
||||
|
||||
retry_after = _parse_retry_after_header(headers)
|
||||
if retry_after is None:
|
||||
retry_match = _RETRY_RE.search(redacted)
|
||||
if retry_match is not None:
|
||||
value = float(retry_match.group(1))
|
||||
retry_after = value / 1000.0 if retry_match.group(2).lower() == "ms" else value
|
||||
|
||||
limit_name_match = _LIMIT_NAME_RE.search(redacted)
|
||||
|
||||
return RateLimitInfo(
|
||||
code=UPSTREAM_RATE_LIMIT,
|
||||
message=redacted,
|
||||
model=model_match.group(1) if model_match else None,
|
||||
limit_name=limit_name_match.group(1).strip() if limit_name_match else None,
|
||||
metric=metric_match.group(1).strip() if metric_match else None,
|
||||
limit=_int_or_none(_LIMIT_RE.search(redacted)),
|
||||
used=_int_or_none(_USED_RE.search(redacted)),
|
||||
requested=_int_or_none(_REQUESTED_RE.search(redacted)),
|
||||
retry_after_seconds=retry_after,
|
||||
)
|
||||
+173
-65
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import re
|
||||
import time
|
||||
import typing
|
||||
from typing import TypedDict
|
||||
@@ -129,6 +130,72 @@ async def send_token(amount: int, unit: str, mint_url: str | None = None) -> str
|
||||
return token
|
||||
|
||||
|
||||
# A foreign mint's fee_reserve is a non-binding estimate (NUT-05): the mint may
|
||||
# demand more when re-quoting or at melt execution. Instead of padding the
|
||||
# estimate with a safety buffer (which strands the margin at the foreign mint
|
||||
# on every swap), the swap retries with the amount recomputed from the fees the
|
||||
# mint actually demands, up to this many attempts.
|
||||
_MAX_SWAP_ATTEMPTS = 3
|
||||
|
||||
_MINT_ERROR_CODE_RE = re.compile(r"\(Code: (\d+)\)")
|
||||
_MELT_SHORTFALL_RE = re.compile(r"Provided: (\d+), needed: (\d+)")
|
||||
|
||||
# Insufficient-melt-inputs failures differ across mint implementations. 11005 is
|
||||
# the registered "Transaction is not balanced" code (cdk), specific enough to
|
||||
# trust on the code alone. 11000 is nutshell's generic, unregistered
|
||||
# TransactionError covering many unrelated failures, so it only counts as a fee
|
||||
# shortfall alongside the "not enough inputs" detail text. With no code suffix at
|
||||
# all, that same text is the only signal.
|
||||
|
||||
|
||||
def _net_minted_amount(amount_msat: int, token_unit: str, fees: int) -> int:
|
||||
"""
|
||||
Convert the token value minus fees (given in the token unit) into an
|
||||
amount in the primary mint's unit.
|
||||
"""
|
||||
fee_msat = fees * 1000 if token_unit == "sat" else fees
|
||||
remaining_msat = amount_msat - fee_msat
|
||||
if settings.primary_mint_unit == "sat":
|
||||
return int(remaining_msat // 1000)
|
||||
return int(remaining_msat)
|
||||
|
||||
|
||||
def _melt_insufficient_shortfall(error: Exception) -> int | None:
|
||||
"""
|
||||
Classify a melt failure: return the observed shortfall (in the token unit)
|
||||
when the mint rejected the inputs as insufficient, or None when the failure
|
||||
is unrelated to fees and must not be retried (e.g. a Lightning payment
|
||||
failure, where a smaller invoice would not help).
|
||||
|
||||
Cashu errors carry no structured amounts (NUT-00 defines only detail/code,
|
||||
flattened to "Mint Error: <detail> (Code: <code>)" by cashu-py), so the
|
||||
classification uses the code and the shortfall must be inferred: the
|
||||
"Provided: X, needed: Y" amounts are nutshell-specific free text and only
|
||||
refine the shortfall when present; otherwise shrink one unit at a time.
|
||||
"""
|
||||
message = str(error)
|
||||
code_match = _MINT_ERROR_CODE_RE.search(message)
|
||||
code = code_match.group(1) if code_match is not None else None
|
||||
has_shortfall_text = "not enough inputs" in message.lower()
|
||||
|
||||
match code:
|
||||
case "11005": # registered TransactionUnbalanced: trust the code
|
||||
pass
|
||||
case "11000" if has_shortfall_text: # generic nutshell error: needs the text
|
||||
pass
|
||||
case None if has_shortfall_text: # no code suffix: text is the only signal
|
||||
pass
|
||||
case _: # other codes, a bare 11000, or no signal: must not retry
|
||||
return None
|
||||
|
||||
amounts = _MELT_SHORTFALL_RE.search(message)
|
||||
if amounts is not None:
|
||||
provided, needed = int(amounts.group(1)), int(amounts.group(2))
|
||||
if needed > provided:
|
||||
return needed - provided
|
||||
return 1
|
||||
|
||||
|
||||
async def _calculate_swap_amount(
|
||||
amount_msat: int,
|
||||
token_unit: str,
|
||||
@@ -167,26 +234,18 @@ async def _calculate_swap_amount(
|
||||
|
||||
fee_reserve = dummy_melt_quote.fee_reserve
|
||||
input_fees = token_wallet.get_fees_for_proofs(proofs)
|
||||
if token_unit == "sat":
|
||||
fee_msat = (fee_reserve + input_fees) * 1000
|
||||
else:
|
||||
fee_msat = fee_reserve + input_fees
|
||||
|
||||
amount_msat_after_fee = amount_msat - fee_msat
|
||||
|
||||
if settings.primary_mint_unit == "sat":
|
||||
minted_amount = int(amount_msat_after_fee // 1000)
|
||||
else:
|
||||
minted_amount = int(amount_msat_after_fee)
|
||||
total_fees = fee_reserve + input_fees
|
||||
minted_amount = _net_minted_amount(amount_msat, token_unit, total_fees)
|
||||
|
||||
if minted_amount <= 0:
|
||||
raise ValueError(f"Fees ({fee_reserve + input_fees} {token_unit}) exceed token amount")
|
||||
raise ValueError(f"Fees ({total_fees} {token_unit}) exceed token amount")
|
||||
|
||||
logger.info(
|
||||
"swap_to_primary_mint: fee estimation result",
|
||||
extra={
|
||||
"token_amount_sat": amount_msat // 1000,
|
||||
"estimated_fee_sat": fee_msat // 1000,
|
||||
"estimated_fee": total_fees,
|
||||
"estimated_fee_unit": token_unit,
|
||||
"input_fees": input_fees,
|
||||
"minted_amount": minted_amount,
|
||||
"minted_unit": settings.primary_mint_unit,
|
||||
@@ -251,67 +310,116 @@ async def swap_to_primary_mint(
|
||||
token_obj.proofs,
|
||||
)
|
||||
|
||||
mint_quote = await primary_wallet.request_mint(minted_amount)
|
||||
logger.info(
|
||||
"swap_to_primary_mint: mint quote received",
|
||||
extra={"mint_quote_id": mint_quote.quote},
|
||||
)
|
||||
# The estimate above is non-binding: the mint may demand a higher fee on the
|
||||
# real quote or reject the melt outright. Retry the quote/melt cycle with the
|
||||
# amount recomputed from the fees the mint actually demands.
|
||||
observed_extra_fee = 0
|
||||
attempt = 0
|
||||
while True:
|
||||
attempt += 1
|
||||
mint_quote = await primary_wallet.request_mint(minted_amount)
|
||||
logger.info(
|
||||
"swap_to_primary_mint: mint quote received",
|
||||
extra={"mint_quote_id": mint_quote.quote, "attempt": attempt},
|
||||
)
|
||||
|
||||
melt_quote = await token_wallet.melt_quote(mint_quote.request)
|
||||
input_fees = token_wallet.get_fees_for_proofs(token_obj.proofs)
|
||||
total_needed = melt_quote.amount + melt_quote.fee_reserve + input_fees
|
||||
logger.info(
|
||||
"swap_to_primary_mint: melt quote received",
|
||||
extra={
|
||||
"melt_quote_id": melt_quote.quote,
|
||||
"melt_amount": melt_quote.amount,
|
||||
"melt_fee_reserve": melt_quote.fee_reserve,
|
||||
"input_fees": input_fees,
|
||||
"total_needed": total_needed,
|
||||
"token_amount": token_amount,
|
||||
},
|
||||
)
|
||||
|
||||
if total_needed > token_amount:
|
||||
logger.warning(
|
||||
"swap_to_primary_mint: insufficient token amount for melt fees",
|
||||
melt_quote = await token_wallet.melt_quote(mint_quote.request)
|
||||
input_fees = token_wallet.get_fees_for_proofs(token_obj.proofs)
|
||||
total_needed = melt_quote.amount + melt_quote.fee_reserve + input_fees
|
||||
logger.info(
|
||||
"swap_to_primary_mint: melt quote received",
|
||||
extra={
|
||||
"token_amount": token_amount,
|
||||
"melt_quote_id": melt_quote.quote,
|
||||
"melt_amount": melt_quote.amount,
|
||||
"melt_fee_reserve": melt_quote.fee_reserve,
|
||||
"input_fees": input_fees,
|
||||
"total_needed": total_needed,
|
||||
"shortfall": total_needed - token_amount,
|
||||
"token_amount": token_amount,
|
||||
"attempt": attempt,
|
||||
},
|
||||
)
|
||||
raise ValueError(
|
||||
f"Token amount ({token_amount} {token_obj.unit}) is insufficient to cover "
|
||||
f"melt fees. Needed: {total_needed} {token_obj.unit} "
|
||||
f"(amount: {melt_quote.amount} + fee: {melt_quote.fee_reserve} + input_fees: {input_fees})"
|
||||
)
|
||||
|
||||
try:
|
||||
_ = await token_wallet.melt(
|
||||
proofs=token_obj.proofs,
|
||||
invoice=mint_quote.request,
|
||||
fee_reserve_sat=melt_quote.fee_reserve,
|
||||
quote_id=melt_quote.quote,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"swap_to_primary_mint: melt failed",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
"foreign_mint": token_obj.mint,
|
||||
"token_amount": token_amount,
|
||||
"melt_quote_id": melt_quote.quote,
|
||||
"total_needed": total_needed,
|
||||
},
|
||||
)
|
||||
raise ValueError(
|
||||
f"Failed to melt token from foreign mint {token_obj.mint}: {e}"
|
||||
) from e
|
||||
if total_needed > token_amount:
|
||||
recomputed = _net_minted_amount(
|
||||
amount_msat,
|
||||
token_obj.unit,
|
||||
melt_quote.fee_reserve + input_fees + observed_extra_fee,
|
||||
)
|
||||
if attempt >= _MAX_SWAP_ATTEMPTS or recomputed <= 0:
|
||||
logger.warning(
|
||||
"swap_to_primary_mint: insufficient token amount for melt fees",
|
||||
extra={
|
||||
"token_amount": token_amount,
|
||||
"melt_amount": melt_quote.amount,
|
||||
"melt_fee_reserve": melt_quote.fee_reserve,
|
||||
"input_fees": input_fees,
|
||||
"total_needed": total_needed,
|
||||
"shortfall": total_needed - token_amount,
|
||||
"attempts": attempt,
|
||||
},
|
||||
)
|
||||
raise ValueError(
|
||||
f"Token amount ({token_amount} {token_obj.unit}) is insufficient to cover "
|
||||
f"melt fees. Needed: {total_needed} {token_obj.unit} "
|
||||
f"(amount: {melt_quote.amount} + fee: {melt_quote.fee_reserve} + input_fees: {input_fees})"
|
||||
)
|
||||
logger.warning(
|
||||
"swap_to_primary_mint: melt quote exceeds token amount, retrying",
|
||||
extra={
|
||||
"total_needed": total_needed,
|
||||
"token_amount": token_amount,
|
||||
"retry_minted_amount": recomputed,
|
||||
"attempt": attempt,
|
||||
},
|
||||
)
|
||||
minted_amount = recomputed
|
||||
continue
|
||||
|
||||
try:
|
||||
_ = await token_wallet.melt(
|
||||
proofs=token_obj.proofs,
|
||||
invoice=mint_quote.request,
|
||||
fee_reserve_sat=melt_quote.fee_reserve,
|
||||
quote_id=melt_quote.quote,
|
||||
)
|
||||
except Exception as e:
|
||||
shortfall = _melt_insufficient_shortfall(e)
|
||||
recomputed = 0
|
||||
if shortfall is not None:
|
||||
observed_extra_fee += shortfall
|
||||
recomputed = _net_minted_amount(
|
||||
amount_msat,
|
||||
token_obj.unit,
|
||||
melt_quote.fee_reserve + input_fees + observed_extra_fee,
|
||||
)
|
||||
if shortfall is None or attempt >= _MAX_SWAP_ATTEMPTS or recomputed <= 0:
|
||||
logger.error(
|
||||
"swap_to_primary_mint: melt failed",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
"foreign_mint": token_obj.mint,
|
||||
"token_amount": token_amount,
|
||||
"melt_quote_id": melt_quote.quote,
|
||||
"total_needed": total_needed,
|
||||
"attempts": attempt,
|
||||
},
|
||||
)
|
||||
raise ValueError(
|
||||
f"Failed to melt token from foreign mint {token_obj.mint}: {e}"
|
||||
) from e
|
||||
logger.warning(
|
||||
"swap_to_primary_mint: mint demanded more than quoted at melt, retrying",
|
||||
extra={
|
||||
"shortfall": shortfall,
|
||||
"retry_minted_amount": recomputed,
|
||||
"attempt": attempt,
|
||||
},
|
||||
)
|
||||
minted_amount = recomputed
|
||||
continue
|
||||
|
||||
break
|
||||
|
||||
logger.info(
|
||||
"swap_to_primary_mint: melt succeeded, minting on primary",
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Regression tests for charging after stale reservation cleanup."""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.payment.cost_calculation import CostData
|
||||
|
||||
|
||||
def _make_key(balance: int, reserved: int) -> ApiKey:
|
||||
return ApiKey(
|
||||
hashed_key=f"test_{uuid.uuid4().hex}",
|
||||
balance=balance,
|
||||
reserved_balance=reserved,
|
||||
total_spent=0,
|
||||
total_requests=1,
|
||||
)
|
||||
|
||||
|
||||
def _cost_data(total_msats: int) -> CostData:
|
||||
return CostData(
|
||||
base_msats=0,
|
||||
input_msats=total_msats // 2,
|
||||
output_msats=total_msats - total_msats // 2,
|
||||
total_msats=total_msats,
|
||||
total_usd=0.0,
|
||||
input_tokens=100,
|
||||
output_tokens=100,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overrun_charges_after_reservation_swept(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Overrun finalize must charge even when the reservation was already released."""
|
||||
from routstr.auth import adjust_payment_for_tokens
|
||||
|
||||
deducted_max_cost = 990 # discounted reservation
|
||||
actual_token_cost = 1000 # actual cost overruns the reservation
|
||||
|
||||
# Sweeper has zeroed reserved_balance but left balance untouched.
|
||||
key = _make_key(balance=1000, reserved=0)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
|
||||
response_data = {
|
||||
"model": "test-model",
|
||||
"usage": {"prompt_tokens": 100, "completion_tokens": 100},
|
||||
}
|
||||
|
||||
with patch(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, integration_session, deducted_max_cost
|
||||
)
|
||||
|
||||
await integration_session.refresh(key)
|
||||
|
||||
assert key.total_spent == actual_token_cost, (
|
||||
f"Request was not billed (total_spent={key.total_spent}) — free response bug"
|
||||
)
|
||||
assert key.balance == 1000 - actual_token_cost, (
|
||||
f"Balance not charged: {key.balance}"
|
||||
)
|
||||
assert key.balance >= 0
|
||||
assert key.reserved_balance == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_free_response_path_closed_end_to_end(
|
||||
integration_session: AsyncSession,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
"""A reservation released by the real sweeper must not yield a free response."""
|
||||
from routstr.auth import adjust_payment_for_tokens, pay_for_request
|
||||
from routstr.core.db import create_session, release_stale_reservations
|
||||
|
||||
deducted_max_cost = 990
|
||||
actual_token_cost = 1000
|
||||
key_hash = f"test_sweep_{uuid.uuid4().hex}"
|
||||
|
||||
async with create_session() as session:
|
||||
session.add(
|
||||
ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=1000,
|
||||
reserved_balance=0,
|
||||
total_spent=0,
|
||||
total_requests=0,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
# Reserve the request, then backdate reserved_at so the sweeper treats it as
|
||||
# stale (simulates a stream that outlived stale_reservation_timeout_seconds).
|
||||
async with create_session() as session:
|
||||
key = await session.get(ApiKey, key_hash)
|
||||
assert key is not None
|
||||
await pay_for_request(key, deducted_max_cost, session)
|
||||
await session.refresh(key)
|
||||
assert key.reserved_balance == deducted_max_cost
|
||||
key.reserved_at = int(time.time()) - 10_000
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
|
||||
# Sweeper releases the stale reservation without charging.
|
||||
async with create_session() as session:
|
||||
released = await release_stale_reservations(session, max_age_seconds=300)
|
||||
assert released == 1
|
||||
|
||||
async with create_session() as session:
|
||||
key = await session.get(ApiKey, key_hash)
|
||||
assert key is not None
|
||||
assert key.reserved_balance == 0, "Precondition: sweeper zeroed the reservation"
|
||||
|
||||
response_data = {
|
||||
"model": "test-model",
|
||||
"usage": {"prompt_tokens": 100, "completion_tokens": 100},
|
||||
}
|
||||
with patch(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, session, deducted_max_cost
|
||||
)
|
||||
|
||||
async with create_session() as session:
|
||||
final = await session.get(ApiKey, key_hash)
|
||||
assert final is not None
|
||||
|
||||
assert final.total_spent == actual_token_cost, (
|
||||
f"Free response: total_spent={final.total_spent}, expected {actual_token_cost}"
|
||||
)
|
||||
assert final.balance == 1000 - actual_token_cost, (
|
||||
f"Balance not charged after sweep: {final.balance}"
|
||||
)
|
||||
assert final.balance >= 0
|
||||
assert final.reserved_balance == 0
|
||||
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
Integration tests for reactive swap fee retries via the wallet topup endpoint.
|
||||
|
||||
Foreign-mint tokens are swapped to the primary mint using the foreign mint's
|
||||
melt quote, whose fee_reserve is a non-binding estimate (NUT-05): the mint may
|
||||
demand more when re-quoting or at melt execution. These tests cover the
|
||||
endpoint behaviour in those cases:
|
||||
|
||||
1. The mint demands one sat more at melt time than every quote reported
|
||||
(the mint.cubabitcoin.org incident): the swap retries with a smaller
|
||||
invoice and the topup succeeds, crediting the recomputed amount.
|
||||
2. The real melt quote reports a higher fee_reserve than the estimate: the
|
||||
swap re-quotes from the observed fee and the topup succeeds.
|
||||
3. The mint escalates its fee demands on every attempt: the retry budget is
|
||||
exhausted and the endpoint returns 400 with a clear error (never 500),
|
||||
without ever executing a melt.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient, Response
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
# Captured at collection time, before the integration_app fixture replaces it
|
||||
# with the testmint stub that bypasses swapping (see conftest.py).
|
||||
from routstr.wallet import recieve_token as _real_recieve_token
|
||||
|
||||
PRIMARY_MINT = "http://primary:3338"
|
||||
|
||||
|
||||
def _make_swap_mocks(
|
||||
token_amount: int,
|
||||
fee_reserves: list[int],
|
||||
input_fees: int = 0,
|
||||
mint_url: str = "http://foreign-mint:3338",
|
||||
) -> tuple[Mock, Mock, Mock]:
|
||||
"""Return (token, token_wallet, primary_wallet) mocks that act like a mint.
|
||||
|
||||
Mint quotes pass the requested amount through their ``request`` field and
|
||||
melt quotes echo that amount back, so the mocks stay consistent for
|
||||
whatever amounts the implementation requests. ``fee_reserves`` supplies the
|
||||
fee_reserve of each successive melt quote (the first serves the estimation
|
||||
pass); requesting more quotes than provided fails the test.
|
||||
"""
|
||||
mock_token = Mock()
|
||||
mock_token.mint = mint_url
|
||||
mock_token.unit = "sat"
|
||||
mock_token.amount = token_amount
|
||||
mock_token.keysets = ["keyset1"]
|
||||
mock_token.proofs = [Mock(amount=token_amount)]
|
||||
|
||||
mock_token_wallet = Mock()
|
||||
mock_token_wallet.load_mint = AsyncMock()
|
||||
mock_token_wallet.load_proofs = AsyncMock()
|
||||
mock_token_wallet.get_fees_for_proofs = Mock(return_value=input_fees)
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.load_mint = AsyncMock()
|
||||
mock_primary_wallet.load_proofs = AsyncMock()
|
||||
mock_primary_wallet.available_balance = Mock(amount=0)
|
||||
mock_primary_wallet.mint = AsyncMock(return_value=Mock())
|
||||
|
||||
fees = iter(fee_reserves)
|
||||
|
||||
def _next_fee() -> int:
|
||||
try:
|
||||
return next(fees)
|
||||
except StopIteration:
|
||||
raise AssertionError(
|
||||
"more melt quotes requested than fee_reserves provided"
|
||||
) from None
|
||||
|
||||
mock_primary_wallet.request_mint = AsyncMock(
|
||||
side_effect=lambda amount: Mock(quote=f"mint_quote_{amount}", request=amount)
|
||||
)
|
||||
mock_token_wallet.melt_quote = AsyncMock(
|
||||
side_effect=lambda invoice: Mock(
|
||||
quote=f"melt_quote_{invoice}", amount=invoice, fee_reserve=_next_fee()
|
||||
)
|
||||
)
|
||||
mock_token_wallet.melt = AsyncMock(return_value=Mock())
|
||||
|
||||
return mock_token, mock_token_wallet, mock_primary_wallet
|
||||
|
||||
|
||||
def _wallet_router(primary_wallet: Mock, token_wallet: Mock) -> Callable[..., Mock]:
|
||||
"""Route get_wallet calls to the primary or foreign wallet mock by URL."""
|
||||
|
||||
def fake_get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Mock:
|
||||
return primary_wallet if mint_url == PRIMARY_MINT else token_wallet
|
||||
|
||||
return fake_get_wallet
|
||||
|
||||
|
||||
async def _post_topup(
|
||||
client: AsyncClient,
|
||||
mock_token: Mock,
|
||||
token_wallet: Mock,
|
||||
primary_wallet: Mock,
|
||||
) -> Response:
|
||||
"""POST /v1/wallet/topup with the swap layer mocked at the mint boundary.
|
||||
|
||||
The conftest's testmint stub for recieve_token is swapped back for the
|
||||
real implementation so the request exercises the actual swap path.
|
||||
"""
|
||||
with patch("routstr.wallet.recieve_token", _real_recieve_token):
|
||||
with patch(
|
||||
"routstr.wallet.deserialize_token_from_string", return_value=mock_token
|
||||
):
|
||||
with patch(
|
||||
"routstr.wallet.get_wallet",
|
||||
side_effect=_wallet_router(primary_wallet, token_wallet),
|
||||
):
|
||||
with patch.object(settings, "primary_mint", PRIMARY_MINT):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch.object(settings, "cashu_mints", [PRIMARY_MINT]):
|
||||
return await client.post(
|
||||
"/v1/wallet/topup",
|
||||
params={"cashu_token": "cashuAtest_foreign_token"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_retries_when_melt_demands_more_than_quoted(
|
||||
authenticated_client: AsyncClient,
|
||||
) -> None:
|
||||
"""A 179-sat token where every quote reports fee_reserve=1 but the mint
|
||||
rejects the first melt demanding 180. The retry shrinks the invoice to 177
|
||||
and the topup credits 177 sats (177_000 msats)."""
|
||||
mock_token, token_wallet, primary_wallet = _make_swap_mocks(
|
||||
179, fee_reserves=[1, 1, 1], mint_url="http://mint.cubabitcoin.org"
|
||||
)
|
||||
token_wallet.melt.side_effect = [
|
||||
Exception(
|
||||
"Mint Error: not enough inputs provided for melt. "
|
||||
"Provided: 179, needed: 180 (Code: 11000)"
|
||||
),
|
||||
Mock(),
|
||||
]
|
||||
|
||||
response = await _post_topup(
|
||||
authenticated_client, mock_token, token_wallet, primary_wallet
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["msats"] == 177_000
|
||||
assert token_wallet.melt.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_retries_when_quote_fee_exceeds_estimate(
|
||||
authenticated_client: AsyncClient,
|
||||
) -> None:
|
||||
"""A 1000-sat token estimated at fee 20, but the real quote demands 23.
|
||||
The retry recomputes 1000 - 23 = 977, which fits, and the topup credits
|
||||
977 sats (977_000 msats) with a single melt."""
|
||||
mock_token, token_wallet, primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[20, 23, 23]
|
||||
)
|
||||
|
||||
response = await _post_topup(
|
||||
authenticated_client, mock_token, token_wallet, primary_wallet
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["msats"] == 977_000
|
||||
assert token_wallet.melt.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_returns_400_when_retries_exhausted(
|
||||
authenticated_client: AsyncClient,
|
||||
) -> None:
|
||||
"""A mint that escalates fee_reserve on every re-quote (1 → 10 → 25 → 50)
|
||||
exhausts the retry budget: clean 400 with an actionable message, melt never
|
||||
executed."""
|
||||
mock_token, token_wallet, primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[1, 10, 25, 50]
|
||||
)
|
||||
|
||||
response = await _post_topup(
|
||||
authenticated_client, mock_token, token_wallet, primary_wallet
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "too small to cover swap fees" in response.json()["detail"]
|
||||
assert token_wallet.melt_quote.call_count == 4 # estimation + 3 attempts
|
||||
token_wallet.melt.assert_not_called()
|
||||
@@ -0,0 +1,95 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
|
||||
os.environ.setdefault("UPSTREAM_API_KEY", "test")
|
||||
os.environ.setdefault("LIGHTNING_ADDRESS", "test@stm.to")
|
||||
|
||||
from routstr.upstream import GenericUpstreamProvider
|
||||
from routstr.upstream.openrouter import OpenRouterUpstreamProvider
|
||||
|
||||
|
||||
def _model(model_id: str = "openai/gpt-4o"): # type: ignore[no-untyped-def]
|
||||
from routstr.payment.models import Architecture, Model, Pricing
|
||||
|
||||
return Model(
|
||||
id=model_id,
|
||||
name=model_id,
|
||||
created=0,
|
||||
description="",
|
||||
context_length=128000,
|
||||
architecture=Architecture(
|
||||
modality="text->text",
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"],
|
||||
tokenizer="GPT",
|
||||
instruct_type=None,
|
||||
),
|
||||
pricing=Pricing(prompt=0.0, completion=0.0),
|
||||
)
|
||||
|
||||
|
||||
def _tool_body() -> dict:
|
||||
return {
|
||||
"model": "openai/gpt-4o",
|
||||
"messages": [{"role": "user", "content": "What's the weather?"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {}},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _prepare(provider, body: dict) -> dict: # type: ignore[no-untyped-def]
|
||||
out = provider.prepare_request_body(json.dumps(body).encode(), _model())
|
||||
assert out is not None
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def test_injects_require_parameters_for_tool_request() -> None:
|
||||
data = _prepare(OpenRouterUpstreamProvider(api_key="test"), _tool_body())
|
||||
assert data["provider"]["require_parameters"] is True
|
||||
|
||||
|
||||
def test_generic_provider_on_openrouter_url_is_left_alone() -> None:
|
||||
# Only OpenRouterUpstreamProvider injects; a generic provider pointed at the
|
||||
# same base URL doesn't.
|
||||
provider = GenericUpstreamProvider(base_url="https://openrouter.ai/api/v1")
|
||||
data = _prepare(provider, _tool_body())
|
||||
assert "provider" not in data
|
||||
|
||||
|
||||
def test_no_injection_without_tools() -> None:
|
||||
body = {"model": "openai/gpt-4o", "messages": [{"role": "user", "content": "hi"}]}
|
||||
data = _prepare(OpenRouterUpstreamProvider(api_key="test"), body)
|
||||
assert "provider" not in data
|
||||
|
||||
|
||||
def test_empty_tools_list_does_not_inject() -> None:
|
||||
body = _tool_body()
|
||||
body["tools"] = []
|
||||
data = _prepare(OpenRouterUpstreamProvider(api_key="test"), body)
|
||||
assert "provider" not in data
|
||||
|
||||
|
||||
def test_direct_provider_does_not_inject() -> None:
|
||||
provider = GenericUpstreamProvider(base_url="https://api.openai.com/v1")
|
||||
data = _prepare(provider, _tool_body())
|
||||
assert "provider" not in data
|
||||
|
||||
|
||||
def test_keeps_client_set_require_parameters() -> None:
|
||||
body = _tool_body()
|
||||
body["provider"] = {"require_parameters": False}
|
||||
data = _prepare(OpenRouterUpstreamProvider(api_key="test"), body)
|
||||
assert data["provider"]["require_parameters"] is False
|
||||
|
||||
|
||||
def test_preserves_other_provider_fields() -> None:
|
||||
body = _tool_body()
|
||||
body["provider"] = {"order": ["openai", "azure"]}
|
||||
data = _prepare(OpenRouterUpstreamProvider(api_key="test"), body)
|
||||
assert data["provider"]["order"] == ["openai", "azure"]
|
||||
assert data["provider"]["require_parameters"] is True
|
||||
@@ -0,0 +1,396 @@
|
||||
"""Tests for upstream rate-limit detection, classification, and org-ID redaction.
|
||||
|
||||
Covers issue #555: upstream OpenAI-compatible providers return rate-limit
|
||||
errors that embed a sensitive organization ID. The proxy must classify these
|
||||
distinctly (``UPSTREAM_RATE_LIMIT``), preserve useful debugging fields, and
|
||||
never emit a raw ``org-*`` identifier in logs, errors, or returned bodies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from routstr.core.redaction import redact_org_ids
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
from routstr.upstream.rate_limit import (
|
||||
UPSTREAM_RATE_LIMIT,
|
||||
RateLimitInfo,
|
||||
classify_rate_limit,
|
||||
)
|
||||
|
||||
# The exact scenario from the issue, with a realistic (fake) org identifier.
|
||||
RAW_ORG_ID = "org-abc123XYZ456def"
|
||||
RATE_LIMIT_MESSAGE = (
|
||||
f"Rate limit reached for gpt-5.5-2026-04-23 (for limit gpt-5.5) in "
|
||||
f"organization {RAW_ORG_ID} on tokens per min (TPM): Limit 180000000, "
|
||||
f"Used 180000000, Requested 8929. Please try again in 2ms. Visit "
|
||||
f"https://platform.openai.com/account/rate-limits to learn more."
|
||||
)
|
||||
|
||||
|
||||
def _make_request(request_id: str = "req-123") -> Mock:
|
||||
request = Mock(spec=["method", "state"])
|
||||
request.method = "POST"
|
||||
request.state = Mock()
|
||||
request.state.request_id = request_id
|
||||
return request
|
||||
|
||||
|
||||
def _make_upstream_response(
|
||||
*,
|
||||
body: bytes,
|
||||
status_code: int = 429,
|
||||
content_type: str | None = "application/json",
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
) -> httpx.Response:
|
||||
headers: dict[str, str] = {}
|
||||
if content_type is not None:
|
||||
headers["content-type"] = content_type
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
return httpx.Response(status_code=status_code, headers=headers, content=body)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provider() -> BaseUpstreamProvider:
|
||||
return BaseUpstreamProvider(
|
||||
base_url="https://privateprovider.xyz", api_key="k", provider_fee=1.0
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Redaction
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_redact_org_ids_replaces_identifier() -> None:
|
||||
assert RAW_ORG_ID not in redact_org_ids(RATE_LIMIT_MESSAGE)
|
||||
assert "org-[REDACTED]" in redact_org_ids(RATE_LIMIT_MESSAGE)
|
||||
|
||||
|
||||
def test_redact_org_ids_is_idempotent() -> None:
|
||||
once = redact_org_ids(RATE_LIMIT_MESSAGE)
|
||||
assert redact_org_ids(once) == once
|
||||
|
||||
|
||||
def test_redact_org_ids_leaves_unrelated_text() -> None:
|
||||
assert redact_org_ids("organize the org-chart") == "organize the org-chart"
|
||||
assert redact_org_ids("") == ""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Classification
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_classify_exact_scenario() -> None:
|
||||
info = classify_rate_limit(429, RATE_LIMIT_MESSAGE)
|
||||
assert isinstance(info, RateLimitInfo)
|
||||
assert info.code == UPSTREAM_RATE_LIMIT
|
||||
assert info.model == "gpt-5.5-2026-04-23"
|
||||
assert info.limit_name == "gpt-5.5"
|
||||
assert info.metric == "tokens per min (TPM)"
|
||||
assert info.limit == 180000000
|
||||
assert info.used == 180000000
|
||||
assert info.requested == 8929
|
||||
assert info.retry_after_seconds == pytest.approx(0.002)
|
||||
# Redaction-safe: no raw org id survives into the structured view.
|
||||
assert RAW_ORG_ID not in info.message
|
||||
assert RAW_ORG_ID not in json.dumps(info.as_details())
|
||||
|
||||
|
||||
def test_classify_by_status_code_without_marker() -> None:
|
||||
info = classify_rate_limit(429, "slow down")
|
||||
assert info is not None
|
||||
assert info.code == UPSTREAM_RATE_LIMIT
|
||||
|
||||
|
||||
def test_classify_by_message_marker_without_429() -> None:
|
||||
info = classify_rate_limit(400, "rate_limit_exceeded for this key")
|
||||
assert info is not None
|
||||
|
||||
|
||||
def test_retry_after_header_takes_precedence() -> None:
|
||||
info = classify_rate_limit(429, RATE_LIMIT_MESSAGE, {"Retry-After": "12"})
|
||||
assert info is not None
|
||||
assert info.retry_after_seconds == pytest.approx(12.0)
|
||||
|
||||
|
||||
def test_non_rate_limit_error_is_not_classified() -> None:
|
||||
assert classify_rate_limit(400, "invalid request: missing field 'model'") is None
|
||||
assert classify_rate_limit(500, "internal server error") is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# forward_upstream_error_response integration
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_json_rate_limit_body_is_redacted_and_forwarded(
|
||||
provider: BaseUpstreamProvider,
|
||||
) -> None:
|
||||
body = json.dumps(
|
||||
{"error": {"message": RATE_LIMIT_MESSAGE, "type": "rate_limit_exceeded"}}
|
||||
).encode()
|
||||
upstream = _make_upstream_response(body=body, status_code=429)
|
||||
|
||||
response = await provider.forward_upstream_error_response(
|
||||
_make_request(), "v1/chat/completions", upstream
|
||||
)
|
||||
|
||||
assert response.status_code == 429
|
||||
raw = bytes(response.body).decode()
|
||||
# No raw organization id may survive in the forwarded body.
|
||||
assert RAW_ORG_ID not in raw
|
||||
assert "org-[REDACTED]" in raw
|
||||
# Body remains valid JSON; the original type is preserved while a stable
|
||||
# rate-limit code is injected so callers can switch on it.
|
||||
payload: dict[str, Any] = json.loads(raw)
|
||||
assert payload["error"]["type"] == "rate_limit_exceeded"
|
||||
assert payload["error"]["code"] == UPSTREAM_RATE_LIMIT
|
||||
assert payload["error"]["details"]["model"] == "gpt-5.5-2026-04-23"
|
||||
# A retry hint extracted from the message is surfaced as a header.
|
||||
assert "retry-after" in {k.lower() for k in response.headers}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_json_rate_limit_envelope_uses_stable_code(
|
||||
provider: BaseUpstreamProvider,
|
||||
) -> None:
|
||||
upstream = _make_upstream_response(
|
||||
body=RATE_LIMIT_MESSAGE.encode(),
|
||||
status_code=429,
|
||||
content_type="text/plain",
|
||||
)
|
||||
|
||||
response = await provider.forward_upstream_error_response(
|
||||
_make_request(), "v1/chat/completions", upstream
|
||||
)
|
||||
|
||||
payload: dict[str, Any] = json.loads(bytes(response.body))
|
||||
assert payload["error"]["code"] == UPSTREAM_RATE_LIMIT
|
||||
assert payload["error"]["details"]["model"] == "gpt-5.5-2026-04-23"
|
||||
serialized = json.dumps(payload)
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_rate_limit_json_error_unchanged(
|
||||
provider: BaseUpstreamProvider,
|
||||
) -> None:
|
||||
body = json.dumps(
|
||||
{"error": {"message": "missing field 'model'", "type": "invalid_request"}}
|
||||
).encode()
|
||||
upstream = _make_upstream_response(body=body, status_code=400)
|
||||
|
||||
response = await provider.forward_upstream_error_response(
|
||||
_make_request(), "v1/chat/completions", upstream
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
payload: dict[str, Any] = json.loads(bytes(response.body))
|
||||
assert payload["error"]["type"] == "invalid_request"
|
||||
assert "retry-after" not in {k.lower() for k in response.headers}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# UpstreamError -> proxy response (preserves code/details/status)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_create_upstream_error_response_preserves_structure() -> None:
|
||||
from routstr.core.exceptions import UpstreamError
|
||||
from routstr.payment.helpers import create_upstream_error_response
|
||||
|
||||
info = classify_rate_limit(429, RATE_LIMIT_MESSAGE)
|
||||
assert info is not None
|
||||
err = UpstreamError(
|
||||
f"Upstream error via litellm: {RATE_LIMIT_MESSAGE}",
|
||||
status_code=429,
|
||||
code=info.code,
|
||||
details=info.as_details(),
|
||||
)
|
||||
|
||||
response = create_upstream_error_response(err, _make_request())
|
||||
|
||||
# Original upstream status is preserved (not flattened to 502).
|
||||
assert response.status_code == 429
|
||||
payload: dict[str, Any] = json.loads(bytes(response.body))
|
||||
assert payload["error"]["type"] == "upstream_error"
|
||||
assert payload["error"]["code"] == UPSTREAM_RATE_LIMIT
|
||||
assert payload["error"]["details"]["requested"] == 8929
|
||||
serialized = json.dumps(payload)
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
|
||||
|
||||
def test_generic_upstream_error_still_defaults_to_502() -> None:
|
||||
from routstr.core.exceptions import UpstreamError
|
||||
from routstr.payment.helpers import create_upstream_error_response
|
||||
|
||||
err = UpstreamError("connection refused") # status_code defaults to 502
|
||||
|
||||
response = create_upstream_error_response(err, _make_request())
|
||||
|
||||
assert response.status_code == 502
|
||||
payload: dict[str, Any] = json.loads(bytes(response.body))
|
||||
assert payload["error"]["type"] == "upstream_error"
|
||||
assert payload["error"]["code"] == 502
|
||||
assert "details" not in payload["error"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Structured log-extra redaction
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_security_filter_redacts_org_id_in_extra() -> None:
|
||||
import logging
|
||||
|
||||
from routstr.core.logging import SecurityFilter
|
||||
|
||||
record = logging.LogRecord(
|
||||
name="test",
|
||||
level=logging.ERROR,
|
||||
pathname=__file__,
|
||||
lineno=1,
|
||||
msg="upstream failed",
|
||||
args=(),
|
||||
exc_info=None,
|
||||
)
|
||||
# Simulate an ``extra={"body_preview": ...}`` field carrying an org id.
|
||||
setattr(record, "body_preview", RATE_LIMIT_MESSAGE)
|
||||
|
||||
assert SecurityFilter().filter(record) is True
|
||||
redacted: str = getattr(record, "body_preview")
|
||||
assert RAW_ORG_ID not in redacted
|
||||
assert "org-[REDACTED]" in redacted
|
||||
|
||||
|
||||
def test_security_filter_redacts_org_id_in_nested_extra() -> None:
|
||||
import logging
|
||||
|
||||
from routstr.core.logging import SecurityFilter
|
||||
|
||||
record = logging.LogRecord(
|
||||
name="test",
|
||||
level=logging.ERROR,
|
||||
pathname=__file__,
|
||||
lineno=1,
|
||||
msg="upstream failed",
|
||||
args=(),
|
||||
exc_info=None,
|
||||
)
|
||||
# Nested structures: dict containing a list containing the org id.
|
||||
setattr(record, "body", {"error": {"messages": [RATE_LIMIT_MESSAGE]}})
|
||||
|
||||
assert SecurityFilter().filter(record) is True
|
||||
serialized = json.dumps(getattr(record, "body"))
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 5xx-wrapped rate limit through forward_upstream_error_response
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_5xx_wrapped_rate_limit_is_classified(
|
||||
provider: BaseUpstreamProvider,
|
||||
) -> None:
|
||||
# Some providers wrap a rate-limit in a 5xx envelope; classification must
|
||||
# key off the message marker, not only the 429 status.
|
||||
body = json.dumps({"error": {"message": RATE_LIMIT_MESSAGE}}).encode()
|
||||
upstream = _make_upstream_response(body=body, status_code=500)
|
||||
|
||||
response = await provider.forward_upstream_error_response(
|
||||
_make_request(), "v1/chat/completions", upstream
|
||||
)
|
||||
|
||||
assert response.status_code == 500
|
||||
payload: dict[str, Any] = json.loads(bytes(response.body))
|
||||
assert payload["error"]["code"] == UPSTREAM_RATE_LIMIT
|
||||
serialized = json.dumps(payload)
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Real proxy loop: structured error surfaced + reservation reverted once
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
from routstr import proxy as proxy_module
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.core.exceptions import UpstreamError
|
||||
|
||||
info = classify_rate_limit(429, RATE_LIMIT_MESSAGE)
|
||||
assert info is not None
|
||||
|
||||
key = ApiKey(hashed_key="rlkey", balance=10_000)
|
||||
|
||||
request = MagicMock()
|
||||
request.method = "POST"
|
||||
request.headers = {"authorization": "Bearer sk-rlkey"}
|
||||
request.body = AsyncMock(return_value=b'{"model": "test-model"}')
|
||||
request.state = MagicMock()
|
||||
request.state.request_id = "req-rl"
|
||||
|
||||
upstream = MagicMock()
|
||||
upstream.provider_type = "test"
|
||||
upstream.prepare_headers = MagicMock(side_effect=lambda h: h)
|
||||
upstream.forward_request = AsyncMock(
|
||||
side_effect=UpstreamError(
|
||||
f"Upstream error via litellm: {RATE_LIMIT_MESSAGE}",
|
||||
status_code=429,
|
||||
code=info.code,
|
||||
details=info.as_details(),
|
||||
)
|
||||
)
|
||||
|
||||
session = MagicMock()
|
||||
revert_mock = AsyncMock(return_value=True)
|
||||
|
||||
with (
|
||||
patch.object(proxy_module, "get_model_instance", return_value=MagicMock()),
|
||||
patch.object(proxy_module, "get_provider_for_model", return_value=[upstream]),
|
||||
patch.object(
|
||||
proxy_module, "get_max_cost_for_model", AsyncMock(return_value=1_000)
|
||||
),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
"calculate_discounted_max_cost",
|
||||
AsyncMock(return_value=1_000),
|
||||
),
|
||||
patch.object(proxy_module, "check_token_balance", MagicMock()),
|
||||
patch.object(
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
|
||||
):
|
||||
response = await proxy_module.proxy(
|
||||
request, "v1/chat/completions", session=session
|
||||
)
|
||||
|
||||
# Original 429 status and the stable code/details survive to the client.
|
||||
assert response.status_code == 429
|
||||
payload: dict[str, Any] = json.loads(bytes(response.body))
|
||||
assert payload["error"]["type"] == "upstream_error"
|
||||
assert payload["error"]["code"] == UPSTREAM_RATE_LIMIT
|
||||
assert payload["error"]["details"]["requested"] == 8929
|
||||
serialized = json.dumps(payload)
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
# Single upstream failed -> reservation reverted exactly once (no double-charge).
|
||||
revert_mock.assert_awaited_once_with(key, session, 1_000)
|
||||
+711
-85
@@ -241,7 +241,9 @@ async def test_credit_balance_rejects_missing_key() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_to_primary_mint_insufficient_for_fees() -> None:
|
||||
"""Token amount is less than melt_quote.amount + melt_quote.fee_reserve."""
|
||||
"""Token amount is less than melt_quote.amount + melt_quote.fee_reserve.
|
||||
The quote mocks are static, so every retry observes the same shortfall —
|
||||
the swap must still give up and raise."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token = Mock()
|
||||
@@ -283,50 +285,6 @@ async def test_swap_to_primary_mint_insufficient_for_fees() -> None:
|
||||
mock_token_wallet.melt.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_to_primary_mint_melt_error_wrapped() -> None:
|
||||
"""Melt failure from cashu lib is wrapped as ValueError."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token = Mock()
|
||||
mock_token.mint = "http://foreign:3338"
|
||||
mock_token.unit = "sat"
|
||||
mock_token.amount = 5000
|
||||
mock_token.keysets = ["keyset1"]
|
||||
mock_token.proofs = [{"amount": 5000}]
|
||||
|
||||
mock_token_wallet = Mock()
|
||||
mock_token_wallet.load_mint = AsyncMock()
|
||||
mock_token_wallet.load_proofs = AsyncMock()
|
||||
mock_token_wallet.get_fees_for_proofs = Mock(return_value=0)
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.load_mint = AsyncMock()
|
||||
mock_primary_wallet.load_proofs = AsyncMock()
|
||||
|
||||
mock_mint_quote = Mock()
|
||||
mock_mint_quote.quote = "mint_quote_456"
|
||||
mock_mint_quote.request = "lnbc1..."
|
||||
mock_primary_wallet.request_mint = AsyncMock(return_value=mock_mint_quote)
|
||||
|
||||
mock_melt_quote = Mock()
|
||||
mock_melt_quote.quote = "melt_quote_456"
|
||||
mock_melt_quote.amount = 4940
|
||||
mock_melt_quote.fee_reserve = 50 # total 4990 < 5000, passes fee check
|
||||
mock_token_wallet.melt_quote = AsyncMock(return_value=mock_melt_quote)
|
||||
mock_token_wallet.melt = AsyncMock(
|
||||
side_effect=Exception("Provided: 5000, needed: 5100 (Code: 11000)")
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
with pytest.raises(ValueError, match="Failed to melt token"):
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recieve_token_untrusted_mint() -> None:
|
||||
mock_wallet = Mock()
|
||||
@@ -394,54 +352,80 @@ async def test_swap_to_primary_mint_already_on_primary() -> None:
|
||||
mock_token_wallet.melt_quote.assert_not_called()
|
||||
|
||||
|
||||
async def test_swap_to_primary_mint_success() -> None:
|
||||
"""Test successful swap with dynamic fee calculation."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
# ---------------------------------------------------------------------------
|
||||
# Swap fee estimation and reactive retry
|
||||
#
|
||||
# Spec: the estimation pass subtracts only observed fees (no safety buffer).
|
||||
# swap_to_primary_mint then runs the mint-quote/melt-quote/melt cycle and, when
|
||||
# the foreign mint demands more than estimated (at quote or at melt time),
|
||||
# retries with the amount recomputed from the observed fee — at most 3 attempts.
|
||||
# Melt failures unrelated to fees are not retried.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_swap_mocks(
|
||||
token_amount: int,
|
||||
fee_reserves: list[int],
|
||||
input_fees: int = 0,
|
||||
mint_url: str = "http://foreign-mint:3338",
|
||||
) -> tuple[Mock, Mock, Mock]:
|
||||
"""Return (token, token_wallet, primary_wallet) mocks that act like a mint.
|
||||
|
||||
Mint quotes pass the requested amount through their ``request`` field and
|
||||
melt quotes echo that amount back, so the mocks stay consistent for
|
||||
whatever amounts the implementation requests. ``fee_reserves`` supplies the
|
||||
fee_reserve of each successive melt quote (the first serves the estimation
|
||||
pass); requesting more quotes than provided fails the test.
|
||||
"""
|
||||
mock_token = Mock()
|
||||
mock_token.mint = "http://foreign:3338"
|
||||
mock_token.mint = mint_url
|
||||
mock_token.unit = "sat"
|
||||
mock_token.amount = 1000
|
||||
mock_token.amount = token_amount
|
||||
mock_token.keysets = ["keyset1"]
|
||||
mock_token.proofs = [{"amount": 1000}]
|
||||
mock_token.proofs = [Mock(amount=token_amount)]
|
||||
|
||||
mock_token_wallet = Mock()
|
||||
mock_token_wallet.load_mint = AsyncMock()
|
||||
mock_token_wallet.load_proofs = AsyncMock()
|
||||
mock_token_wallet.get_fees_for_proofs = Mock(return_value=0)
|
||||
mock_token_wallet.get_fees_for_proofs = Mock(return_value=input_fees)
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.load_mint = AsyncMock()
|
||||
mock_primary_wallet.load_proofs = AsyncMock()
|
||||
mock_primary_wallet.available_balance = Mock(amount=0)
|
||||
mock_primary_wallet.mint = AsyncMock(return_value=Mock())
|
||||
|
||||
# Mocks for the estimation phase
|
||||
# 1. request_mint(dummy_amount=1000) -> invoice_dummy
|
||||
# 2. melt_quote(invoice_dummy) -> fee=10
|
||||
fees = iter(fee_reserves)
|
||||
|
||||
# Mocks for the execution phase
|
||||
# 3. request_mint(minted_amount=990) -> invoice_real
|
||||
# 4. melt_quote(invoice_real) -> amount=990, fee=10
|
||||
# 5. melt() -> success
|
||||
# 6. mint() -> success
|
||||
def _next_fee() -> int:
|
||||
try:
|
||||
return next(fees)
|
||||
except StopIteration:
|
||||
raise AssertionError(
|
||||
"more melt quotes requested than fee_reserves provided"
|
||||
) from None
|
||||
|
||||
mock_mint_quote_dummy = Mock(quote="dummy_quote", request="lnbc_dummy")
|
||||
mock_mint_quote_real = Mock(quote="real_quote", request="lnbc_real")
|
||||
|
||||
# side_effect for request_mint to return dummy then real
|
||||
mock_primary_wallet.request_mint = AsyncMock(
|
||||
side_effect=[mock_mint_quote_dummy, mock_mint_quote_real]
|
||||
side_effect=lambda amount: Mock(quote=f"mint_quote_{amount}", request=amount)
|
||||
)
|
||||
|
||||
mock_melt_quote_dummy = Mock(amount=1000, fee_reserve=10)
|
||||
mock_melt_quote_real = Mock(amount=990, fee_reserve=10)
|
||||
|
||||
# side_effect for melt_quote
|
||||
mock_token_wallet.melt_quote = AsyncMock(
|
||||
side_effect=[mock_melt_quote_dummy, mock_melt_quote_real]
|
||||
side_effect=lambda invoice: Mock(
|
||||
quote=f"melt_quote_{invoice}", amount=invoice, fee_reserve=_next_fee()
|
||||
)
|
||||
)
|
||||
mock_token_wallet.melt = AsyncMock(return_value=Mock())
|
||||
|
||||
mock_token_wallet.melt = AsyncMock(return_value="melted_proofs")
|
||||
mock_primary_wallet.mint = AsyncMock(return_value="minted_proofs")
|
||||
return mock_token, mock_token_wallet, mock_primary_wallet
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_to_primary_mint_success() -> None:
|
||||
"""No retry needed: real quote matches the estimate, full net amount minted."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[10, 10]
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
@@ -452,17 +436,659 @@ async def test_swap_to_primary_mint_success() -> None:
|
||||
mock_token, mock_token_wallet
|
||||
)
|
||||
|
||||
assert amount == 990 # 1000 - 10
|
||||
assert unit == "sat"
|
||||
assert mint == "http://primary:3338"
|
||||
assert amount == 990 # 1000 - fee_reserve(10), no buffer subtracted
|
||||
assert unit == "sat"
|
||||
assert mint == "http://primary:3338"
|
||||
assert mock_primary_wallet.request_mint.call_count == 2
|
||||
mock_primary_wallet.request_mint.assert_any_call(1000)
|
||||
mock_primary_wallet.request_mint.assert_any_call(990)
|
||||
assert mock_token_wallet.melt_quote.call_count == 2
|
||||
assert mock_token_wallet.melt.call_count == 1
|
||||
assert mock_primary_wallet.mint.called
|
||||
|
||||
# Verify call order/counts
|
||||
assert mock_primary_wallet.request_mint.call_count == 2
|
||||
# First call with full amount for estimation
|
||||
mock_primary_wallet.request_mint.assert_any_call(1000)
|
||||
# Second call with calculated amount
|
||||
mock_primary_wallet.request_mint.assert_any_call(990)
|
||||
|
||||
assert mock_token_wallet.melt_quote.call_count == 2
|
||||
assert mock_token_wallet.melt.called
|
||||
assert mock_primary_wallet.mint.called
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fee_reserve", [1, 10, 100])
|
||||
async def test_calculate_swap_amount_subtracts_only_observed_fees(
|
||||
fee_reserve: int,
|
||||
) -> None:
|
||||
"""Estimation: minted_amount = token - fee_reserve, with no safety buffer."""
|
||||
from routstr.wallet import _calculate_swap_amount
|
||||
|
||||
_, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[fee_reserve]
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
result = await _calculate_swap_amount(
|
||||
amount_msat=1_000_000,
|
||||
token_unit="sat",
|
||||
token_mint_url="http://foreign-mint:3338",
|
||||
token_wallet=mock_token_wallet,
|
||||
primary_wallet=mock_primary_wallet,
|
||||
proofs=[],
|
||||
)
|
||||
|
||||
assert result == 1000 - fee_reserve
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_calculate_swap_amount_includes_input_fees() -> None:
|
||||
"""Estimation subtracts NUT-02 input fees alongside the melt fee_reserve."""
|
||||
from routstr.wallet import _calculate_swap_amount
|
||||
|
||||
_, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
500, fee_reserves=[10], input_fees=3
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
result = await _calculate_swap_amount(
|
||||
amount_msat=500_000,
|
||||
token_unit="sat",
|
||||
token_mint_url="http://foreign-mint:3338",
|
||||
token_wallet=mock_token_wallet,
|
||||
primary_wallet=mock_primary_wallet,
|
||||
proofs=[],
|
||||
)
|
||||
|
||||
assert result == 487 # 500 - 10 - 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_retries_when_real_quote_exceeds_estimate() -> None:
|
||||
"""The real melt quote demands a higher fee than the estimate (20 → 23).
|
||||
Instead of failing, the swap recomputes the amount from the observed fee
|
||||
and re-quotes: 1000 - 23 = 977, which fits (977 + 23 <= 1000)."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[20, 23, 23]
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
amount, unit, mint = await swap_to_primary_mint(
|
||||
mock_token, mock_token_wallet
|
||||
)
|
||||
|
||||
assert amount == 977
|
||||
assert unit == "sat"
|
||||
mock_primary_wallet.request_mint.assert_any_call(980)
|
||||
mock_primary_wallet.request_mint.assert_any_call(977)
|
||||
assert mock_token_wallet.melt_quote.call_count == 3 # estimation + 2 attempts
|
||||
assert mock_token_wallet.melt.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_retries_when_melt_demands_more_than_quoted() -> None:
|
||||
"""The mint.cubabitcoin.org incident: every quote reports fee_reserve=1,
|
||||
but the mint demands 2 sats at melt time ("Provided: 179, needed: 180").
|
||||
The swap must retry with a smaller invoice (177) so the second melt fits,
|
||||
instead of failing the topup."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
179, fee_reserves=[1, 1, 1], mint_url="http://mint.cubabitcoin.org"
|
||||
)
|
||||
mock_token_wallet.melt.side_effect = [
|
||||
Exception(
|
||||
"Mint Error: not enough inputs provided for melt. "
|
||||
"Provided: 179, needed: 180 (Code: 11000)"
|
||||
),
|
||||
Mock(),
|
||||
]
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
amount, unit, mint = await swap_to_primary_mint(
|
||||
mock_token, mock_token_wallet
|
||||
)
|
||||
|
||||
assert amount == 177 # 179 - 1 (estimate) - 1 (observed melt shortfall)
|
||||
assert mock_token_wallet.melt.call_count == 2
|
||||
mock_primary_wallet.request_mint.assert_any_call(178)
|
||||
mock_primary_wallet.request_mint.assert_any_call(177)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_retries_on_cdk_unbalanced_error() -> None:
|
||||
"""cdk-based mints report insufficient melt inputs as the registered code
|
||||
11005 (TransactionUnbalanced) with their own message wording — no
|
||||
Provided/needed amounts to parse. The retry must classify it by code and
|
||||
fall back to shrinking by 1."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
179, fee_reserves=[1, 1, 1]
|
||||
)
|
||||
mock_token_wallet.melt.side_effect = [
|
||||
Exception("Mint Error: Transaction unbalanced: 179, 178, 2 (Code: 11005)"),
|
||||
Mock(),
|
||||
]
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
amount, unit, mint = await swap_to_primary_mint(
|
||||
mock_token, mock_token_wallet
|
||||
)
|
||||
|
||||
assert amount == 177
|
||||
assert mock_token_wallet.melt.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_quote_retries_exhausted() -> None:
|
||||
"""A mint that escalates fee_reserve on every re-quote exhausts the retry
|
||||
budget (3 attempts) and fails cleanly; melt is never executed."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[1, 10, 25, 50]
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
with pytest.raises(ValueError, match="insufficient to cover melt fees"):
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
assert mock_token_wallet.melt_quote.call_count == 4 # estimation + 3 attempts
|
||||
mock_token_wallet.melt.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_melt_retries_exhausted() -> None:
|
||||
"""A mint that always demands more at melt time than it quoted exhausts
|
||||
the retry budget; the last melt failure is wrapped as ValueError."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
5000, fee_reserves=[50, 50, 50, 50]
|
||||
)
|
||||
mock_token_wallet.melt = AsyncMock(
|
||||
side_effect=Exception(
|
||||
"Mint Error: not enough inputs provided for melt. "
|
||||
"Provided: 5000, needed: 5200 (Code: 11000)"
|
||||
)
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
with pytest.raises(ValueError, match="Failed to melt token"):
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
assert mock_token_wallet.melt.call_count == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"primary_unit,token_unit,amount_msat,fees,expected",
|
||||
[
|
||||
("sat", "sat", 179_000, 2, 177),
|
||||
("msat", "sat", 179_000, 2, 177_000),
|
||||
("sat", "msat", 179_000, 2_000, 177),
|
||||
],
|
||||
)
|
||||
async def test_net_minted_amount_unit_conversions(
|
||||
primary_unit: str, token_unit: str, amount_msat: int, fees: int, expected: int
|
||||
) -> None:
|
||||
"""Fee subtraction converts correctly between sat and msat on either side."""
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _net_minted_amount
|
||||
|
||||
with patch.object(settings, "primary_mint_unit", primary_unit):
|
||||
assert _net_minted_amount(amount_msat, token_unit, fees) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"message,expected",
|
||||
[
|
||||
# nutshell: retryable with exact shortfall from the detail text
|
||||
(
|
||||
"Mint Error: not enough inputs provided for melt. "
|
||||
"Provided: 179, needed: 182 (Code: 11000)",
|
||||
3,
|
||||
),
|
||||
# verbatim production error from issue #468, including cashu-py's
|
||||
# "could not pay invoice" wrapper around the mint detail
|
||||
(
|
||||
"could not pay invoice: Mint Error: not enough inputs provided "
|
||||
"for melt. Provided: 179, needed: 180 (Code: 11000)",
|
||||
1,
|
||||
),
|
||||
# cdk: registered TransactionUnbalanced code, no parsable amounts
|
||||
("Mint Error: Transaction unbalanced: 179, 178, 2 (Code: 11005)", 1),
|
||||
# nutshell wording without a code suffix
|
||||
("not enough inputs provided for melt", 1),
|
||||
# nonsensical amounts (needed <= provided) fall back to the minimal step
|
||||
(
|
||||
"Mint Error: not enough inputs provided for melt. "
|
||||
"Provided: 180, needed: 179 (Code: 11000)",
|
||||
1,
|
||||
),
|
||||
# a generic 11000 without the shortfall text is not a fee shortfall:
|
||||
# 11000 is nutshell's catch-all TransactionError, so retrying (shrinking
|
||||
# the invoice) would never help and only masks the real error
|
||||
("Mint Error: Duplicate inputs provided. (Code: 11000)", None),
|
||||
# spent proofs must never be retried: the funds are gone
|
||||
("Mint Error: Token already spent. (Code: 11001)", None),
|
||||
# Lightning failures must never be retried: a smaller invoice won't help
|
||||
("Mint Error: Lightning payment failed. (Code: 20004)", None),
|
||||
# unrecognizable errors (timeouts, bugs) must never be retried
|
||||
("Connection timeout", None),
|
||||
],
|
||||
)
|
||||
def test_melt_shortfall_classifier(message: str, expected: int | None) -> None:
|
||||
"""Retry classification across mint implementations and failure classes."""
|
||||
from routstr.wallet import _melt_insufficient_shortfall
|
||||
|
||||
assert _melt_insufficient_shortfall(Exception(message)) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_calculate_swap_amount_same_mint_short_circuit() -> None:
|
||||
"""When the token is already on the primary mint no fees apply and no
|
||||
quotes are requested."""
|
||||
from routstr.wallet import _calculate_swap_amount
|
||||
|
||||
_, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[]
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
result = await _calculate_swap_amount(
|
||||
amount_msat=1_000_000,
|
||||
token_unit="sat",
|
||||
token_mint_url="http://primary:3338",
|
||||
token_wallet=mock_token_wallet,
|
||||
primary_wallet=mock_primary_wallet,
|
||||
proofs=[],
|
||||
)
|
||||
|
||||
assert result == 1000
|
||||
mock_primary_wallet.request_mint.assert_not_called()
|
||||
mock_token_wallet.melt_quote.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_calculate_swap_amount_msat_primary_unit() -> None:
|
||||
"""With an msat primary mint the dummy quote and result stay in msats."""
|
||||
from routstr.wallet import _calculate_swap_amount
|
||||
|
||||
_, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
179, fee_reserves=[2]
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "msat"):
|
||||
result = await _calculate_swap_amount(
|
||||
amount_msat=179_000,
|
||||
token_unit="sat",
|
||||
token_mint_url="http://foreign-mint:3338",
|
||||
token_wallet=mock_token_wallet,
|
||||
primary_wallet=mock_primary_wallet,
|
||||
proofs=[],
|
||||
)
|
||||
|
||||
assert result == 177_000 # 179_000 msat - 2 sat fee
|
||||
mock_primary_wallet.request_mint.assert_called_once_with(179_000)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_calculate_swap_amount_fees_exceed_token() -> None:
|
||||
"""Fees larger than the token itself fail fast, before any melt."""
|
||||
from routstr.wallet import _calculate_swap_amount
|
||||
|
||||
_, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
179, fee_reserves=[200]
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with pytest.raises(ValueError, match="exceed token amount"):
|
||||
await _calculate_swap_amount(
|
||||
amount_msat=179_000,
|
||||
token_unit="sat",
|
||||
token_mint_url="http://foreign-mint:3338",
|
||||
token_wallet=mock_token_wallet,
|
||||
primary_wallet=mock_primary_wallet,
|
||||
proofs=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_calculate_swap_amount_wraps_estimation_failure() -> None:
|
||||
"""Estimation infrastructure failures surface as a single clear ValueError."""
|
||||
from routstr.wallet import _calculate_swap_amount
|
||||
|
||||
_, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
179, fee_reserves=[]
|
||||
)
|
||||
mock_primary_wallet.request_mint = AsyncMock(
|
||||
side_effect=Exception("mint offline")
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with pytest.raises(ValueError, match="Failed to estimate fees"):
|
||||
await _calculate_swap_amount(
|
||||
amount_msat=179_000,
|
||||
token_unit="sat",
|
||||
token_mint_url="http://foreign-mint:3338",
|
||||
token_wallet=mock_token_wallet,
|
||||
primary_wallet=mock_primary_wallet,
|
||||
proofs=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_coerces_non_integer_amount() -> None:
|
||||
"""Token amounts arriving as floats are coerced before any arithmetic."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[10, 10]
|
||||
)
|
||||
mock_token.amount = 1000.0
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
amount, unit, mint = await swap_to_primary_mint(
|
||||
mock_token, mock_token_wallet
|
||||
)
|
||||
|
||||
assert amount == 990
|
||||
assert isinstance(amount, int)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_rejects_unknown_unit() -> None:
|
||||
"""Units other than sat/msat are rejected before any quote is requested."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[]
|
||||
)
|
||||
mock_token.unit = "usd"
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
with pytest.raises(ValueError, match="Invalid unit"):
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
mock_primary_wallet.request_mint.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_msat_token_already_on_primary() -> None:
|
||||
"""msat-denominated tokens on the primary mint short-circuit unchanged."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, _ = _make_swap_mocks(
|
||||
179_000, fee_reserves=[], mint_url="http://primary:3338"
|
||||
)
|
||||
mock_token.unit = "msat"
|
||||
mock_token_wallet.split = AsyncMock()
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_token_wallet):
|
||||
amount, unit, mint = await swap_to_primary_mint(
|
||||
mock_token, mock_token_wallet
|
||||
)
|
||||
|
||||
assert (amount, unit, mint) == (179_000, "msat", "http://primary:3338")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mint-on-primary failure handling after a successful melt
|
||||
#
|
||||
# At this point the foreign proofs are already spent: failures here mean funds
|
||||
# are in limbo, so errors must propagate (never be swallowed) and recovery must
|
||||
# never credit proofs the wallet does not actually hold.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _with_recovery_mocks(
|
||||
mock_primary_wallet: Mock, mint_error: str, balances: list[int]
|
||||
) -> None:
|
||||
"""Make primary mint() fail and stage available_balance per load_proofs call."""
|
||||
mock_primary_wallet.mint = AsyncMock(side_effect=Exception(mint_error))
|
||||
mock_primary_wallet.keysets = ["keyset_primary"]
|
||||
balance_iter = iter(balances)
|
||||
|
||||
def advance_balance(reload: bool = False) -> None:
|
||||
mock_primary_wallet.available_balance = Mock(amount=next(balance_iter))
|
||||
|
||||
mock_primary_wallet.load_proofs = AsyncMock(side_effect=advance_balance)
|
||||
mock_primary_wallet.restore_tokens_for_keyset = AsyncMock()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_mint_failure_propagates_unwrapped() -> None:
|
||||
"""A non-recoverable mint failure after melt propagates as-is (a 500, not a
|
||||
client error): the melt already spent the foreign proofs."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[10, 10]
|
||||
)
|
||||
_with_recovery_mocks(
|
||||
mock_primary_wallet, "Mint Error: Quote is expired (Code: 20007)", [0]
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
with pytest.raises(Exception, match="Quote is expired") as exc_info:
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
assert type(exc_info.value) is Exception # original error, not wrapped
|
||||
assert mock_token_wallet.melt.call_count == 1
|
||||
mock_primary_wallet.restore_tokens_for_keyset.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_recovers_orphaned_proofs_on_outputs_already_signed() -> None:
|
||||
"""11003 (outputs already signed): a recovery scan that restores the full
|
||||
minted amount lets the swap complete normally."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[10, 10]
|
||||
)
|
||||
_with_recovery_mocks(
|
||||
mock_primary_wallet,
|
||||
"Mint Error: outputs already signed (Code: 11003)",
|
||||
[0, 990], # pre-mint balance, post-recovery balance
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
amount, unit, mint = await swap_to_primary_mint(
|
||||
mock_token, mock_token_wallet
|
||||
)
|
||||
|
||||
assert amount == 990
|
||||
mock_primary_wallet.restore_tokens_for_keyset.assert_awaited_once_with(
|
||||
"keyset_primary", to=1, batch=25
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_recovery_shortfall_refuses_credit() -> None:
|
||||
"""When the recovery scan restores less than the minted amount, the swap
|
||||
must fail rather than credit proofs the wallet does not hold."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[10, 10]
|
||||
)
|
||||
_with_recovery_mocks(
|
||||
mock_primary_wallet,
|
||||
"Mint Error: outputs already signed (Code: 11003)",
|
||||
[0, 100], # recovery restores only 100 of the expected 990
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
with pytest.raises(ValueError, match="Swap recovery failed"):
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_recovery_failure_wrapped() -> None:
|
||||
"""When the recovery scan itself fails, the error is wrapped and raised —
|
||||
never swallowed."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[10, 10]
|
||||
)
|
||||
_with_recovery_mocks(
|
||||
mock_primary_wallet,
|
||||
"Mint Error: outputs already signed (Code: 11003)",
|
||||
[0],
|
||||
)
|
||||
mock_primary_wallet.restore_tokens_for_keyset = AsyncMock(
|
||||
side_effect=Exception("wallet db locked")
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
with pytest.raises(ValueError, match="recovery unsuccessful"):
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recieve_token_rejects_multiple_keysets() -> None:
|
||||
"""Multi-keyset tokens are rejected before touching any wallet."""
|
||||
with patch("routstr.wallet.deserialize_token_from_string") as mock_deserialize:
|
||||
mock_token = Mock()
|
||||
mock_token.keysets = ["keyset1", "keyset2"]
|
||||
mock_deserialize.return_value = mock_token
|
||||
|
||||
with pytest.raises(ValueError, match="Multiple keysets"):
|
||||
await recieve_token("cashuAmultikeyset")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_credit_balance_msat_unit_not_converted() -> None:
|
||||
"""msat-denominated redemptions are credited as-is, without a 1000x."""
|
||||
mock_key = Mock()
|
||||
mock_key.balance = 0
|
||||
mock_key.hashed_key = "test_hash"
|
||||
mock_session = AsyncMock()
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "cashu_mints", ["http://mint:3338"]):
|
||||
with patch(
|
||||
"routstr.wallet.recieve_token",
|
||||
return_value=(1_000_000, "msat", "http://mint:3338"),
|
||||
):
|
||||
amount = await credit_balance("cashuAtest", mock_key, mock_session)
|
||||
|
||||
assert amount == 1_000_000
|
||||
assert mock_session.commit.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_credit_balance_survives_audit_store_failure() -> None:
|
||||
"""A failure writing the CashuTransaction history record must not undo the
|
||||
already-committed balance credit. (The silent swallow is a known
|
||||
audit-trail gap slated for its own fix — this test pins the financial
|
||||
invariant that the user keeps their credit, not the swallow itself.)"""
|
||||
mock_key = Mock()
|
||||
mock_key.balance = 0
|
||||
mock_key.hashed_key = "test_hash"
|
||||
mock_session = AsyncMock()
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "cashu_mints", ["http://mint:3338"]):
|
||||
with patch(
|
||||
"routstr.wallet.recieve_token",
|
||||
return_value=(1000, "sat", "http://mint:3338"),
|
||||
):
|
||||
with patch(
|
||||
"routstr.wallet.store_cashu_transaction",
|
||||
side_effect=Exception("history table locked"),
|
||||
):
|
||||
amount = await credit_balance("cashuAtest", mock_key, mock_session)
|
||||
|
||||
assert amount == 1_000_000
|
||||
assert mock_session.commit.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swap_does_not_retry_on_payment_failure() -> None:
|
||||
"""Melt failures unrelated to fees (e.g. routing failure) are not retried:
|
||||
a smaller invoice would not help, and the error must surface immediately."""
|
||||
from routstr.wallet import swap_to_primary_mint
|
||||
|
||||
mock_token, mock_token_wallet, mock_primary_wallet = _make_swap_mocks(
|
||||
1000, fee_reserves=[10, 10]
|
||||
)
|
||||
mock_token_wallet.melt = AsyncMock(
|
||||
side_effect=Exception("Mint Error: Lightning payment failed. (Code: 20004)")
|
||||
)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "primary_mint", "http://primary:3338"):
|
||||
with patch.object(settings, "primary_mint_unit", "sat"):
|
||||
with patch("routstr.wallet.get_wallet", return_value=mock_primary_wallet):
|
||||
with pytest.raises(ValueError, match="Failed to melt token"):
|
||||
await swap_to_primary_mint(mock_token, mock_token_wallet)
|
||||
|
||||
assert mock_token_wallet.melt.call_count == 1
|
||||
assert mock_primary_wallet.request_mint.call_count == 2
|
||||
|
||||
+32
-24
@@ -1,36 +1,44 @@
|
||||
This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app).
|
||||
# Routstr node admin UI
|
||||
|
||||
## Getting Started
|
||||
A [Next.js](https://nextjs.org) app (App Router, **static export**) that provides the
|
||||
admin dashboard for a `routstr-core` node: login, settings, providers, balances,
|
||||
transactions, usage, and logs.
|
||||
|
||||
First, run the development server:
|
||||
There is no separate web server in production. `next build` produces a fully static
|
||||
export (`next.config.ts` sets `output: 'export'`), and the FastAPI backend serves it
|
||||
directly from `../ui_out/` (see `routstr/core/main.py`). So the UI and the API are
|
||||
served from the **same origin** in production.
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
# or
|
||||
yarn dev
|
||||
# or
|
||||
pnpm dev
|
||||
# or
|
||||
bun dev
|
||||
```
|
||||
## Developing the UI (hot reload)
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
|
||||
The everyday loop runs two processes side by side — you do **not** rebuild the static
|
||||
export while developing:
|
||||
|
||||
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
|
||||
1. Start the backend on `:8000` — from the repo root: `make docker-up` (or
|
||||
`uvicorn routstr.core.main:app --reload`).
|
||||
2. Start the Next.js dev server on `:3000` — from the repo root: `make ui-dev`
|
||||
(or `cd ui && pnpm dev`). Edits hot-reload instantly.
|
||||
|
||||
This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
|
||||
Open http://localhost:3000. With no `NEXT_PUBLIC_API_URL` set, the UI falls back to
|
||||
`http://127.0.0.1:8000` in development (see `lib/api/services/configuration.ts`), so it
|
||||
talks to the local backend out of the box.
|
||||
|
||||
## Learn More
|
||||
Because dev is cross-origin (`:3000` → `:8000`), it relies on the backend's CORS
|
||||
allowing the UI origin. The default `cors_origins` is `["*"]`; if you tighten CORS,
|
||||
keep `http://localhost:3000` allowed for development.
|
||||
|
||||
To learn more about Next.js, take a look at the following resources:
|
||||
## Building the integrated/static UI (what production serves)
|
||||
|
||||
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
|
||||
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
|
||||
To produce the bundle that FastAPI serves from `../ui_out/`:
|
||||
|
||||
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
|
||||
- `make ui-build` — builds with local Node/pnpm (`scripts/build-ui.sh`), then moves
|
||||
`ui/out/*` to `../ui_out/`.
|
||||
- `make ui-build-docker` — same, but inside Docker (no local Node needed).
|
||||
|
||||
## Deploy on Vercel
|
||||
`NEXT_PUBLIC_*` variables are read from the repo-root `.env` at build time and baked in.
|
||||
For a same-origin deployment leave `NEXT_PUBLIC_API_URL` empty (relative paths); the UI
|
||||
uses `window.location.origin` at runtime. After building, start the backend and open
|
||||
http://localhost:8000 — the dashboard is served at `/` and `/admin`.
|
||||
|
||||
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
|
||||
|
||||
Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.
|
||||
If `../ui_out/` does not exist, the backend logs a warning at startup and serves the API
|
||||
only (hitting a UI route returns a small JSON fallback instead of the dashboard).
|
||||
|
||||
Reference in New Issue
Block a user