mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-07-30 15:26:14 +00:00
Compare commits
35
Commits
main
...
443c910b9e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
443c910b9e | ||
|
|
3befe063f4 | ||
|
|
f8adaee362 | ||
|
|
895ea90bfa | ||
|
|
c75dee147a | ||
|
|
48c11eb7bc | ||
|
|
c829685f80 | ||
|
|
1138cdd4ef | ||
|
|
92246b78d0 | ||
|
|
6023c03959 | ||
|
|
a9a6381614 | ||
|
|
040799a4d7 | ||
|
|
586af15a1b | ||
|
|
3e906605a0 | ||
|
|
1957e716a3 | ||
|
|
69f19ff991 | ||
|
|
8b942f3c14 | ||
|
|
09e1c7bf2d | ||
|
|
cc2a96e2ef | ||
|
|
93ab1d927b | ||
|
|
39970d8bee | ||
|
|
d7c401d204 | ||
|
|
d44b98fd0d | ||
|
|
6fa3610423 | ||
|
|
65702171e4 | ||
|
|
65abcbce92 | ||
|
|
40153d4c36 | ||
|
|
40bf976fbc | ||
|
|
eae20f04a7 | ||
|
|
acb630f6cf | ||
|
|
1230d528de | ||
|
|
d23c90b939 | ||
|
|
d8db2a3051 | ||
|
|
0bbbf902cd | ||
|
|
7ed18a9d02 |
@@ -0,0 +1,25 @@
|
||||
"""add mint url to lightning invoices
|
||||
|
||||
Revision ID: bf76270b66c4
|
||||
Revises: aa50fde387a2
|
||||
Create Date: 2026-07-30 00:54:30.306876
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "bf76270b66c4"
|
||||
down_revision = "aa50fde387a2"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"lightning_invoices", sa.Column("mint_url", sa.String(), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("lightning_invoices", "mint_url")
|
||||
+22
-14
@@ -51,6 +51,24 @@ ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
|
||||
ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
|
||||
|
||||
|
||||
def _format_msat_amount(amount: int) -> str:
|
||||
sats = f"{amount / 1000:.3f}".rstrip("0").rstrip(".")
|
||||
return f"{sats} sats ({amount} msats)"
|
||||
|
||||
|
||||
def _model_balance_error(required: int, available: int) -> dict[str, dict[str, str]]:
|
||||
return {
|
||||
"error": {
|
||||
"message": (
|
||||
f"Insufficient balance: {_format_msat_amount(required)} required "
|
||||
f"for this model; {_format_msat_amount(available)} available."
|
||||
),
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReservationSnapshot:
|
||||
release_id: str
|
||||
@@ -264,13 +282,7 @@ async def _validate_bearer_key_locked(
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": {
|
||||
"message": f"Insufficient balance: {min_cost} mSats required for this model. {billing_key.total_balance} available.",
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
},
|
||||
detail=_model_balance_error(min_cost, billing_key.total_balance),
|
||||
)
|
||||
|
||||
# Early check: Spending limit check (Child key limit)
|
||||
@@ -360,13 +372,9 @@ async def _validate_bearer_key_locked(
|
||||
if min_cost > 0 and existing_key.total_balance < min_cost:
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": {
|
||||
"message": f"Insufficient balance: {min_cost} mSats required for this model. {existing_key.total_balance} available.",
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
},
|
||||
detail=_model_balance_error(
|
||||
min_cost, existing_key.total_balance
|
||||
),
|
||||
)
|
||||
|
||||
return existing_key
|
||||
|
||||
+84
-13
@@ -30,6 +30,7 @@ from .wallet import (
|
||||
recieve_token,
|
||||
send_to_lnurl,
|
||||
send_token,
|
||||
token_mint_url,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
@@ -184,6 +185,17 @@ class TopupRequest(BaseModel):
|
||||
cashu_token: str
|
||||
|
||||
|
||||
def _error_chain(error: BaseException) -> list[dict[str, str]]:
|
||||
chain: list[dict[str, str]] = []
|
||||
current: BaseException | None = error
|
||||
seen: set[int] = set()
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
chain.append({"type": type(current).__name__, "message": str(current)})
|
||||
current = current.__cause__ or current.__context__
|
||||
return chain
|
||||
|
||||
|
||||
@router.post("/topup")
|
||||
async def topup_wallet_endpoint(
|
||||
cashu_token: str | None = None,
|
||||
@@ -201,6 +213,18 @@ async def topup_wallet_endpoint(
|
||||
cashu_token = cashu_token.replace("\n", "").replace("\r", "").replace("\t", "")
|
||||
if len(cashu_token) < 10 or "cashu" not in cashu_token:
|
||||
raise HTTPException(status_code=400, detail="Invalid token format")
|
||||
|
||||
source_mint = token_mint_url(cashu_token, "unknown")
|
||||
logger.warning(
|
||||
"Cashu wallet top-up started",
|
||||
extra={
|
||||
"event": "cashu_topup_started",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"key_hash": billing_key.hashed_key[:8],
|
||||
},
|
||||
)
|
||||
try:
|
||||
amount_msats = await credit_balance(cashu_token, billing_key, session)
|
||||
except Exception as e:
|
||||
@@ -209,12 +233,41 @@ async def topup_wallet_endpoint(
|
||||
classified = classify_redemption_error(e)
|
||||
if classified is None:
|
||||
logger.error(
|
||||
"topup_wallet_endpoint: unhandled error",
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
"Cashu wallet top-up failed with an unhandled error",
|
||||
extra={
|
||||
"event": "cashu_topup_failed",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"error_chain": _error_chain(e),
|
||||
},
|
||||
)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
_type, status_code, message, _code = classified
|
||||
error_type, status_code, message, error_code = classified
|
||||
logger.warning(
|
||||
"Cashu wallet top-up failed",
|
||||
extra={
|
||||
"event": "cashu_topup_failed",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"status_code": status_code,
|
||||
"error_type": error_type,
|
||||
"error_code": error_code,
|
||||
"error_chain": _error_chain(e),
|
||||
},
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=message)
|
||||
|
||||
logger.warning(
|
||||
"Cashu wallet top-up completed",
|
||||
extra={
|
||||
"event": "cashu_topup_completed",
|
||||
"source_mint": source_mint,
|
||||
"credited_msats": amount_msats,
|
||||
"key_hash": billing_key.hashed_key[:8],
|
||||
},
|
||||
)
|
||||
return {"msats": amount_msats}
|
||||
|
||||
|
||||
@@ -260,7 +313,11 @@ async def _lookup_key_no_create(
|
||||
|
||||
|
||||
async def _restore_balance(
|
||||
session: AsyncSession, hashed_key: str, balance: int, reserved_balance: int, mint_url: str
|
||||
session: AsyncSession,
|
||||
hashed_key: str,
|
||||
balance: int,
|
||||
reserved_balance: int,
|
||||
mint_url: str,
|
||||
) -> None:
|
||||
"""Restore balance after a failed refund mint attempt."""
|
||||
restore_stmt = (
|
||||
@@ -275,7 +332,11 @@ async def _restore_balance(
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"refund_wallet_endpoint: balance restored after mint failure",
|
||||
extra={"hashed_key": hashed_key, "restored_balance": balance, "mint_url": mint_url},
|
||||
extra={
|
||||
"hashed_key": hashed_key,
|
||||
"restored_balance": balance,
|
||||
"mint_url": mint_url,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -418,15 +479,14 @@ async def refund_wallet_endpoint(
|
||||
detail="Balance changed concurrently. Please retry the refund.",
|
||||
)
|
||||
|
||||
# --- MINT: balance is locked at zero, safe to create the refund token ---
|
||||
# Proofs from untrusted mints are swapped to primary_mint on receive.
|
||||
# Use primary_mint unless key.refund_mint_url is an explicitly trusted mint.
|
||||
# The balance is locked at zero, so it is safe to create the refund token.
|
||||
effective_refund_mint = (
|
||||
key.refund_mint_url
|
||||
if key.refund_mint_url and key.refund_mint_url in settings.cashu_mints
|
||||
else settings.primary_mint
|
||||
)
|
||||
try:
|
||||
refund_currency = key.refund_currency or "sat"
|
||||
if key.refund_address:
|
||||
await send_to_lnurl(
|
||||
remaining_balance,
|
||||
@@ -436,10 +496,10 @@ async def refund_wallet_endpoint(
|
||||
)
|
||||
result = {"recipient": key.refund_address}
|
||||
else:
|
||||
refund_currency = key.refund_currency or "sat"
|
||||
token = await send_token(
|
||||
remaining_balance, refund_currency, effective_refund_mint
|
||||
)
|
||||
effective_refund_mint = token_mint_url(token, effective_refund_mint)
|
||||
result = {"token": token}
|
||||
|
||||
if key.refund_currency == "sat":
|
||||
@@ -460,11 +520,23 @@ async def refund_wallet_endpoint(
|
||||
|
||||
except HTTPException:
|
||||
# Minting failed — restore the debited balance
|
||||
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
|
||||
await _restore_balance(
|
||||
session,
|
||||
key.hashed_key,
|
||||
pre_debit_balance,
|
||||
pre_debit_reserved,
|
||||
key.refund_mint_url or "",
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
# Minting failed — restore the debited balance
|
||||
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
|
||||
await _restore_balance(
|
||||
session,
|
||||
key.hashed_key,
|
||||
pre_debit_balance,
|
||||
pre_debit_reserved,
|
||||
key.refund_mint_url or "",
|
||||
)
|
||||
error_msg = str(e)
|
||||
logger.error(
|
||||
"refund_wallet_endpoint: mint/send failed",
|
||||
@@ -491,7 +563,7 @@ async def refund_wallet_endpoint(
|
||||
token=result["token"],
|
||||
amount=remaining_balance,
|
||||
unit=key.refund_currency or "sat",
|
||||
mint_url=key.refund_mint_url,
|
||||
mint_url=effective_refund_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="apikey",
|
||||
@@ -685,7 +757,6 @@ async def reset_child_key_spent(
|
||||
return {"success": True, "message": "Child key balance reset successfully."}
|
||||
|
||||
|
||||
|
||||
@router.api_route(
|
||||
"/{path:path}",
|
||||
methods=["GET", "POST", "PUT", "DELETE"],
|
||||
|
||||
@@ -389,6 +389,9 @@ class LightningInvoice(SQLModel, table=True): # type: ignore
|
||||
default=None, description="Associated API key hash for topup operations"
|
||||
)
|
||||
purpose: str = Field(description="create or topup")
|
||||
mint_url: str | None = Field(
|
||||
default=None, description="Mint URL where the quote was created (fallback tracking)"
|
||||
)
|
||||
created_at: int = Field(
|
||||
default_factory=lambda: int(time.time()), description="Unix timestamp"
|
||||
)
|
||||
|
||||
@@ -53,6 +53,18 @@ class Settings(BaseSettings):
|
||||
payout_interval_seconds: int = Field(
|
||||
default=900, gt=0, env="PAYOUT_INTERVAL_SECONDS"
|
||||
)
|
||||
# Timeout (seconds) for individual mint API operations (melt, mint, swap,
|
||||
# checkstate). When a mint is slow or rate-limiting, operations are
|
||||
# cancelled after this delay instead of hanging indefinitely.
|
||||
mint_operation_timeout_seconds: int = Field(
|
||||
default=30, gt=0, env="MINT_OPERATION_TIMEOUT_SECONDS"
|
||||
)
|
||||
# Maximum concurrent API operations per mint. Actual mint quotas vary by
|
||||
# endpoint, so 429 responses drive adaptive cooldown instead of fixed RPM
|
||||
# pacing. 0 = unlimited concurrency.
|
||||
mint_max_concurrency: int = Field(default=4, ge=0, env="MINT_MAX_CONCURRENCY")
|
||||
# Max retries when a mint returns 429 or times out (exponential backoff).
|
||||
mint_retry_max_attempts: int = Field(default=3, ge=0, env="MINT_RETRY_MAX_ATTEMPTS")
|
||||
|
||||
# Pricing
|
||||
# Default behavior: derive pricing from MODELS
|
||||
@@ -101,7 +113,9 @@ class Settings(BaseSettings):
|
||||
enable_pricing_refresh: bool = Field(default=True, env="ENABLE_PRICING_REFRESH")
|
||||
enable_models_refresh: bool = Field(default=True, env="ENABLE_MODELS_REFRESH")
|
||||
refund_cache_ttl_seconds: int = Field(default=3600, env="REFUND_CACHE_TTL_SECONDS")
|
||||
refund_sweep_ttl_seconds: int = Field(default=604800, env="REFUND_SWEEP_TTL_SECONDS")
|
||||
refund_sweep_ttl_seconds: int = Field(
|
||||
default=604800, env="REFUND_SWEEP_TTL_SECONDS"
|
||||
)
|
||||
refund_sweep_claim_timeout_seconds: int = Field(
|
||||
default=900, gt=0, env="REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS"
|
||||
)
|
||||
@@ -138,9 +152,8 @@ class Settings(BaseSettings):
|
||||
|
||||
# Discovery
|
||||
relays: list[str] = Field(default_factory=list, env="RELAYS")
|
||||
enable_analytics_sharing: bool = Field(
|
||||
default=True, env="ENABLE_ANALYTICS_SHARING"
|
||||
)
|
||||
enable_analytics_sharing: bool = Field(default=True, env="ENABLE_ANALYTICS_SHARING")
|
||||
|
||||
|
||||
def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Discard unknown keys from persisted settings."""
|
||||
|
||||
+363
-156
@@ -1,7 +1,10 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -12,12 +15,59 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
from .core.db import ApiKey, LightningInvoice, create_session, get_session
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
from .wallet import get_wallet, wallet_operation_guard
|
||||
from .wallet import (
|
||||
MintConnectionError,
|
||||
_is_mint_rate_limited,
|
||||
_mint_cooldown_remaining,
|
||||
_mint_operation,
|
||||
get_wallet,
|
||||
is_mint_connection_error,
|
||||
wallet_operation_guard,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
lightning_router = APIRouter(prefix="/lightning")
|
||||
|
||||
# Avoid duplicate work within one process. Cross-process credit fencing is done
|
||||
# by the conditional pending -> paid update in _finalize_invoice_settlement().
|
||||
_invoice_settlement_locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _InvoiceSettlement:
|
||||
id: str
|
||||
payment_hash: str
|
||||
amount_sats: int
|
||||
purpose: str
|
||||
api_key_hash: str | None
|
||||
mint_url: str | None
|
||||
balance_limit: int | None
|
||||
balance_limit_reset: str | None
|
||||
validity_date: int | None
|
||||
|
||||
@classmethod
|
||||
def from_invoice(cls, invoice: LightningInvoice) -> "_InvoiceSettlement":
|
||||
return cls(
|
||||
id=invoice.id,
|
||||
payment_hash=invoice.payment_hash,
|
||||
amount_sats=invoice.amount_sats,
|
||||
purpose=invoice.purpose,
|
||||
api_key_hash=invoice.api_key_hash,
|
||||
mint_url=invoice.mint_url,
|
||||
balance_limit=invoice.balance_limit,
|
||||
balance_limit_reset=invoice.balance_limit_reset,
|
||||
validity_date=invoice.validity_date,
|
||||
)
|
||||
|
||||
|
||||
def _publish_invoice_value(invoice: LightningInvoice, key: str, value: Any) -> None:
|
||||
"""Update a caller view without marking a mapped object dirty."""
|
||||
try:
|
||||
set_committed_value(invoice, key, value)
|
||||
except AttributeError:
|
||||
setattr(invoice, key, value)
|
||||
|
||||
|
||||
class InvoiceCreateRequest(BaseModel):
|
||||
amount_sats: int = Field(gt=0, le=1_000_000, description="Amount in satoshis")
|
||||
@@ -65,12 +115,80 @@ class InvoiceRecoverRequest(BaseModel):
|
||||
bolt11: str = Field(description="BOLT11 invoice string")
|
||||
|
||||
|
||||
def _trusted_mint_candidates() -> list[str]:
|
||||
return [
|
||||
mint
|
||||
for mint in dict.fromkeys([settings.primary_mint, *settings.cashu_mints])
|
||||
if mint
|
||||
]
|
||||
|
||||
|
||||
async def _request_mint_with_fallback(
|
||||
amount_sats: int,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
"""Request a quote, falling back only among the allowed trusted mints.
|
||||
|
||||
Guards against amount_sats <= 0: the cashu library's PostMintQuoteRequest
|
||||
enforces ``amount > 0`` (Pydantic Field(gt=0)), so passing 0 raises a
|
||||
cryptic validation error deep in the stack. Fail fast with context.
|
||||
"""
|
||||
if amount_sats <= 0:
|
||||
raise ValueError(
|
||||
f"generate_lightning_invoice: amount_sats must be > 0, got {amount_sats}."
|
||||
)
|
||||
tried: list[str] = []
|
||||
configured = allowed_mints or [settings.primary_mint, *settings.cashu_mints]
|
||||
candidates = list(dict.fromkeys(configured))
|
||||
for mint_url in candidates:
|
||||
cooldown = _mint_cooldown_remaining(mint_url)
|
||||
if cooldown > 0:
|
||||
tried.append(f"{mint_url}: cooling down")
|
||||
logger.info(
|
||||
"Skipping rate-limited mint",
|
||||
extra={
|
||||
"mint_url": mint_url,
|
||||
"cooldown_seconds": round(cooldown, 2),
|
||||
"op_name": "request_mint_invoice",
|
||||
},
|
||||
)
|
||||
continue
|
||||
try:
|
||||
wallet = await get_wallet(mint_url, "sat", retry_on_rate_limit=False)
|
||||
quote = await _mint_operation(
|
||||
lambda: wallet.request_mint(amount_sats),
|
||||
op_name="request_mint_invoice",
|
||||
mint_url=mint_url,
|
||||
retry_on_rate_limit=False,
|
||||
)
|
||||
return quote.request, quote.quote, mint_url
|
||||
except Exception as e:
|
||||
tried.append(f"{mint_url}: {type(e).__name__}")
|
||||
if not is_mint_connection_error(e) and not _is_mint_rate_limited(e):
|
||||
raise
|
||||
logger.warning(
|
||||
"request_mint failed, trying fallback mint",
|
||||
extra={
|
||||
"failed_mint": mint_url,
|
||||
"error": str(e),
|
||||
"tried": tried,
|
||||
},
|
||||
)
|
||||
continue
|
||||
raise MintConnectionError(f"All mints failed for request_mint: {tried}")
|
||||
|
||||
|
||||
async def generate_lightning_invoice(
|
||||
amount_sats: int, description: str
|
||||
) -> tuple[str, str]:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
quote = await wallet.request_mint(amount_sats)
|
||||
return quote.request, quote.quote
|
||||
amount_sats: int,
|
||||
description: str,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
bolt11, payment_hash, mint_url = await _request_mint_with_fallback(
|
||||
amount_sats, allowed_mints=allowed_mints
|
||||
)
|
||||
return bolt11, payment_hash, mint_url
|
||||
|
||||
|
||||
def generate_invoice_id() -> str:
|
||||
@@ -84,6 +202,7 @@ async def create_invoice(
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> InvoiceCreateResponse:
|
||||
api_key_token = _extract_bearer_api_key(authorization) or request.api_key
|
||||
topup_api_key: ApiKey | None = None
|
||||
|
||||
if request.purpose == "topup":
|
||||
if not api_key_token:
|
||||
@@ -94,14 +213,23 @@ async def create_invoice(
|
||||
if not api_key_token.startswith("sk-"):
|
||||
raise HTTPException(status_code=400, detail="Invalid API key format")
|
||||
|
||||
api_key = await session.get(ApiKey, api_key_token[3:])
|
||||
if not api_key:
|
||||
topup_api_key = await session.get(ApiKey, api_key_token[3:])
|
||||
if not topup_api_key:
|
||||
raise HTTPException(status_code=404, detail="API key not found")
|
||||
|
||||
try:
|
||||
description = f"Routstr {request.purpose} {request.amount_sats} sats"
|
||||
bolt11, payment_hash = await generate_lightning_invoice(
|
||||
request.amount_sats, description
|
||||
allowed_mints = None
|
||||
if request.purpose == "topup":
|
||||
assert topup_api_key is not None
|
||||
# A key's liabilities are attributed to a single refund mint. Keep
|
||||
# top-up collateral on that same mint so balances and payouts cannot
|
||||
# misclassify funds held by another mint as owner profit.
|
||||
allowed_mints = [
|
||||
topup_api_key.refund_mint_url or settings.primary_mint
|
||||
]
|
||||
bolt11, payment_hash, mint_url = await generate_lightning_invoice(
|
||||
request.amount_sats, description, allowed_mints=allowed_mints
|
||||
)
|
||||
|
||||
invoice_id = generate_invoice_id()
|
||||
@@ -116,6 +244,7 @@ async def create_invoice(
|
||||
status="pending",
|
||||
api_key_hash=api_key_token[3:] if api_key_token else None,
|
||||
purpose=request.purpose,
|
||||
mint_url=mint_url,
|
||||
balance_limit=request.balance_limit,
|
||||
balance_limit_reset=request.balance_limit_reset,
|
||||
validity_date=request.validity_date,
|
||||
@@ -223,158 +352,180 @@ async def recover_invoice(
|
||||
async def check_invoice_payment(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> None:
|
||||
# Minting makes proofs visible before database finalization. Share the
|
||||
# cross-process wallet guard with owner payout so that visibility and the
|
||||
# corresponding liability commit are observed atomically by the payout loop.
|
||||
async with wallet_operation_guard():
|
||||
await _check_invoice_payment_locked(invoice, session)
|
||||
|
||||
|
||||
async def _check_invoice_payment_locked(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> None:
|
||||
minted = False
|
||||
invoice_id = invoice.id
|
||||
invoice_purpose = invoice.purpose
|
||||
invoice_amount_sats = invoice.amount_sats
|
||||
invoice_payment_hash = invoice.payment_hash
|
||||
finalized_api_key_hash = invoice.api_key_hash
|
||||
try:
|
||||
# A preceding invoice lookup starts a transaction. End it before the
|
||||
# potentially slow mint request so it cannot pin a pool connection.
|
||||
await session.commit()
|
||||
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
mint_status = await wallet.get_mint_quote(invoice_payment_hash)
|
||||
if not mint_status.paid:
|
||||
return
|
||||
|
||||
# Do not redeem a paid top-up quote if its target has already been
|
||||
# pruned. This validation owns a short-lived session and releases its
|
||||
# connection before mint redemption starts.
|
||||
if invoice_purpose == "topup":
|
||||
if not finalized_api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
async with create_session() as validation_session:
|
||||
target = await validation_session.get(ApiKey, finalized_api_key_hash)
|
||||
if target is None:
|
||||
terminal = await validation_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice_id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(status="reconciliation_required")
|
||||
)
|
||||
await validation_session.commit()
|
||||
if terminal.rowcount == 1:
|
||||
set_committed_value(
|
||||
invoice, "status", "reconciliation_required"
|
||||
)
|
||||
else:
|
||||
committed_invoice = await validation_session.get(
|
||||
LightningInvoice, invoice_id
|
||||
)
|
||||
if committed_invoice is not None:
|
||||
set_committed_value(
|
||||
invoice, "status", committed_invoice.status
|
||||
)
|
||||
logger.critical(
|
||||
"Paid topup invoice target API key was not found; reconciliation required",
|
||||
extra={"invoice_id": invoice_id},
|
||||
)
|
||||
return
|
||||
|
||||
# The mint enforces single-use quotes, so a concurrent checker that
|
||||
# races us here fails inside wallet.mint rather than double-minting.
|
||||
await wallet.mint(invoice_amount_sats, quote_id=invoice_payment_hash)
|
||||
minted = True
|
||||
|
||||
# Paid finalization owns a fresh session. The API/watcher session is
|
||||
# never rolled back by this function, so its invoice and sibling ORM
|
||||
# objects remain usable after a DB failure or lost CAS race.
|
||||
async with create_session() as finalization_session:
|
||||
if invoice_purpose == "create":
|
||||
api_key = await _create_api_key_record(invoice, finalization_session)
|
||||
finalized_api_key_hash = api_key.hashed_key
|
||||
elif invoice_purpose == "topup":
|
||||
await _credit_topup_record(invoice, finalization_session)
|
||||
|
||||
# Conditional transition guards against double-credit: the credit
|
||||
# above and this status flip commit atomically, and a lost race
|
||||
# rolls both back in the owned finalization session.
|
||||
paid_at = int(time.time())
|
||||
finalized = await finalization_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice_id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(
|
||||
status="paid",
|
||||
paid_at=paid_at,
|
||||
api_key_hash=finalized_api_key_hash,
|
||||
)
|
||||
)
|
||||
if finalized.rowcount != 1:
|
||||
await finalization_session.rollback()
|
||||
committed_invoice = await finalization_session.get(
|
||||
LightningInvoice, invoice_id
|
||||
)
|
||||
await finalization_session.commit()
|
||||
if committed_invoice is not None:
|
||||
# A concurrent finalizer won the CAS. Publish only the
|
||||
# state observed from the database after ending the owned
|
||||
# read transaction; never refresh the caller's session.
|
||||
set_committed_value(
|
||||
invoice, "api_key_hash", committed_invoice.api_key_hash
|
||||
)
|
||||
set_committed_value(invoice, "status", committed_invoice.status)
|
||||
set_committed_value(invoice, "paid_at", committed_invoice.paid_at)
|
||||
lock = _invoice_settlement_locks.setdefault(invoice.id, asyncio.Lock())
|
||||
async with lock, wallet_operation_guard():
|
||||
minted = False
|
||||
try:
|
||||
# Snapshot the row and end the caller's read transaction before any
|
||||
# potentially slow mint I/O. All final DB mutations use owned,
|
||||
# short-lived sessions below.
|
||||
await session.refresh(invoice)
|
||||
if invoice.status != "pending":
|
||||
await session.commit()
|
||||
return
|
||||
await finalization_session.commit()
|
||||
settlement = _InvoiceSettlement.from_invoice(invoice)
|
||||
await session.commit()
|
||||
|
||||
# Only publish finalized values to the caller-owned object after the
|
||||
# owned transaction has committed successfully.
|
||||
set_committed_value(invoice, "api_key_hash", finalized_api_key_hash)
|
||||
set_committed_value(invoice, "status", "paid")
|
||||
set_committed_value(invoice, "paid_at", paid_at)
|
||||
|
||||
logger.info(
|
||||
"Lightning invoice paid",
|
||||
extra={
|
||||
"invoice_id": invoice_id,
|
||||
"amount_sats": invoice_amount_sats,
|
||||
"purpose": invoice_purpose,
|
||||
"api_key_hash": finalized_api_key_hash[:8] + "..."
|
||||
if finalized_api_key_hash
|
||||
else None,
|
||||
},
|
||||
)
|
||||
except BaseException as e:
|
||||
# BaseException so task cancellation (e.g. client disconnect) after a
|
||||
# successful mint still triggers the reconciliation alert. Any rollback
|
||||
# belongs to create_session(), never to the caller-owned session.
|
||||
if minted:
|
||||
logger.critical(
|
||||
"Invoice mint succeeded but DB finalization failed; reconciliation required",
|
||||
extra={"invoice_id": invoice_id, "purpose": invoice_purpose},
|
||||
mint_url = settlement.mint_url or settings.primary_mint
|
||||
wallet = await get_wallet(mint_url, "sat")
|
||||
mint_status = await _mint_operation(
|
||||
lambda: wallet.get_mint_quote(settlement.payment_hash),
|
||||
op_name="get_mint_quote",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
if not isinstance(e, Exception):
|
||||
if not mint_status.paid:
|
||||
return
|
||||
|
||||
# Reject a paid top-up whose target was pruned before redeeming its
|
||||
# single-use quote. The validation session is closed before mint I/O.
|
||||
if settlement.purpose == "topup":
|
||||
if not settlement.api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
async with create_session() as validation_session:
|
||||
target = await validation_session.get(
|
||||
ApiKey, settlement.api_key_hash
|
||||
)
|
||||
if target is None:
|
||||
terminal = await validation_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == settlement.id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(status="reconciliation_required")
|
||||
)
|
||||
await validation_session.commit()
|
||||
if terminal.rowcount == 1:
|
||||
_publish_invoice_value(
|
||||
invoice, "status", "reconciliation_required"
|
||||
)
|
||||
else:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
logger.critical(
|
||||
"Paid topup invoice target API key was not found; reconciliation required",
|
||||
extra={"invoice_id": settlement.id},
|
||||
)
|
||||
return
|
||||
|
||||
# Quote-linked proof verification makes an ambiguous mint response
|
||||
# retryable without crediting unrelated wallet balance growth.
|
||||
await _mint_invoice_quote(wallet, settlement)
|
||||
minted = True
|
||||
|
||||
paid_at = int(time.time())
|
||||
async with create_session() as finalization_session:
|
||||
settled, api_key_hash = await _finalize_invoice_settlement(
|
||||
settlement, finalization_session, paid_at
|
||||
)
|
||||
if not settled:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
return
|
||||
|
||||
_publish_invoice_value(invoice, "status", "paid")
|
||||
_publish_invoice_value(invoice, "paid_at", paid_at)
|
||||
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
|
||||
logger.info(
|
||||
"Lightning invoice paid",
|
||||
extra={
|
||||
"invoice_id": settlement.id,
|
||||
"amount_sats": settlement.amount_sats,
|
||||
"purpose": settlement.purpose,
|
||||
"api_key_hash": api_key_hash[:8] + "..."
|
||||
if api_key_hash
|
||||
else None,
|
||||
},
|
||||
)
|
||||
except BaseException as error:
|
||||
# Never roll back the caller-owned session: doing so expires invoice
|
||||
# and sibling ORM objects. Owned sessions roll themselves back.
|
||||
if minted:
|
||||
logger.critical(
|
||||
"Invoice mint succeeded but DB finalization failed; reconciliation required",
|
||||
extra={"invoice_id": invoice.id, "purpose": invoice.purpose},
|
||||
)
|
||||
try:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
except Exception:
|
||||
pass
|
||||
if not isinstance(error, Exception):
|
||||
raise
|
||||
logger.error(f"Failed to check invoice payment: {error}")
|
||||
|
||||
|
||||
def _is_outputs_already_signed(error: BaseException) -> bool:
|
||||
message = str(error)
|
||||
return bool(
|
||||
re.search(
|
||||
r"\boutputs?\s+(?:have\s+)?already\s+(?:been\s+)?signed(?:\s+before)?\b",
|
||||
message,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
and re.search(r"\bcode\s*:\s*11003\b", message, re.IGNORECASE)
|
||||
)
|
||||
|
||||
|
||||
def _invoice_quote_proof_amount(wallet: Any, quote_id: str) -> int:
|
||||
"""Return spendable wallet value minted by one Lightning quote."""
|
||||
return sum(
|
||||
proof.amount
|
||||
for proof in wallet.proofs
|
||||
if proof.mint_id == quote_id and not proof.reserved
|
||||
)
|
||||
|
||||
|
||||
async def _mint_invoice_quote(
|
||||
wallet: Any, invoice: LightningInvoice | _InvoiceSettlement
|
||||
) -> None:
|
||||
"""Mint a paid quote, proving quote-linked outputs before DB credit."""
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
await wallet.load_proofs(reload=True)
|
||||
if _invoice_quote_proof_amount(wallet, invoice.payment_hash) >= invoice.amount_sats:
|
||||
return
|
||||
|
||||
try:
|
||||
await _mint_operation(
|
||||
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
|
||||
op_name=f"invoice_mint_{invoice.purpose}",
|
||||
mint_url=mint_url,
|
||||
retry_timeouts=False,
|
||||
)
|
||||
except Exception as error:
|
||||
if not _is_outputs_already_signed(error):
|
||||
raise
|
||||
logger.error(f"Failed to check invoice payment: {e}")
|
||||
|
||||
for keyset_id in wallet.keysets:
|
||||
await wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
|
||||
await wallet.load_proofs(reload=True)
|
||||
recovered = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
|
||||
if recovered < invoice.amount_sats:
|
||||
raise RuntimeError(
|
||||
"Invoice outputs were already signed but quote-linked recovery returned "
|
||||
f"{recovered} sats; expected at least {invoice.amount_sats}"
|
||||
) from error
|
||||
else:
|
||||
await wallet.load_proofs(reload=True)
|
||||
minted_amount = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
|
||||
if minted_amount < invoice.amount_sats:
|
||||
raise RuntimeError(
|
||||
"Invoice mint succeeded but quote-linked proofs total "
|
||||
f"{minted_amount} sats; expected at least {invoice.amount_sats}"
|
||||
)
|
||||
|
||||
|
||||
def _invoice_api_key_hash(invoice: LightningInvoice | _InvoiceSettlement) -> str:
|
||||
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
|
||||
return hashlib.sha256(dummy_token.encode()).hexdigest()
|
||||
|
||||
|
||||
async def _create_api_key_record(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> ApiKey:
|
||||
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
|
||||
hashed_key = hashlib.sha256(dummy_token.encode()).hexdigest()
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
api_key = ApiKey(
|
||||
hashed_key=hashed_key,
|
||||
hashed_key=_invoice_api_key_hash(invoice),
|
||||
balance=invoice.amount_sats * 1000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url=settings.primary_mint,
|
||||
refund_mint_url=mint_url,
|
||||
balance_limit=invoice.balance_limit,
|
||||
balance_limit_reset=invoice.balance_limit_reset,
|
||||
validity_date=invoice.validity_date,
|
||||
@@ -384,21 +535,77 @@ async def _create_api_key_record(
|
||||
return api_key
|
||||
|
||||
|
||||
async def _credit_topup_record(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
async def _topup_api_key_record(
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> None:
|
||||
if not invoice.api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
credited = await session.exec( # type: ignore[call-overload]
|
||||
result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == invoice.api_key_hash)
|
||||
.values(balance=col(ApiKey.balance) + invoice.amount_sats * 1000)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if credited.rowcount != 1:
|
||||
if result.rowcount != 1:
|
||||
raise ValueError("Associated API key not found")
|
||||
|
||||
|
||||
INVOICE_WATCH_INTERVAL_SECONDS = 5
|
||||
async def _finalize_invoice_settlement(
|
||||
invoice: _InvoiceSettlement, session: AsyncSession, paid_at: int
|
||||
) -> tuple[bool, str | None]:
|
||||
"""Atomically fence and apply one invoice credit in the provided owned session."""
|
||||
api_key_hash = (
|
||||
_invoice_api_key_hash(invoice)
|
||||
if invoice.purpose == "create"
|
||||
else invoice.api_key_hash
|
||||
)
|
||||
claim = await session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(col(LightningInvoice.id) == invoice.id)
|
||||
.where(col(LightningInvoice.status) == "pending")
|
||||
.values(status="paid", paid_at=paid_at, api_key_hash=api_key_hash)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if claim.rowcount != 1:
|
||||
await session.rollback()
|
||||
return False, None
|
||||
|
||||
if invoice.purpose == "create":
|
||||
await _create_api_key_record(invoice, session)
|
||||
elif invoice.purpose == "topup":
|
||||
await _topup_api_key_record(invoice, session)
|
||||
else:
|
||||
raise ValueError(f"Unsupported invoice purpose: {invoice.purpose}")
|
||||
await session.commit()
|
||||
return True, api_key_hash
|
||||
|
||||
|
||||
async def _reload_invoice_view(
|
||||
invoice: LightningInvoice, _caller_session: AsyncSession
|
||||
) -> None:
|
||||
"""Publish committed invoice state without touching the caller transaction."""
|
||||
async with create_session() as reload_session:
|
||||
stored = await reload_session.get(LightningInvoice, invoice.id)
|
||||
if stored is None:
|
||||
return
|
||||
status = stored.status
|
||||
paid_at = stored.paid_at
|
||||
api_key_hash = stored.api_key_hash
|
||||
await reload_session.commit()
|
||||
_publish_invoice_value(invoice, "status", status)
|
||||
_publish_invoice_value(invoice, "paid_at", paid_at)
|
||||
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
|
||||
|
||||
|
||||
async def _credit_topup_record(
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> None:
|
||||
await _topup_api_key_record(invoice, session)
|
||||
|
||||
|
||||
# Nutshell mints throttle Lightning backend lookups to once per 10s per
|
||||
# quote, so polling faster just burns the global request budget for nothing.
|
||||
INVOICE_WATCH_INTERVAL_SECONDS = 10
|
||||
INVOICE_WATCH_BATCH_LIMIT = 100
|
||||
|
||||
|
||||
|
||||
@@ -18,6 +18,17 @@ from ..wallet import deserialize_token_from_string
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Interim policy: when Routstr must move value to another trusted mint, the
|
||||
# cross-mint Lightning round trip can consume fees that are not visible to the
|
||||
# client. Reserve 5% headroom until the fee-payer policy is made explicit.
|
||||
_MINT_FEE_ALLOWANCE = 0.05
|
||||
|
||||
|
||||
def apply_mint_fee_allowance(cost_msat: int) -> int:
|
||||
"""Reserve headroom for possible trusted-mint fallback fees."""
|
||||
adjusted = math.ceil(cost_msat * (1 - _MINT_FEE_ALLOWANCE))
|
||||
return max(settings.min_request_msat, adjusted)
|
||||
|
||||
|
||||
def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> None:
|
||||
if x_cashu := headers.get("x-cashu", None):
|
||||
|
||||
@@ -9,7 +9,8 @@ from cashu.wallet.wallet import Proof, Wallet
|
||||
|
||||
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
|
||||
# very slow mint can block a melt (and any caller, e.g. the payout loop)
|
||||
# indefinitely. Bound it here so callers fail instead of hanging forever.
|
||||
# indefinitely. _mint_operation (imported lazily in raw_send_to_lnurl to avoid
|
||||
# a circular import with wallet.py) bounds it via MINT_OPERATION_TIMEOUT_SECONDS.
|
||||
MELT_TIMEOUT_SECONDS = 60
|
||||
|
||||
try:
|
||||
@@ -221,22 +222,33 @@ async def raw_send_to_lnurl(
|
||||
lnurl_data["callback_url"], final_amount
|
||||
)
|
||||
|
||||
melt_quote_resp = await wallet.melt_quote(invoice=bolt11_invoice)
|
||||
from ..wallet import _mint_operation
|
||||
|
||||
melt_quote_resp = await _mint_operation(
|
||||
lambda: wallet.melt_quote(invoice=bolt11_invoice),
|
||||
op_name="lnurl_melt_quote",
|
||||
mint_url=str(wallet.url),
|
||||
)
|
||||
|
||||
if amount:
|
||||
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
|
||||
|
||||
try:
|
||||
_ = await asyncio.wait_for(
|
||||
wallet.melt(
|
||||
proofs=proofs,
|
||||
invoice=bolt11_invoice,
|
||||
fee_reserve_sat=melt_quote_resp.fee_reserve,
|
||||
quote_id=melt_quote_resp.quote,
|
||||
_mint_operation(
|
||||
lambda: wallet.melt(
|
||||
proofs=proofs,
|
||||
invoice=bolt11_invoice,
|
||||
fee_reserve_sat=melt_quote_resp.fee_reserve,
|
||||
quote_id=melt_quote_resp.quote,
|
||||
),
|
||||
op_name="lnurl_melt",
|
||||
mint_url=str(wallet.url),
|
||||
retry_timeouts=False,
|
||||
),
|
||||
timeout=MELT_TIMEOUT_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError as e:
|
||||
except (httpx.TimeoutException, asyncio.TimeoutError) as e:
|
||||
raise LNURLError(
|
||||
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
|
||||
) from e
|
||||
|
||||
+25
-12
@@ -26,8 +26,8 @@ from .core.db import (
|
||||
)
|
||||
from .core.exceptions import UpstreamError
|
||||
from .core.not_found import build_not_found_response
|
||||
from .core.settings import settings
|
||||
from .payment.helpers import (
|
||||
apply_mint_fee_allowance,
|
||||
calculate_discounted_max_cost,
|
||||
check_token_balance,
|
||||
create_error_response,
|
||||
@@ -354,8 +354,7 @@ async def _proxy(
|
||||
max_cost_for_model = await calculate_discounted_max_cost(
|
||||
_max_cost_for_model, request_body_dict, model_obj=model_obj
|
||||
)
|
||||
# Ensure max_cost_for_model is at least the minimum allowed request cost
|
||||
max_cost_for_model = max(max_cost_for_model, settings.min_request_msat)
|
||||
max_cost_for_model = apply_mint_fee_allowance(max_cost_for_model)
|
||||
|
||||
check_token_balance(headers, request_body_dict, max_cost_for_model)
|
||||
|
||||
@@ -494,7 +493,9 @@ async def _proxy(
|
||||
candidate_max = await calculate_discounted_max_cost(
|
||||
candidate_max, request_body_dict, model_obj=model_obj
|
||||
)
|
||||
candidate_max = max(candidate_max, settings.min_request_msat)
|
||||
# Apply the same interim 5% trusted-mint fee headroom used for the
|
||||
# first candidate; failover must not silently change admission.
|
||||
candidate_max = apply_mint_fee_allowance(candidate_max)
|
||||
if candidate_max > max_cost_for_model:
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
@@ -794,17 +795,29 @@ async def get_bearer_token_key(
|
||||
},
|
||||
)
|
||||
return key
|
||||
except Exception as e:
|
||||
key_preview = bearer_key[:20] + "..." if len(bearer_key) > 20 else bearer_key
|
||||
logger.error(
|
||||
f"Bearer token validation failed: {type(e).__name__}: {e} path={path} model={model_id!r} min_cost={min_cost} key={key_preview!r}",
|
||||
except HTTPException as error:
|
||||
detail: dict[str, Any] = error.detail if isinstance(error.detail, dict) else {}
|
||||
raw_error = detail.get("error")
|
||||
error_info = raw_error if isinstance(raw_error, dict) else {}
|
||||
logger.warning(
|
||||
"Bearer token rejected",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
"status_code": error.status_code,
|
||||
"error_code": error_info.get("code"),
|
||||
"path": path,
|
||||
"model_id": model_id,
|
||||
"min_cost_msat": min_cost,
|
||||
"bearer_key_preview": key_preview,
|
||||
"required_msat": min_cost,
|
||||
},
|
||||
)
|
||||
raise
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
"Bearer token validation failed",
|
||||
extra={
|
||||
"error_type": type(error).__name__,
|
||||
"path": path,
|
||||
"model_id": model_id,
|
||||
"required_msat": min_cost,
|
||||
},
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -12,7 +12,7 @@ from ..core.db import (
|
||||
from ..core.db import (
|
||||
store_cashu_transaction_with_retry as store_cashu_transaction,
|
||||
)
|
||||
from ..wallet import send_token
|
||||
from ..wallet import release_token_reservation, send_token, token_mint_url
|
||||
from .routstr import RoutstrUpstreamProvider
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -144,12 +144,13 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
|
||||
)
|
||||
return
|
||||
|
||||
actual_mint_url = token_mint_url(token, mint_url)
|
||||
try:
|
||||
await store_cashu_transaction(
|
||||
token=token,
|
||||
amount=amount,
|
||||
unit="sat",
|
||||
mint_url=mint_url,
|
||||
mint_url=actual_mint_url,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="auto_topup",
|
||||
@@ -157,8 +158,24 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
|
||||
except Exception:
|
||||
logger.critical(
|
||||
"Aborting auto top-up because its cashu token could not be persisted",
|
||||
extra={"provider_id": row.id, "mint_url": mint_url},
|
||||
extra={"provider_id": row.id, "mint_url": actual_mint_url},
|
||||
)
|
||||
try:
|
||||
await release_token_reservation(token)
|
||||
except Exception as error:
|
||||
logger.critical(
|
||||
"Failed to release untracked auto-topup token",
|
||||
extra={
|
||||
"provider_id": row.id,
|
||||
"mint_url": actual_mint_url,
|
||||
"error": str(error),
|
||||
},
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Auto-topup token was released after persistence failed",
|
||||
extra={"provider_id": row.id, "mint_url": actual_mint_url},
|
||||
)
|
||||
return
|
||||
|
||||
result = await provider.topup(token)
|
||||
|
||||
@@ -53,6 +53,7 @@ from ..wallet import (
|
||||
classify_redemption_error,
|
||||
recieve_token,
|
||||
send_token,
|
||||
token_mint_url,
|
||||
)
|
||||
from . import messages_dispatch
|
||||
from .cache_breakpoints import (
|
||||
@@ -3520,7 +3521,7 @@ class BaseUpstreamProvider:
|
||||
token=refund_token,
|
||||
amount=amount,
|
||||
unit=unit,
|
||||
mint_url=mint,
|
||||
mint_url=token_mint_url(refund_token, mint),
|
||||
typ="out",
|
||||
request_id=request_id,
|
||||
)
|
||||
@@ -3873,7 +3874,7 @@ class BaseUpstreamProvider:
|
||||
token=refund_token,
|
||||
amount=emergency_refund,
|
||||
unit=unit,
|
||||
mint_url=mint,
|
||||
mint_url=token_mint_url(refund_token, mint),
|
||||
typ="out",
|
||||
request_id=request_id,
|
||||
)
|
||||
@@ -4843,7 +4844,7 @@ class BaseUpstreamProvider:
|
||||
token=refund_token,
|
||||
amount=emergency_refund,
|
||||
unit=unit,
|
||||
mint_url=mint,
|
||||
mint_url=token_mint_url(refund_token, mint),
|
||||
typ="out",
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
+1076
-128
File diff suppressed because it is too large
Load Diff
@@ -207,8 +207,30 @@ async def test_pay_for_request_succeeds_when_balance_equals_cost(
|
||||
assert key.balance == model_cost # balance unchanged, only reserved goes up
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_five_percent_mint_fallback_headroom_is_admitted_and_reserved(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
from routstr.auth import pay_for_request, validate_bearer_key
|
||||
from routstr.payment.helpers import apply_mint_fee_allowance
|
||||
|
||||
key = _key(balance=95_000)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
|
||||
admission_cost = apply_mint_fee_allowance(100_000)
|
||||
validated = await validate_bearer_key(
|
||||
f"sk-{key.hashed_key}", integration_session, min_cost=admission_cost
|
||||
)
|
||||
await pay_for_request(validated, admission_cost, integration_session)
|
||||
|
||||
await integration_session.refresh(key)
|
||||
assert admission_cost == 95_000
|
||||
assert key.reserved_balance == 95_000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 6 — HTTP layer returns 402 JSON with the right shape
|
||||
# HTTP layer returns 402 JSON with the right shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -266,8 +288,8 @@ async def test_http_402_response_shape_on_insufficient_balance(
|
||||
error = body["detail"]["error"]
|
||||
assert error["code"] == "insufficient_balance"
|
||||
assert error["type"] == "insufficient_quota"
|
||||
assert str(model_cost) in error["message"]
|
||||
assert str(user_balance) in error["message"]
|
||||
assert "591.744 sats (591744 msats) required" in error["message"]
|
||||
assert "20.32 sats (20320 msats) available" in error["message"]
|
||||
|
||||
# Balance must be completely untouched
|
||||
await integration_session.refresh(key)
|
||||
|
||||
@@ -14,6 +14,7 @@ import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import Proof
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
@@ -22,6 +23,12 @@ from routstr.core.db import ApiKey, LightningInvoice
|
||||
from routstr.lightning import _create_api_key_record
|
||||
|
||||
|
||||
def _configure_quote_proof_wallet(wallet: MagicMock) -> None:
|
||||
wallet.proofs = []
|
||||
wallet.keysets = {}
|
||||
wallet.load_proofs = AsyncMock()
|
||||
|
||||
|
||||
def _make_invoice(**kwargs: object) -> LightningInvoice:
|
||||
base = dict(
|
||||
id="inv_test_001",
|
||||
@@ -42,7 +49,15 @@ def _make_invoice(**kwargs: object) -> LightningInvoice:
|
||||
def mock_wallet_mint() -> object:
|
||||
with patch("routstr.lightning.get_wallet") as mock_get_wallet:
|
||||
wallet = AsyncMock()
|
||||
wallet.mint = AsyncMock(return_value=[])
|
||||
wallet.proofs = []
|
||||
wallet.load_proofs = AsyncMock()
|
||||
|
||||
async def mint(amount: int, quote_id: str) -> list[Proof]:
|
||||
proofs = [Proof(amount=amount, mint_id=quote_id)]
|
||||
wallet.proofs.extend(proofs)
|
||||
return proofs
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=mint)
|
||||
mock_get_wallet.return_value = wallet
|
||||
yield mock_get_wallet
|
||||
|
||||
@@ -183,6 +198,7 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
|
||||
mint_calls = 0
|
||||
@@ -196,7 +212,9 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
|
||||
await asyncio.sleep(0.05)
|
||||
if call_number > 1:
|
||||
raise Exception("quote already issued")
|
||||
return []
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=single_use_mint)
|
||||
|
||||
@@ -238,6 +256,7 @@ async def test_failed_mint_keeps_invoice_pending_for_retry(
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
wallet.mint = AsyncMock(side_effect=TimeoutError("mint unavailable"))
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
@@ -349,8 +368,15 @@ async def test_post_mint_db_failure_keeps_invoice_pending_for_reconciliation(
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
wallet.mint = AsyncMock(return_value=[])
|
||||
|
||||
async def successful_mint(*args: object, **kwargs: object) -> list[Proof]:
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=successful_mint)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
stored = await session.get(LightningInvoice, invoice.id)
|
||||
stored_sibling = await session.get(LightningInvoice, sibling.id)
|
||||
@@ -428,11 +454,14 @@ async def test_db_guard_credits_once_when_both_mints_succeed(
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
|
||||
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[object]:
|
||||
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[Proof]:
|
||||
await asyncio.sleep(0.05)
|
||||
return []
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=always_succeeding_mint)
|
||||
|
||||
@@ -472,7 +501,7 @@ async def test_db_guard_credits_once_when_both_mints_succeed(
|
||||
assert first_invoice not in first.dirty
|
||||
assert second_invoice not in second.dirty
|
||||
|
||||
assert wallet.mint.await_count == 2
|
||||
assert wallet.mint.await_count == 1
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_invoice = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored_invoice is not None
|
||||
|
||||
@@ -26,11 +26,17 @@ async def patch_invoice_generation() -> Any:
|
||||
"""Stub out `generate_lightning_invoice` so no mint round-trip is needed."""
|
||||
counter = {"n": 0}
|
||||
|
||||
async def fake_generate(amount_sats: int, description: str) -> tuple[str, str]:
|
||||
async def fake_generate(
|
||||
amount_sats: int,
|
||||
description: str,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
counter["n"] += 1
|
||||
return (
|
||||
f"lnbc{amount_sats}n1pfakeinvoice{counter['n']}",
|
||||
f"payment_hash_{counter['n']}",
|
||||
"http://localhost:3338",
|
||||
)
|
||||
|
||||
with patch(
|
||||
@@ -95,6 +101,8 @@ async def test_topup_with_authorization_header(
|
||||
body = resp.json()
|
||||
assert body["amount_sats"] == 500
|
||||
assert body["bolt11"].startswith("lnbc")
|
||||
allowed_mints = patch_invoice_generation.call_args.kwargs["allowed_mints"]
|
||||
assert allowed_mints == ["http://localhost:3338"]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import Proof
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlmodel import col, update
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey, LightningInvoice
|
||||
from routstr.lightning import (
|
||||
_finalize_invoice_settlement,
|
||||
_InvoiceSettlement,
|
||||
check_invoice_payment,
|
||||
)
|
||||
|
||||
|
||||
def _lightning_invoice(**overrides: object) -> LightningInvoice:
|
||||
suffix = uuid.uuid4().hex
|
||||
values = {
|
||||
"id": f"invoice-{suffix}",
|
||||
"bolt11": f"lnbc-{suffix}",
|
||||
"amount_sats": 100,
|
||||
"description": "settlement test",
|
||||
"payment_hash": f"quote-{suffix}",
|
||||
"status": "pending",
|
||||
"purpose": "create",
|
||||
"mint_url": "http://mint:3338",
|
||||
"expires_at": int(time.time()) + 3600,
|
||||
}
|
||||
values.update(overrides)
|
||||
return LightningInvoice(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_read_transaction_closes_before_external_mint_io(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
invoice = _lightning_invoice()
|
||||
integration_session.add(invoice)
|
||||
await integration_session.commit()
|
||||
stored = await integration_session.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=False)))
|
||||
|
||||
async def get_wallet_without_open_db_transaction(
|
||||
*args: object, **kwargs: object
|
||||
) -> Mock:
|
||||
assert not integration_session.in_transaction()
|
||||
return wallet
|
||||
|
||||
with patch(
|
||||
"routstr.lightning.get_wallet", side_effect=get_wallet_without_open_db_transaction
|
||||
):
|
||||
await check_invoice_payment(stored, integration_session)
|
||||
|
||||
assert not integration_session.in_transaction()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_separate_sessions_cas_topup_credit_exactly_once(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup",
|
||||
api_key_hash=key_hash,
|
||||
amount_sats=100,
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
snapshot_a = _InvoiceSettlement.from_invoice(invoice)
|
||||
snapshot_b = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with (
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as session_a,
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as session_b,
|
||||
):
|
||||
results = await asyncio.gather(
|
||||
_finalize_invoice_settlement(snapshot_a, session_a, 1_700_000_000),
|
||||
_finalize_invoice_settlement(snapshot_b, session_b, 1_700_000_001),
|
||||
)
|
||||
|
||||
assert sorted(settled for settled, _ in results) == [False, True]
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_invoice = await verify.get(LightningInvoice, invoice.id)
|
||||
stored_key = await verify.get(ApiKey, key_hash)
|
||||
assert stored_invoice is not None
|
||||
assert stored_invoice.status == "paid"
|
||||
assert stored_key is not None
|
||||
assert stored_key.balance == 200_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_atomic_increment_preserves_concurrent_balance_mutation(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup", api_key_hash=key_hash, amount_sats=100
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
async def debit_balance(session: AsyncSession) -> None:
|
||||
result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key_hash)
|
||||
.values(balance=col(ApiKey.balance) - 10_000)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
assert result.rowcount == 1
|
||||
await session.commit()
|
||||
|
||||
snapshot = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with (
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as settlement,
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as debit,
|
||||
):
|
||||
settlement_result, _ = await asyncio.gather(
|
||||
_finalize_invoice_settlement(snapshot, settlement, 1_700_000_000),
|
||||
debit_balance(debit),
|
||||
)
|
||||
|
||||
assert settlement_result[0]
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_key = await verify.get(ApiKey, key_hash)
|
||||
assert stored_key is not None
|
||||
assert stored_key.balance == 190_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_final_commit_rolls_back_claim_and_credit_for_retry(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup",
|
||||
api_key_hash=key_hash,
|
||||
amount_sats=100,
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
snapshot = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
|
||||
with patch.object(
|
||||
failed, "commit", AsyncMock(side_effect=Exception("db unavailable"))
|
||||
):
|
||||
with pytest.raises(Exception, match="db unavailable"):
|
||||
await _finalize_invoice_settlement(snapshot, failed, 1_700_000_000)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
pending = await verify.get(LightningInvoice, invoice.id)
|
||||
unchanged = await verify.get(ApiKey, key_hash)
|
||||
assert pending is not None
|
||||
assert pending.status == "pending"
|
||||
assert unchanged is not None
|
||||
assert unchanged.balance == 100_000
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
|
||||
settled, _ = await _finalize_invoice_settlement(
|
||||
snapshot, retry, 1_700_000_001
|
||||
)
|
||||
assert settled
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
paid = await verify.get(LightningInvoice, invoice.id)
|
||||
credited = await verify.get(ApiKey, key_hash)
|
||||
assert paid is not None
|
||||
assert paid.status == "paid"
|
||||
assert credited is not None
|
||||
assert credited.balance == 200_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_invoice_payment_retries_after_mint_success_and_db_failure(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup", api_key_hash=key_hash, amount_sats=100
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
wallet = Mock(
|
||||
proofs=[],
|
||||
keysets={"keyset-1": Mock()},
|
||||
load_proofs=AsyncMock(),
|
||||
get_mint_quote=AsyncMock(return_value=Mock(paid=True)),
|
||||
restore_tokens_for_keyset=AsyncMock(),
|
||||
)
|
||||
|
||||
async def mint(amount: int, quote_id: str) -> list[Proof]:
|
||||
proofs = [Proof(amount=amount, mint_id=quote_id)]
|
||||
wallet.proofs.extend(proofs)
|
||||
return proofs
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=mint)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
|
||||
stored = await failed.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch(
|
||||
"routstr.lightning._finalize_invoice_settlement",
|
||||
AsyncMock(side_effect=Exception("db unavailable")),
|
||||
),
|
||||
):
|
||||
await check_invoice_payment(stored, failed)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
pending = await verify.get(LightningInvoice, invoice.id)
|
||||
unchanged = await verify.get(ApiKey, key_hash)
|
||||
assert pending is not None
|
||||
assert pending.status == "pending"
|
||||
assert unchanged is not None
|
||||
assert unchanged.balance == 100_000
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
|
||||
stored = await retry.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
await check_invoice_payment(stored, retry)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
paid = await verify.get(LightningInvoice, invoice.id)
|
||||
credited = await verify.get(ApiKey, key_hash)
|
||||
assert paid is not None
|
||||
assert paid.status == "paid"
|
||||
assert credited is not None
|
||||
assert credited.balance == 200_000
|
||||
|
||||
wallet.mint.assert_awaited_once_with(100, quote_id=invoice.payment_hash)
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
@@ -89,7 +89,12 @@ def _make_swap_mocks(
|
||||
def _wallet_router(primary_wallet: Mock, token_wallet: Mock) -> Callable[..., Mock]:
|
||||
"""Route get_wallet calls to the primary or foreign wallet mock by URL."""
|
||||
|
||||
def fake_get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Mock:
|
||||
def fake_get_wallet(
|
||||
mint_url: str,
|
||||
unit: str = "sat",
|
||||
load: bool = True,
|
||||
**kwargs: object,
|
||||
) -> Mock:
|
||||
return primary_wallet if mint_url == PRIMARY_MINT else token_wallet
|
||||
|
||||
return fake_get_wallet
|
||||
|
||||
@@ -66,6 +66,10 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(return_value=True),
|
||||
) as store,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.token_mint_url",
|
||||
return_value="https://fallback-mint.test",
|
||||
),
|
||||
patch("routstr.upstream.auto_topup.create_session", return_value=session),
|
||||
):
|
||||
await _check_and_topup(_row())
|
||||
@@ -74,7 +78,7 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
|
||||
token="cashu-token",
|
||||
amount=50,
|
||||
unit="sat",
|
||||
mint_url="https://mint.test",
|
||||
mint_url="https://fallback-mint.test",
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="auto_topup",
|
||||
@@ -138,6 +142,12 @@ async def test_auto_topup_does_not_send_untracked_token() -> None:
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(side_effect=RuntimeError("database unavailable")),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.release_token_reservation",
|
||||
AsyncMock(),
|
||||
) as reclaim,
|
||||
):
|
||||
await _check_and_topup(_row())
|
||||
|
||||
reclaim.assert_awaited_once_with("cashu-token")
|
||||
provider.topup.assert_not_awaited()
|
||||
|
||||
@@ -534,6 +534,29 @@ async def test_topup_mint_unreachable_returns_503(error: Exception) -> None:
|
||||
assert exc_info.value.detail == "Cashu mint is unreachable"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_unreachable_source_mint_explains_why_fallback_is_impossible() -> None:
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.wallet import SourceMintConnectionError
|
||||
|
||||
key = _make_api_key(balance=1000)
|
||||
session = MagicMock()
|
||||
error = SourceMintConnectionError("Issuing Cashu mint is unreachable")
|
||||
|
||||
with (
|
||||
patch("routstr.balance.get_billing_key", AsyncMock(return_value=key)),
|
||||
patch("routstr.balance.credit_balance", AsyncMock(side_effect=error)),
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await topup_wallet_endpoint(
|
||||
cashu_token="cashuAtoken", key=key, session=session
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 503
|
||||
assert "cannot be redeemed at another mint" in exc_info.value.detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_already_spent_still_returns_400() -> None:
|
||||
"""Regression: the mint-unreachable short-circuit must not swallow the
|
||||
|
||||
@@ -37,7 +37,7 @@ def test_fresh_node_migrates_fee_payout_schema_to_head(tmp_path: Path) -> None:
|
||||
"payout_in_progress_msats, payout_started_at FROM routstr_fees"
|
||||
).fetchone()
|
||||
|
||||
assert version == ("aa50fde387a2",)
|
||||
assert version == ("bf76270b66c4",)
|
||||
assert {
|
||||
"id",
|
||||
"accumulated_msats",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import AsyncGenerator, Generator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlmodel import SQLModel
|
||||
@@ -12,6 +13,21 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
from routstr.wallet import fetch_all_balances
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_balance_fetch_state() -> Generator[None, None, None]:
|
||||
from routstr import wallet
|
||||
|
||||
wallet._balance_fetch_failures.clear()
|
||||
wallet._balance_fetch_locks.clear()
|
||||
wallet._mint_supported_units.clear()
|
||||
wallet._MintRateGuard._guards.clear()
|
||||
yield
|
||||
wallet._balance_fetch_failures.clear()
|
||||
wallet._balance_fetch_locks.clear()
|
||||
wallet._mint_supported_units.clear()
|
||||
wallet._MintRateGuard._guards.clear()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _fake_session(): # type: ignore[no-untyped-def]
|
||||
yield MagicMock()
|
||||
@@ -29,7 +45,7 @@ def _patches( # type: ignore[no-untyped-def]
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
AsyncMock(side_effect=lambda proofs, wallet, **kwargs: proofs),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.balances_by_mint_and_unit",
|
||||
@@ -63,6 +79,161 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
|
||||
assert total_wallet == 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_uses_units_advertised_by_mint() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch(
|
||||
"routstr.wallet._get_supported_mint_units",
|
||||
AsyncMock(return_value=["sat"]),
|
||||
) as supported_units,
|
||||
):
|
||||
for p in _patches(proof_amount=1000):
|
||||
p.start()
|
||||
try:
|
||||
details, *_ = await fetch_all_balances()
|
||||
finally:
|
||||
patch.stopall()
|
||||
|
||||
supported_units.assert_awaited_once_with("http://mint:3338")
|
||||
assert [detail["unit"] for detail in details] == ["sat"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unit_discovery_failure_returns_structured_balance_error() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
get_wallet = AsyncMock()
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch(
|
||||
"routstr.wallet._get_supported_mint_units",
|
||||
AsyncMock(side_effect=httpx.ConnectError("mint unavailable")),
|
||||
),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
):
|
||||
details, *_ = await fetch_all_balances()
|
||||
|
||||
assert details[0]["unit"] == settings.primary_mint_unit
|
||||
assert details[0]["error_code"] == "unreachable"
|
||||
assert details[0]["retry_after_seconds"] > 0
|
||||
get_wallet.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_supported_mint_units_come_from_active_keysets() -> None:
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _get_supported_mint_units
|
||||
|
||||
# Cashu versions/mints may deserialize keyset units as either strings or
|
||||
# Unit enum-like objects. Both representations must be accepted.
|
||||
sat = MagicMock(active=True, unit="sat")
|
||||
msat = MagicMock(active=False, unit="msat")
|
||||
usd = MagicMock(active=True)
|
||||
usd.unit.name = "usd"
|
||||
wallet = MagicMock()
|
||||
wallet._get_keysets = AsyncMock(return_value=[usd, msat, sat])
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint_unit", "sat"),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)),
|
||||
):
|
||||
units = await _get_supported_mint_units("http://mint:3338")
|
||||
cached_units = await _get_supported_mint_units("http://mint:3338")
|
||||
|
||||
assert units == ["sat", "usd"]
|
||||
assert cached_units == units
|
||||
wallet._get_keysets.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_backs_off_after_connection_failure() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.wallet.time.monotonic", return_value=10),
|
||||
patch("routstr.wallet.logger.warning") as warning,
|
||||
):
|
||||
first = await fetch_all_balances(units=["sat"])
|
||||
second = await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert first[0][0]["error"] == "mint unavailable"
|
||||
assert first[0][0]["error_code"] == "unreachable"
|
||||
assert first[0][0]["retry_after_seconds"] == 60
|
||||
assert second[0][0]["error"] == "mint unavailable"
|
||||
assert second[0][0]["error_code"] == "unreachable"
|
||||
assert get_wallet.await_count == 1
|
||||
warning.assert_called_once()
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.wallet.time.monotonic", return_value=71),
|
||||
patch("routstr.wallet.logger.warning"),
|
||||
):
|
||||
await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert get_wallet.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_reports_rate_limit_status() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
request = httpx.Request("GET", "http://mint:3338/v1/keysets")
|
||||
response = httpx.Response(429, request=request, headers={"Retry-After": "45"})
|
||||
error = httpx.HTTPStatusError("rate limited", request=request, response=response)
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(side_effect=error)),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert details[0]["error_code"] == "rate_limited"
|
||||
assert details[0]["retry_after_seconds"] == 60
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_balance_failure_applies_mint_cooldown_to_other_units() -> None:
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _mint_cooldown_remaining
|
||||
|
||||
mint = "http://mint:3338"
|
||||
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", [mint]),
|
||||
patch.object(settings, "primary_mint", mint),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.wallet.time.monotonic", return_value=10),
|
||||
patch("routstr.wallet.logger.warning") as warning,
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat", "msat"])
|
||||
cooldown = _mint_cooldown_remaining(mint)
|
||||
|
||||
assert get_wallet.await_count == 1
|
||||
assert warning.call_count == 1
|
||||
assert cooldown == 60
|
||||
assert details[0]["error"] == "mint unavailable"
|
||||
assert details[0]["error_code"] == "unreachable"
|
||||
assert details[1]["error"] == "Mint is unreachable"
|
||||
assert details[1]["error_code"] == "unreachable"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_closes_db_session_before_concurrent_mint_io() -> None:
|
||||
"""Slow mint checks must never run while the balance DB session is open."""
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from cashu.core.base import Proof
|
||||
|
||||
from routstr.lightning import (
|
||||
_invoice_settlement_locks,
|
||||
_is_outputs_already_signed,
|
||||
_mint_invoice_quote,
|
||||
check_invoice_payment,
|
||||
)
|
||||
from routstr.wallet import Wallet
|
||||
|
||||
|
||||
def _invoice(**overrides: object) -> SimpleNamespace:
|
||||
values = {
|
||||
"id": "invoice-1",
|
||||
"payment_hash": "quote-1",
|
||||
"amount_sats": 100,
|
||||
"purpose": "create",
|
||||
"status": "pending",
|
||||
"paid_at": None,
|
||||
"api_key_hash": None,
|
||||
"mint_url": "http://mint:3338",
|
||||
"balance_limit": None,
|
||||
"balance_limit_reset": None,
|
||||
"validity_date": None,
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def _proof(amount: int, mint_id: str, *, reserved: bool = False) -> Proof:
|
||||
return Proof(amount=amount, mint_id=mint_id, reserved=reserved)
|
||||
|
||||
|
||||
def _recovery_wallet(
|
||||
error: Exception,
|
||||
*,
|
||||
proofs_before: list[Proof] | None = None,
|
||||
proofs_after: list[Proof] | None = None,
|
||||
) -> Mock:
|
||||
async def load_proofs(*, reload: bool) -> None:
|
||||
if wallet.load_proofs.await_count >= 2 and proofs_after is not None:
|
||||
wallet.proofs = list(proofs_after)
|
||||
|
||||
wallet = Mock(
|
||||
mint=AsyncMock(side_effect=error),
|
||||
keysets={"keyset-1": Mock()},
|
||||
restore_tokens_for_keyset=AsyncMock(),
|
||||
load_proofs=AsyncMock(side_effect=load_proofs),
|
||||
proofs=list(proofs_before or []),
|
||||
)
|
||||
return wallet
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_recovers_quote_linked_outputs_already_signed() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs have already been signed before (Code: 11003)"),
|
||||
proofs_after=[_proof(100, "quote-1")],
|
||||
)
|
||||
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
wallet.restore_tokens_for_keyset.assert_awaited_once_with(
|
||||
"keyset-1", to=1, batch=25
|
||||
)
|
||||
assert wallet.load_proofs.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_accepts_preloaded_quote_linked_proofs() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("must not mint"),
|
||||
proofs_before=[_proof(64, "quote-1"), _proof(36, "quote-1")],
|
||||
)
|
||||
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
wallet.mint.assert_not_awaited()
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_does_not_accept_unrelated_11003_text() -> None:
|
||||
invoice = _invoice()
|
||||
error = Exception("backend request 11003 failed")
|
||||
wallet = _recovery_wallet(error)
|
||||
|
||||
with pytest.raises(Exception) as caught:
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
assert caught.value is error
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_installed_cashu_error_shape_recognizes_realistic_11003_phrase() -> None:
|
||||
request = httpx.Request("POST", "http://mint:3338/v1/mint/bolt11")
|
||||
response = httpx.Response(
|
||||
400,
|
||||
request=request,
|
||||
json={"detail": "outputs have already been signed before", "code": 11003},
|
||||
)
|
||||
|
||||
with pytest.raises(Exception) as caught:
|
||||
Wallet.raise_on_error_request(response)
|
||||
|
||||
assert _is_outputs_already_signed(caught.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("recovered", [0, 99])
|
||||
async def test_invoice_mint_rejects_empty_or_short_quote_recovery(
|
||||
recovered: int,
|
||||
) -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs already signed (Code: 11003)"),
|
||||
proofs_after=[_proof(recovered, "quote-1")] if recovered else [],
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="expected at least 100"):
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_rejects_unrelated_concurrent_balance_growth() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs already signed (Code: 11003)"),
|
||||
proofs_after=[_proof(10_000, "different-quote")],
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="quote-linked recovery returned 0"):
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_pending_invoice_is_not_minted() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(status="expired")
|
||||
session = AsyncMock()
|
||||
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock()) as get_wallet:
|
||||
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
get_wallet.assert_not_awaited()
|
||||
session.commit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ambiguous_invoice_mint_timeout_does_not_expose_paid() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice()
|
||||
session = AsyncMock()
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch(
|
||||
"routstr.lightning._mint_invoice_quote",
|
||||
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
|
||||
),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert invoice.status == "pending"
|
||||
session.rollback.assert_not_awaited()
|
||||
# One commit closes the initial read transaction before external I/O.
|
||||
session.commit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_invoice_checks_finalize_once_in_process() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice()
|
||||
session = AsyncMock()
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
|
||||
|
||||
async def refresh(obj: SimpleNamespace) -> None:
|
||||
return None
|
||||
|
||||
session.refresh = AsyncMock(side_effect=refresh)
|
||||
|
||||
@asynccontextmanager
|
||||
async def owned_session() -> AsyncIterator[AsyncMock]:
|
||||
yield AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning.create_session", owned_session),
|
||||
patch("routstr.lightning._mint_invoice_quote", AsyncMock()),
|
||||
patch(
|
||||
"routstr.lightning._finalize_invoice_settlement",
|
||||
AsyncMock(return_value=(True, "b" * 64)),
|
||||
) as finalize,
|
||||
):
|
||||
await asyncio.gather(
|
||||
check_invoice_payment(invoice, session), # type: ignore[arg-type]
|
||||
check_invoice_payment(invoice, session), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert invoice.status == "paid"
|
||||
finalize.assert_awaited_once()
|
||||
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _run_alembic(root: Path, database_url: str, command: str, revision: str) -> None:
|
||||
env = os.environ.copy()
|
||||
env["DATABASE_URL"] = database_url
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", command, revision],
|
||||
cwd=root,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def _lightning_invoice_columns(database_path: Path) -> set[str]:
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
return {
|
||||
row[1]
|
||||
for row in connection.execute("PRAGMA table_info(lightning_invoices)")
|
||||
}
|
||||
|
||||
|
||||
def test_mint_url_migration_upgrades_and_downgrades_from_main_head(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
database_path = tmp_path / "mint-url-migration.db"
|
||||
database_url = f"sqlite+aiosqlite:///{database_path}"
|
||||
previous_head = "aa50fde387a2"
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", previous_head)
|
||||
assert "mint_url" not in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", "bf76270b66c4")
|
||||
assert "mint_url" in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "downgrade", previous_head)
|
||||
assert "mint_url" not in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", "head")
|
||||
assert "mint_url" in _lightning_invoice_columns(database_path)
|
||||
@@ -7,7 +7,21 @@ os.environ["UPSTREAM_BASE_URL"] = "http://test"
|
||||
os.environ["UPSTREAM_API_KEY"] = "test"
|
||||
|
||||
from routstr.core.settings import settings # noqa: E402
|
||||
from routstr.payment.helpers import get_max_cost_for_model # noqa: E402
|
||||
from routstr.payment.helpers import ( # noqa: E402
|
||||
apply_mint_fee_allowance,
|
||||
get_max_cost_for_model,
|
||||
)
|
||||
|
||||
|
||||
def test_mint_fee_allowance_reserves_five_percent_fallback_headroom() -> None:
|
||||
# Interim policy: Routstr may pay hidden cross-mint Lightning fees when a
|
||||
# trusted-mint fallback is required.
|
||||
assert apply_mint_fee_allowance(124_886) == 118_642
|
||||
|
||||
|
||||
def test_mint_fee_allowance_never_drops_below_minimum() -> None:
|
||||
with patch.object(settings, "min_request_msat", 100):
|
||||
assert apply_mint_fee_allowance(50) == 100
|
||||
|
||||
|
||||
async def test_get_max_cost_for_model_known() -> None:
|
||||
|
||||
@@ -71,7 +71,9 @@ async def test_pay_for_request_sets_reserved_at(session: AsyncSession) -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pay_for_request_sets_reserved_at_on_child_key(session: AsyncSession) -> None:
|
||||
async def test_pay_for_request_sets_reserved_at_on_child_key(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
parent = ApiKey(hashed_key="parentkey", balance=10_000)
|
||||
child = ApiKey(hashed_key="childkey", balance=0, parent_key_hash="parentkey")
|
||||
session.add(parent)
|
||||
@@ -204,7 +206,9 @@ async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) ->
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncSession) -> None:
|
||||
async def test_release_stale_reservations_skips_null_reserved_at(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
# Reservations without a timestamp may belong to instances running older
|
||||
# code (rolling deploy) — the background sweeper must not touch them.
|
||||
key = ApiKey(
|
||||
@@ -224,7 +228,9 @@ async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_all_reserved_balances_clears_reserved_at(session: AsyncSession) -> None:
|
||||
async def test_reset_all_reserved_balances_clears_reserved_at(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
key = ApiKey(
|
||||
hashed_key="resetkey",
|
||||
balance=5_000,
|
||||
@@ -404,9 +410,7 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
|
||||
AsyncMock(return_value=1_000),
|
||||
),
|
||||
patch.object(proxy_module, "check_token_balance", MagicMock()),
|
||||
patch.object(
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
@@ -418,6 +422,4 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await proxy_module.proxy(request, "v1/chat/completions", session=session)
|
||||
|
||||
revert_mock.assert_awaited_once_with(
|
||||
key, session, 1_000, reservation_snapshot
|
||||
)
|
||||
revert_mock.assert_awaited_once_with(key, session, 950, reservation_snapshot)
|
||||
|
||||
@@ -383,9 +383,7 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
AsyncMock(return_value=1_000),
|
||||
),
|
||||
patch.object(proxy_module, "check_token_balance", MagicMock()),
|
||||
patch.object(
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
@@ -408,4 +406,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
# Single upstream failed -> reservation reverted exactly once (no double-charge).
|
||||
revert_mock.assert_awaited_once_with(key, session, 1_000, reservation)
|
||||
revert_mock.assert_awaited_once_with(key, session, 950, reservation)
|
||||
|
||||
+1077
-41
File diff suppressed because it is too large
Load Diff
@@ -105,6 +105,23 @@ export function DetailedWalletBalance({
|
||||
const formatMintLabel = (detail: BalanceDetail) =>
|
||||
`${detail.mint_url.replace('https://', '').replace('http://', '')} • ${detail.unit.toUpperCase()}`;
|
||||
|
||||
const formatBalanceError = (detail: BalanceDetail) => {
|
||||
const labels: Record<string, string> = {
|
||||
rate_limited: 'rate limited',
|
||||
unreachable: 'unreachable',
|
||||
cooldown: 'cooling down',
|
||||
mint_error: 'mint error',
|
||||
};
|
||||
const label =
|
||||
(detail.error_code ? labels[detail.error_code] : undefined) ??
|
||||
detail.error ??
|
||||
'error';
|
||||
const retryAfter = detail.retry_after_seconds;
|
||||
return retryAfter && retryAfter > 0
|
||||
? `${label} (retry in ${Math.ceil(retryAfter)}s)`
|
||||
: label;
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Card>
|
||||
@@ -262,9 +279,12 @@ export function DetailedWalletBalance({
|
||||
<TableCell className='max-w-md font-mono text-xs break-all whitespace-normal'>
|
||||
{formatMintLabel(detail)}
|
||||
</TableCell>
|
||||
<TableCell className='text-right font-mono'>
|
||||
<TableCell
|
||||
className='text-right font-mono'
|
||||
title={detail.error}
|
||||
>
|
||||
{detail.error
|
||||
? 'error'
|
||||
? formatBalanceError(detail)
|
||||
: formatAmount(walletMsat)}
|
||||
</TableCell>
|
||||
<TableCell className='text-right font-mono'>
|
||||
@@ -306,9 +326,12 @@ export function DetailedWalletBalance({
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Wallet
|
||||
</p>
|
||||
<p className='font-mono text-sm'>
|
||||
<p
|
||||
className='font-mono text-sm'
|
||||
title={detail.error}
|
||||
>
|
||||
{detail.error
|
||||
? 'error'
|
||||
? formatBalanceError(detail)
|
||||
: formatAmount(walletMsat)}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -36,6 +36,8 @@ export interface BalanceDetail {
|
||||
user_balance: number;
|
||||
owner_balance: number;
|
||||
error?: string;
|
||||
error_code?: 'rate_limited' | 'unreachable' | 'cooldown' | 'mint_error';
|
||||
retry_after_seconds?: number;
|
||||
}
|
||||
|
||||
export interface WithdrawResponse {
|
||||
|
||||
Reference in New Issue
Block a user