mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-10-05 20:28:23 +00:00
Merge pull request #597 from Routstr/fix/mint-rate-limit-and-fallback
fix: per-mint rate limiting, trusted-mint fallback, and retry factory…
This commit is contained in:
@@ -45,6 +45,9 @@ ROUTSTR_SECRET_KEY=
|
||||
# ENABLE_ANALYTICS_SHARING=true
|
||||
# CASHU_MINTS="https://mint.minibits.cash/Bitcoin,https://mint.cubabitcoin.org,https://ecashmint.otrta.me"
|
||||
# MINT_OPERATION_CONCURRENCY=4
|
||||
# MINT_OPERATION_TIMEOUT_SECONDS=30
|
||||
# MINT_MAX_CONCURRENCY=4
|
||||
# MINT_RETRY_MAX_ATTEMPTS=3
|
||||
# RECEIVE_LN_ADDRESS=
|
||||
# REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS=900
|
||||
|
||||
|
||||
@@ -136,6 +136,10 @@ Use environment variables for:
|
||||
| `NSEC` | Legacy seed for the Nostr private key (otherwise set from the admin UI) | — |
|
||||
| `ENABLE_ANALYTICS_SHARING` | Enable usage analytics sharing to Nostr | `true` |
|
||||
| `CASHU_MINTS` | Comma-separated mint URLs | `https://mint.minibits.cash/Bitcoin` |
|
||||
| `MINT_OPERATION_CONCURRENCY` | Concurrent mint/unit balance reads | `4` |
|
||||
| `MINT_OPERATION_TIMEOUT_SECONDS` | Per-attempt timeout for mint network calls | `30` |
|
||||
| `MINT_MAX_CONCURRENCY` | Concurrent operations allowed per mint (`0` disables the limit) | `4` |
|
||||
| `MINT_RETRY_MAX_ATTEMPTS` | Retries after a timeout or HTTP 429 (`0` disables retries) | `3` |
|
||||
| `RECEIVE_LN_ADDRESS` | Lightning address for withdrawals | — |
|
||||
| `MIN_PAYOUT_SAT` | Min payout balance in sats (applies to all mints) | `210` |
|
||||
| `PAYOUT_INTERVAL_SECONDS` | Payout loop interval (seconds) | `900` |
|
||||
@@ -145,6 +149,11 @@ Use environment variables for:
|
||||
| `MODEL_PATHS_REFRESH_INTERVAL_SECONDS` | How often to refresh `/v1/models/paths` discovery data; set `0` to pause the refresh (previously discovered paths keep being served) | `600` |
|
||||
| `ENABLE_MODEL_PATHS_REFRESH` | Kill switch for the background model-path refresh (OpenRouter endpoint fan-out) | `true` |
|
||||
|
||||
Mint HTTP 429 responses create a per-mint cooldown. Operations that already hold
|
||||
Routstr's wallet mutation lock fail fast during that cooldown instead of waiting
|
||||
while blocking every other wallet mutation. Callers receive an error and may retry
|
||||
later; the current response does not include the cooldown duration.
|
||||
|
||||
### Priority
|
||||
|
||||
Environment variables are read on startup. Dashboard settings override them and persist in the database. Once you change a setting in the dashboard, the env var is ignored for that setting.
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""add mint url to lightning invoices
|
||||
|
||||
Revision ID: ecfa0d6e2a36
|
||||
Revises: 64ed5594df1f
|
||||
Create Date: 2026-08-02 23:53:00.037456
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "ecfa0d6e2a36"
|
||||
down_revision = "64ed5594df1f"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _resolve_backfill_mint_url(bind: sa.engine.Connection) -> str | None:
|
||||
"""Best-effort resolution of the mint that issued pre-existing invoices.
|
||||
|
||||
Order: persisted settings JSON -> PRIMARY_MINT_URL env -> first CASHU_MINTS entry.
|
||||
"""
|
||||
try:
|
||||
row = bind.execute(
|
||||
sa.text("SELECT data FROM settings ORDER BY id LIMIT 1")
|
||||
).fetchone()
|
||||
if row and row[0]:
|
||||
data = json.loads(row[0])
|
||||
mint = data.get("primary_mint") or next(
|
||||
iter(data.get("cashu_mints") or []), None
|
||||
)
|
||||
if mint:
|
||||
return str(mint)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
env_mint = os.environ.get("PRIMARY_MINT_URL", "").strip()
|
||||
if env_mint:
|
||||
return env_mint
|
||||
|
||||
cashu_mints = os.environ.get("CASHU_MINTS", "").strip()
|
||||
if cashu_mints:
|
||||
return cashu_mints.split(",")[0].strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"lightning_invoices", sa.Column("mint_url", sa.String(), nullable=True)
|
||||
)
|
||||
|
||||
bind = op.get_bind()
|
||||
backfill_mint = _resolve_backfill_mint_url(bind)
|
||||
if backfill_mint:
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"UPDATE lightning_invoices SET mint_url = :mint WHERE mint_url IS NULL"
|
||||
),
|
||||
{"mint": backfill_mint},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("lightning_invoices", "mint_url")
|
||||
+36
-16
@@ -51,6 +51,24 @@ ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
|
||||
ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
|
||||
|
||||
|
||||
def _format_msat_amount(amount: int) -> str:
|
||||
sats = f"{amount / 1000:.3f}".rstrip("0").rstrip(".")
|
||||
return f"{sats} sats ({amount} msats)"
|
||||
|
||||
|
||||
def _model_balance_error(required: int, available: int) -> dict[str, dict[str, str]]:
|
||||
return {
|
||||
"error": {
|
||||
"message": (
|
||||
f"Insufficient balance: {_format_msat_amount(required)} required "
|
||||
f"for this model; {_format_msat_amount(available)} available."
|
||||
),
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReservationSnapshot:
|
||||
release_id: str
|
||||
@@ -264,13 +282,7 @@ async def _validate_bearer_key_locked(
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": {
|
||||
"message": f"Insufficient balance: {min_cost} mSats required for this model. {billing_key.total_balance} available.",
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
},
|
||||
detail=_model_balance_error(min_cost, billing_key.total_balance),
|
||||
)
|
||||
|
||||
# Early check: Spending limit check (Child key limit)
|
||||
@@ -360,13 +372,9 @@ async def _validate_bearer_key_locked(
|
||||
if min_cost > 0 and existing_key.total_balance < min_cost:
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": {
|
||||
"message": f"Insufficient balance: {min_cost} mSats required for this model. {existing_key.total_balance} available.",
|
||||
"type": "insufficient_quota",
|
||||
"code": "insufficient_balance",
|
||||
}
|
||||
},
|
||||
detail=_model_balance_error(
|
||||
min_cost, existing_key.total_balance
|
||||
),
|
||||
)
|
||||
|
||||
return existing_key
|
||||
@@ -379,11 +387,23 @@ async def _validate_bearer_key_locked(
|
||||
"has_expiry_time": bool(key_expiry_time),
|
||||
},
|
||||
)
|
||||
if token_obj.mint in settings.cashu_mints:
|
||||
if token_obj.mint == settings.primary_mint:
|
||||
if token_obj.unit != settings.primary_mint_unit:
|
||||
raise redemption_error_to_http_exception(
|
||||
ValueError(
|
||||
"Cashu token unit does not match the configured primary "
|
||||
f"mint unit: expected {settings.primary_mint_unit}, "
|
||||
f"got {token_obj.unit}"
|
||||
)
|
||||
)
|
||||
refund_currency = token_obj.unit
|
||||
refund_mint_url = settings.primary_mint
|
||||
elif token_obj.mint in settings.cashu_mints:
|
||||
refund_currency = token_obj.unit
|
||||
refund_mint_url = token_obj.mint
|
||||
else:
|
||||
refund_currency = "sat"
|
||||
# Foreign tokens are swapped into the configured primary mint.
|
||||
refund_currency = settings.primary_mint_unit
|
||||
refund_mint_url = settings.primary_mint
|
||||
|
||||
new_key = ApiKey(
|
||||
|
||||
+107
-13
@@ -23,6 +23,7 @@ from .core.db import (
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
from .lightning import lightning_router
|
||||
from .payment.lnurl import MeltOutcomeAmbiguousError
|
||||
from .wallet import (
|
||||
classify_redemption_error,
|
||||
credit_balance,
|
||||
@@ -30,6 +31,7 @@ from .wallet import (
|
||||
recieve_token,
|
||||
send_to_lnurl,
|
||||
send_token,
|
||||
token_mint_url,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
@@ -184,6 +186,17 @@ class TopupRequest(BaseModel):
|
||||
cashu_token: str
|
||||
|
||||
|
||||
def _error_chain(error: BaseException) -> list[dict[str, str]]:
|
||||
chain: list[dict[str, str]] = []
|
||||
current: BaseException | None = error
|
||||
seen: set[int] = set()
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
chain.append({"type": type(current).__name__, "message": str(current)})
|
||||
current = current.__cause__ or current.__context__
|
||||
return chain
|
||||
|
||||
|
||||
@router.post("/topup")
|
||||
async def topup_wallet_endpoint(
|
||||
cashu_token: str | None = None,
|
||||
@@ -201,6 +214,18 @@ async def topup_wallet_endpoint(
|
||||
cashu_token = cashu_token.replace("\n", "").replace("\r", "").replace("\t", "")
|
||||
if len(cashu_token) < 10 or "cashu" not in cashu_token:
|
||||
raise HTTPException(status_code=400, detail="Invalid token format")
|
||||
|
||||
source_mint = token_mint_url(cashu_token, "unknown")
|
||||
logger.info(
|
||||
"Cashu wallet top-up started",
|
||||
extra={
|
||||
"event": "cashu_topup_started",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"key_hash": billing_key.hashed_key[:8],
|
||||
},
|
||||
)
|
||||
try:
|
||||
amount_msats = await credit_balance(cashu_token, billing_key, session)
|
||||
except Exception as e:
|
||||
@@ -209,12 +234,41 @@ async def topup_wallet_endpoint(
|
||||
classified = classify_redemption_error(e)
|
||||
if classified is None:
|
||||
logger.error(
|
||||
"topup_wallet_endpoint: unhandled error",
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
"Cashu wallet top-up failed with an unhandled error",
|
||||
extra={
|
||||
"event": "cashu_topup_failed",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"error_chain": _error_chain(e),
|
||||
},
|
||||
)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
_type, status_code, message, _code = classified
|
||||
error_type, status_code, message, error_code = classified
|
||||
logger.warning(
|
||||
"Cashu wallet top-up failed",
|
||||
extra={
|
||||
"event": "cashu_topup_failed",
|
||||
"source_mint": source_mint,
|
||||
"primary_mint": settings.primary_mint,
|
||||
"trusted_mints": settings.cashu_mints,
|
||||
"status_code": status_code,
|
||||
"error_type": error_type,
|
||||
"error_code": error_code,
|
||||
"error_chain": _error_chain(e),
|
||||
},
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=message)
|
||||
|
||||
logger.info(
|
||||
"Cashu wallet top-up completed",
|
||||
extra={
|
||||
"event": "cashu_topup_completed",
|
||||
"source_mint": source_mint,
|
||||
"credited_msats": amount_msats,
|
||||
"key_hash": billing_key.hashed_key[:8],
|
||||
},
|
||||
)
|
||||
return {"msats": amount_msats}
|
||||
|
||||
|
||||
@@ -290,7 +344,11 @@ async def _get_persisted_api_key_refund(
|
||||
|
||||
|
||||
async def _restore_balance(
|
||||
session: AsyncSession, hashed_key: str, balance: int, reserved_balance: int, mint_url: str
|
||||
session: AsyncSession,
|
||||
hashed_key: str,
|
||||
balance: int,
|
||||
reserved_balance: int,
|
||||
mint_url: str,
|
||||
) -> None:
|
||||
"""Restore balance after a failed refund mint attempt."""
|
||||
restore_stmt = (
|
||||
@@ -305,7 +363,11 @@ async def _restore_balance(
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"refund_wallet_endpoint: balance restored after mint failure",
|
||||
extra={"hashed_key": hashed_key, "restored_balance": balance, "mint_url": mint_url},
|
||||
extra={
|
||||
"hashed_key": hashed_key,
|
||||
"restored_balance": balance,
|
||||
"mint_url": mint_url,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -450,15 +512,14 @@ async def refund_wallet_endpoint(
|
||||
detail="Balance changed concurrently. Please retry the refund.",
|
||||
)
|
||||
|
||||
# --- MINT: balance is locked at zero, safe to create the refund token ---
|
||||
# Proofs from untrusted mints are swapped to primary_mint on receive.
|
||||
# Use primary_mint unless key.refund_mint_url is an explicitly trusted mint.
|
||||
# The balance is locked at zero, so it is safe to create the refund token.
|
||||
effective_refund_mint = (
|
||||
key.refund_mint_url
|
||||
if key.refund_mint_url and key.refund_mint_url in settings.cashu_mints
|
||||
else settings.primary_mint
|
||||
)
|
||||
try:
|
||||
refund_currency = key.refund_currency or "sat"
|
||||
if key.refund_address:
|
||||
await send_to_lnurl(
|
||||
remaining_balance,
|
||||
@@ -468,10 +529,10 @@ async def refund_wallet_endpoint(
|
||||
)
|
||||
result = {"recipient": key.refund_address}
|
||||
else:
|
||||
refund_currency = key.refund_currency or "sat"
|
||||
token = await send_token(
|
||||
remaining_balance, refund_currency, effective_refund_mint
|
||||
)
|
||||
effective_refund_mint = token_mint_url(token, effective_refund_mint)
|
||||
result = {"token": token}
|
||||
|
||||
if key.refund_currency == "sat":
|
||||
@@ -490,13 +551,47 @@ async def refund_wallet_endpoint(
|
||||
},
|
||||
)
|
||||
|
||||
except MeltOutcomeAmbiguousError as e:
|
||||
# The melt was dispatched and may still settle. Restoring the balance
|
||||
# here would let the same debit be paid out twice; keep the debit and
|
||||
# leave the outcome to reconciliation.
|
||||
logger.error(
|
||||
"refund_wallet_endpoint: melt outcome ambiguous; balance withheld "
|
||||
"pending reconciliation",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"hashed_key": key.hashed_key,
|
||||
"remaining_balance": remaining_balance,
|
||||
"refund_currency": key.refund_currency,
|
||||
"refund_mint_url": key.refund_mint_url,
|
||||
},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=(
|
||||
"Refund was dispatched but its outcome is unconfirmed; the "
|
||||
"balance is withheld until reconciliation completes"
|
||||
),
|
||||
)
|
||||
except HTTPException:
|
||||
# Minting failed — restore the debited balance
|
||||
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
|
||||
await _restore_balance(
|
||||
session,
|
||||
key.hashed_key,
|
||||
pre_debit_balance,
|
||||
pre_debit_reserved,
|
||||
key.refund_mint_url or "",
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
# Minting failed — restore the debited balance
|
||||
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
|
||||
await _restore_balance(
|
||||
session,
|
||||
key.hashed_key,
|
||||
pre_debit_balance,
|
||||
pre_debit_reserved,
|
||||
key.refund_mint_url or "",
|
||||
)
|
||||
error_msg = str(e)
|
||||
logger.error(
|
||||
"refund_wallet_endpoint: mint/send failed",
|
||||
@@ -523,7 +618,7 @@ async def refund_wallet_endpoint(
|
||||
token=result["token"],
|
||||
amount=remaining_balance,
|
||||
unit=key.refund_currency or "sat",
|
||||
mint_url=key.refund_mint_url,
|
||||
mint_url=effective_refund_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="apikey",
|
||||
@@ -717,7 +812,6 @@ async def reset_child_key_spent(
|
||||
return {"success": True, "message": "Child key balance reset successfully."}
|
||||
|
||||
|
||||
|
||||
@router.api_route(
|
||||
"/{path:path}",
|
||||
methods=["GET", "POST", "PUT", "DELETE"],
|
||||
|
||||
+15
-27
@@ -13,13 +13,7 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..payment.models import _row_to_model, list_models
|
||||
from ..proxy import refresh_model_maps, reinitialize_upstreams
|
||||
from ..wallet import (
|
||||
fetch_all_balances,
|
||||
get_proofs_per_mint_and_unit,
|
||||
get_wallet,
|
||||
send_token,
|
||||
slow_filter_spend_proofs,
|
||||
)
|
||||
from ..wallet import fetch_all_balances, send_token, token_mint_url
|
||||
from . import vault
|
||||
from .db import (
|
||||
ApiKey,
|
||||
@@ -442,37 +436,31 @@ class WithdrawRequest(BaseModel):
|
||||
async def withdraw(
|
||||
request: Request, withdraw_request: WithdrawRequest
|
||||
) -> dict[str, str]:
|
||||
# Get wallet and check balance
|
||||
from .settings import settings as global_settings
|
||||
|
||||
effective_mint = withdraw_request.mint_url or global_settings.primary_mint
|
||||
wallet = await get_wallet(effective_mint, withdraw_request.unit)
|
||||
proofs = get_proofs_per_mint_and_unit(
|
||||
wallet,
|
||||
effective_mint,
|
||||
withdraw_request.unit,
|
||||
not_reserved=True,
|
||||
)
|
||||
proofs = await slow_filter_spend_proofs(proofs, wallet)
|
||||
current_balance = sum(proof.amount for proof in proofs)
|
||||
|
||||
if withdraw_request.amount <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Withdrawal amount must be positive"
|
||||
)
|
||||
|
||||
if withdraw_request.amount > current_balance:
|
||||
raise HTTPException(status_code=400, detail="Insufficient wallet balance")
|
||||
|
||||
token = await send_token(
|
||||
withdraw_request.amount, withdraw_request.unit, effective_mint
|
||||
)
|
||||
try:
|
||||
token = await send_token(
|
||||
withdraw_request.amount, withdraw_request.unit, effective_mint
|
||||
)
|
||||
except ValueError as error:
|
||||
if not str(error).startswith("No trusted mint has "):
|
||||
raise
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Insufficient wallet balance"
|
||||
) from error
|
||||
actual_mint = token_mint_url(token, effective_mint)
|
||||
try:
|
||||
await store_cashu_transaction(
|
||||
token=token,
|
||||
amount=withdraw_request.amount,
|
||||
unit=withdraw_request.unit,
|
||||
mint_url=effective_mint,
|
||||
mint_url=actual_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="admin",
|
||||
@@ -483,10 +471,10 @@ async def withdraw(
|
||||
extra={
|
||||
"amount": withdraw_request.amount,
|
||||
"unit": withdraw_request.unit,
|
||||
"mint_url": effective_mint,
|
||||
"mint_url": actual_mint,
|
||||
},
|
||||
)
|
||||
return {"token": token}
|
||||
return {"token": token, "mint_url": actual_mint}
|
||||
|
||||
|
||||
class ModelCreate(BaseModel):
|
||||
|
||||
+11
-3
@@ -293,7 +293,7 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
|
||||
"""Delete dead parentless API keys; return the count removed.
|
||||
|
||||
Dead = 0 balance/reservation/spend/requests, older than the grace period,
|
||||
no parent, no children, no pending invoice. Cashu rows are unlinked (not
|
||||
no parent, no children, no retryable invoice. Cashu rows are unlinked (not
|
||||
deleted) first to keep the audit trail.
|
||||
"""
|
||||
cutoff = int(time.time()) - min_age_seconds
|
||||
@@ -307,7 +307,9 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
|
||||
pending_invoice = (
|
||||
select(LightningInvoice.id)
|
||||
.where(col(LightningInvoice.api_key_hash) == col(ApiKey.hashed_key))
|
||||
.where(col(LightningInvoice.status) == "pending")
|
||||
.where(
|
||||
col(LightningInvoice.status).in_(("pending", "settlement_pending"))
|
||||
)
|
||||
).exists()
|
||||
|
||||
eligible_hashes = (
|
||||
@@ -435,12 +437,18 @@ class LightningInvoice(SQLModel, table=True): # type: ignore
|
||||
payment_hash: str = Field(description="Payment hash for tracking", unique=True)
|
||||
status: str = Field(
|
||||
default="pending",
|
||||
description="pending, paid, expired, cancelled, reconciliation_required",
|
||||
description=(
|
||||
"pending, settlement_pending, paid, expired, cancelled, "
|
||||
"reconciliation_required"
|
||||
),
|
||||
)
|
||||
api_key_hash: str | None = Field(
|
||||
default=None, description="Associated API key hash for topup operations"
|
||||
)
|
||||
purpose: str = Field(description="create or topup")
|
||||
mint_url: str | None = Field(
|
||||
default=None, description="Mint URL where the quote was created (fallback tracking)"
|
||||
)
|
||||
created_at: int = Field(
|
||||
default_factory=lambda: int(time.time()), description="Unix timestamp"
|
||||
)
|
||||
|
||||
@@ -53,6 +53,18 @@ class Settings(BaseSettings):
|
||||
payout_interval_seconds: int = Field(
|
||||
default=900, gt=0, env="PAYOUT_INTERVAL_SECONDS"
|
||||
)
|
||||
# Timeout (seconds) for individual mint API operations (melt, mint, swap,
|
||||
# checkstate). When a mint is slow or rate-limiting, operations are
|
||||
# cancelled after this delay instead of hanging indefinitely.
|
||||
mint_operation_timeout_seconds: int = Field(
|
||||
default=30, gt=0, env="MINT_OPERATION_TIMEOUT_SECONDS"
|
||||
)
|
||||
# Maximum concurrent API operations per mint. Actual mint quotas vary by
|
||||
# endpoint, so 429 responses drive adaptive cooldown instead of fixed RPM
|
||||
# pacing. 0 = unlimited concurrency.
|
||||
mint_max_concurrency: int = Field(default=4, ge=0, env="MINT_MAX_CONCURRENCY")
|
||||
# Max retries when a mint returns 429 or times out (exponential backoff).
|
||||
mint_retry_max_attempts: int = Field(default=3, ge=0, env="MINT_RETRY_MAX_ATTEMPTS")
|
||||
|
||||
# Pricing
|
||||
# Default behavior: derive pricing from MODELS
|
||||
|
||||
+541
-175
@@ -1,8 +1,13 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, AsyncGenerator
|
||||
|
||||
from cashu.core.base import MintQuoteState
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm.attributes import set_committed_value
|
||||
@@ -12,12 +17,85 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
from .core.db import ApiKey, LightningInvoice, create_session, get_session
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
from .wallet import get_wallet, wallet_operation_guard
|
||||
from .mint import (
|
||||
is_mint_rate_limited,
|
||||
mint_cooldown_remaining,
|
||||
run_mint_operation,
|
||||
)
|
||||
from .wallet import (
|
||||
MintConnectionError,
|
||||
get_wallet,
|
||||
is_mint_connection_error,
|
||||
wallet_operation_guard,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
lightning_router = APIRouter(prefix="/lightning")
|
||||
|
||||
# Avoid duplicate work within one process. Cross-process settlement is fenced
|
||||
# by claiming a paid quote before minting and by the final conditional update.
|
||||
@dataclass
|
||||
class _InvoiceLockEntry:
|
||||
lock: asyncio.Lock
|
||||
users: int = 0
|
||||
|
||||
|
||||
_invoice_settlement_locks: dict[str, _InvoiceLockEntry] = {}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _invoice_settlement_lock(invoice_id: str) -> AsyncGenerator[None, None]:
|
||||
"""Serialize one invoice and remove its lock after the last waiter leaves."""
|
||||
|
||||
entry = _invoice_settlement_locks.get(invoice_id)
|
||||
if entry is None:
|
||||
entry = _InvoiceLockEntry(asyncio.Lock())
|
||||
_invoice_settlement_locks[invoice_id] = entry
|
||||
entry.users += 1
|
||||
try:
|
||||
async with entry.lock:
|
||||
yield
|
||||
finally:
|
||||
entry.users -= 1
|
||||
if entry.users == 0 and _invoice_settlement_locks.get(invoice_id) is entry:
|
||||
del _invoice_settlement_locks[invoice_id]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _InvoiceSettlement:
|
||||
id: str
|
||||
payment_hash: str
|
||||
amount_sats: int
|
||||
purpose: str
|
||||
api_key_hash: str | None
|
||||
mint_url: str | None
|
||||
balance_limit: int | None
|
||||
balance_limit_reset: str | None
|
||||
validity_date: int | None
|
||||
|
||||
@classmethod
|
||||
def from_invoice(cls, invoice: LightningInvoice) -> "_InvoiceSettlement":
|
||||
return cls(
|
||||
id=invoice.id,
|
||||
payment_hash=invoice.payment_hash,
|
||||
amount_sats=invoice.amount_sats,
|
||||
purpose=invoice.purpose,
|
||||
api_key_hash=invoice.api_key_hash,
|
||||
mint_url=invoice.mint_url,
|
||||
balance_limit=invoice.balance_limit,
|
||||
balance_limit_reset=invoice.balance_limit_reset,
|
||||
validity_date=invoice.validity_date,
|
||||
)
|
||||
|
||||
|
||||
def _publish_invoice_value(invoice: LightningInvoice, key: str, value: Any) -> None:
|
||||
"""Update a caller view without marking a mapped object dirty."""
|
||||
try:
|
||||
set_committed_value(invoice, key, value)
|
||||
except AttributeError:
|
||||
setattr(invoice, key, value)
|
||||
|
||||
|
||||
class InvoiceCreateRequest(BaseModel):
|
||||
amount_sats: int = Field(gt=0, le=1_000_000, description="Amount in satoshis")
|
||||
@@ -61,16 +139,102 @@ class InvoiceStatusResponse(BaseModel):
|
||||
expires_at: int
|
||||
|
||||
|
||||
_RETRYABLE_INVOICE_STATUSES = ("pending", "settlement_pending")
|
||||
|
||||
|
||||
class InvoiceRecoverRequest(BaseModel):
|
||||
bolt11: str = Field(description="BOLT11 invoice string")
|
||||
|
||||
|
||||
def _trusted_mint_candidates() -> list[str]:
|
||||
return [
|
||||
mint
|
||||
for mint in dict.fromkeys([settings.primary_mint, *settings.cashu_mints])
|
||||
if mint
|
||||
]
|
||||
|
||||
|
||||
async def _request_mint_with_fallback(
|
||||
amount_sats: int,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
"""Request a quote, falling back only among the allowed trusted mints.
|
||||
|
||||
Guards against amount_sats <= 0: the cashu library's PostMintQuoteRequest
|
||||
enforces ``amount > 0`` (Pydantic Field(gt=0)), so passing 0 raises a
|
||||
cryptic validation error deep in the stack. Fail fast with context.
|
||||
"""
|
||||
if amount_sats <= 0:
|
||||
raise ValueError(
|
||||
f"generate_lightning_invoice: amount_sats must be > 0, got {amount_sats}."
|
||||
)
|
||||
tried: list[str] = []
|
||||
trusted = _trusted_mint_candidates()
|
||||
if allowed_mints:
|
||||
# Persisted mint preferences (e.g. an API key's refund_mint_url) must
|
||||
# not outlive the operator's trusted-mint configuration.
|
||||
candidates = [m for m in dict.fromkeys(allowed_mints) if m in trusted]
|
||||
if not candidates:
|
||||
logger.warning(
|
||||
"Requested mints are no longer trusted; falling back to "
|
||||
"configured mints",
|
||||
extra={
|
||||
"requested_mints": list(dict.fromkeys(allowed_mints)),
|
||||
"op_name": "request_mint_invoice",
|
||||
},
|
||||
)
|
||||
candidates = trusted
|
||||
else:
|
||||
candidates = trusted
|
||||
for mint_url in candidates:
|
||||
cooldown = mint_cooldown_remaining(mint_url)
|
||||
if cooldown > 0:
|
||||
tried.append(f"{mint_url}: cooling down")
|
||||
logger.info(
|
||||
"Skipping mint during cooldown",
|
||||
extra={
|
||||
"mint_url": mint_url,
|
||||
"cooldown_seconds": round(cooldown, 2),
|
||||
"op_name": "request_mint_invoice",
|
||||
},
|
||||
)
|
||||
continue
|
||||
try:
|
||||
wallet = await get_wallet(mint_url, "sat", retry_on_rate_limit=False)
|
||||
quote = await run_mint_operation(
|
||||
lambda: wallet.request_mint(amount_sats),
|
||||
op_name="request_mint_invoice",
|
||||
mint_url=mint_url,
|
||||
retry_on_rate_limit=False,
|
||||
)
|
||||
return quote.request, quote.quote, mint_url
|
||||
except Exception as e:
|
||||
tried.append(f"{mint_url}: {type(e).__name__}")
|
||||
if not is_mint_connection_error(e) and not is_mint_rate_limited(e):
|
||||
raise
|
||||
logger.warning(
|
||||
"request_mint failed, trying fallback mint",
|
||||
extra={
|
||||
"failed_mint": mint_url,
|
||||
"error": str(e),
|
||||
"tried": tried,
|
||||
},
|
||||
)
|
||||
continue
|
||||
raise MintConnectionError(f"All mints failed for request_mint: {tried}")
|
||||
|
||||
|
||||
async def generate_lightning_invoice(
|
||||
amount_sats: int, description: str
|
||||
) -> tuple[str, str]:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
quote = await wallet.request_mint(amount_sats)
|
||||
return quote.request, quote.quote
|
||||
amount_sats: int,
|
||||
description: str,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
bolt11, payment_hash, mint_url = await _request_mint_with_fallback(
|
||||
amount_sats, allowed_mints=allowed_mints
|
||||
)
|
||||
return bolt11, payment_hash, mint_url
|
||||
|
||||
|
||||
def generate_invoice_id() -> str:
|
||||
@@ -84,6 +248,7 @@ async def create_invoice(
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> InvoiceCreateResponse:
|
||||
api_key_token = _extract_bearer_api_key(authorization) or request.api_key
|
||||
topup_api_key: ApiKey | None = None
|
||||
|
||||
if request.purpose == "topup":
|
||||
if not api_key_token:
|
||||
@@ -94,14 +259,23 @@ async def create_invoice(
|
||||
if not api_key_token.startswith("sk-"):
|
||||
raise HTTPException(status_code=400, detail="Invalid API key format")
|
||||
|
||||
api_key = await session.get(ApiKey, api_key_token[3:])
|
||||
if not api_key:
|
||||
topup_api_key = await session.get(ApiKey, api_key_token[3:])
|
||||
if not topup_api_key:
|
||||
raise HTTPException(status_code=404, detail="API key not found")
|
||||
|
||||
try:
|
||||
description = f"Routstr {request.purpose} {request.amount_sats} sats"
|
||||
bolt11, payment_hash = await generate_lightning_invoice(
|
||||
request.amount_sats, description
|
||||
allowed_mints = None
|
||||
if request.purpose == "topup":
|
||||
assert topup_api_key is not None
|
||||
# A key's liabilities are attributed to a single refund mint. Keep
|
||||
# top-up collateral on that same mint so balances and payouts cannot
|
||||
# misclassify funds held by another mint as owner profit.
|
||||
allowed_mints = [
|
||||
topup_api_key.refund_mint_url or settings.primary_mint
|
||||
]
|
||||
bolt11, payment_hash, mint_url = await generate_lightning_invoice(
|
||||
request.amount_sats, description, allowed_mints=allowed_mints
|
||||
)
|
||||
|
||||
invoice_id = generate_invoice_id()
|
||||
@@ -116,6 +290,7 @@ async def create_invoice(
|
||||
status="pending",
|
||||
api_key_hash=api_key_token[3:] if api_key_token else None,
|
||||
purpose=request.purpose,
|
||||
mint_url=mint_url,
|
||||
balance_limit=request.balance_limit,
|
||||
balance_limit_reset=request.balance_limit_reset,
|
||||
validity_date=request.validity_date,
|
||||
@@ -161,12 +336,12 @@ async def get_invoice_status(
|
||||
if not invoice:
|
||||
raise HTTPException(status_code=404, detail="Invoice not found")
|
||||
|
||||
if invoice.status == "pending":
|
||||
await check_invoice_payment(invoice, session)
|
||||
|
||||
if invoice.status == "pending" and int(time.time()) > invoice.expires_at:
|
||||
invoice.status = "expired"
|
||||
await session.commit()
|
||||
definitively_unpaid = False
|
||||
if invoice.status in _RETRYABLE_INVOICE_STATUSES:
|
||||
definitively_unpaid = await check_invoice_payment(invoice, session)
|
||||
await _expire_invoice_if_authoritatively_unpaid(
|
||||
invoice, session, definitively_unpaid
|
||||
)
|
||||
|
||||
api_key = None
|
||||
if invoice.status == "paid" and invoice.purpose == "create":
|
||||
@@ -200,8 +375,12 @@ async def recover_invoice(
|
||||
if not invoice:
|
||||
raise HTTPException(status_code=404, detail="Invoice not found")
|
||||
|
||||
if invoice.status == "pending":
|
||||
await check_invoice_payment(invoice, session)
|
||||
definitively_unpaid = False
|
||||
if invoice.status in _RETRYABLE_INVOICE_STATUSES:
|
||||
definitively_unpaid = await check_invoice_payment(invoice, session)
|
||||
await _expire_invoice_if_authoritatively_unpaid(
|
||||
invoice, session, definitively_unpaid
|
||||
)
|
||||
|
||||
api_key = None
|
||||
if invoice.status == "paid":
|
||||
@@ -220,161 +399,256 @@ async def recover_invoice(
|
||||
)
|
||||
|
||||
|
||||
async def _claim_paid_invoice_for_settlement(
|
||||
invoice: LightningInvoice,
|
||||
caller_session: AsyncSession,
|
||||
observed_status: str,
|
||||
) -> bool:
|
||||
"""Claim an authoritative paid quote before consuming it at the mint."""
|
||||
if observed_status == "settlement_pending":
|
||||
return True
|
||||
if observed_status != "pending":
|
||||
await _reload_invoice_view(invoice, caller_session)
|
||||
return False
|
||||
|
||||
async with create_session() as claim_session:
|
||||
claim = await claim_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice.id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(status="settlement_pending")
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
await claim_session.commit()
|
||||
|
||||
if claim.rowcount != 1:
|
||||
await _reload_invoice_view(invoice, caller_session)
|
||||
return False
|
||||
|
||||
_publish_invoice_value(invoice, "status", "settlement_pending")
|
||||
return True
|
||||
|
||||
|
||||
async def check_invoice_payment(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> None:
|
||||
# Minting makes proofs visible before database finalization. Share the
|
||||
# cross-process wallet guard with owner payout so that visibility and the
|
||||
# corresponding liability commit are observed atomically by the payout loop.
|
||||
async with wallet_operation_guard():
|
||||
await _check_invoice_payment_locked(invoice, session)
|
||||
) -> bool:
|
||||
"""Settle an invoice and report whether its quote is definitively unpaid.
|
||||
|
||||
False covers paid, pending, and ambiguous transport/DB outcomes so callers
|
||||
never expire a quote merely because reconciliation could not complete.
|
||||
"""
|
||||
async with _invoice_settlement_lock(invoice.id), wallet_operation_guard():
|
||||
minted = False
|
||||
payment_confirmed = False
|
||||
try:
|
||||
# Snapshot the row and end the caller's read transaction before any
|
||||
# potentially slow mint I/O. All final DB mutations use owned,
|
||||
# short-lived sessions below.
|
||||
await session.refresh(invoice)
|
||||
if invoice.status not in _RETRYABLE_INVOICE_STATUSES:
|
||||
await session.commit()
|
||||
return False
|
||||
observed_status = invoice.status
|
||||
settlement = _InvoiceSettlement.from_invoice(invoice)
|
||||
await session.commit()
|
||||
|
||||
async def _check_invoice_payment_locked(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
) -> None:
|
||||
minted = False
|
||||
invoice_id = invoice.id
|
||||
invoice_purpose = invoice.purpose
|
||||
invoice_amount_sats = invoice.amount_sats
|
||||
invoice_payment_hash = invoice.payment_hash
|
||||
finalized_api_key_hash = invoice.api_key_hash
|
||||
try:
|
||||
# A preceding invoice lookup starts a transaction. End it before the
|
||||
# potentially slow mint request so it cannot pin a pool connection.
|
||||
await session.commit()
|
||||
mint_url = settlement.mint_url or settings.primary_mint
|
||||
wallet = await get_wallet(mint_url, "sat")
|
||||
mint_status = await run_mint_operation(
|
||||
lambda: wallet.get_mint_quote(settlement.payment_hash),
|
||||
op_name="get_mint_quote",
|
||||
mint_url=mint_url,
|
||||
)
|
||||
if not mint_status.paid:
|
||||
return getattr(mint_status, "state", None) == MintQuoteState.unpaid
|
||||
payment_confirmed = True
|
||||
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
mint_status = await wallet.get_mint_quote(invoice_payment_hash)
|
||||
if not mint_status.paid:
|
||||
return
|
||||
# Fence expiry and other workers before consuming the paid quote.
|
||||
# If a concurrent expiry/finalization won, this worker must not mint.
|
||||
if not await _claim_paid_invoice_for_settlement(
|
||||
invoice, session, observed_status
|
||||
):
|
||||
return False
|
||||
|
||||
# Do not redeem a paid top-up quote if its target has already been
|
||||
# pruned. This validation owns a short-lived session and releases its
|
||||
# connection before mint redemption starts.
|
||||
if invoice_purpose == "topup":
|
||||
if not finalized_api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
async with create_session() as validation_session:
|
||||
target = await validation_session.get(ApiKey, finalized_api_key_hash)
|
||||
if target is None:
|
||||
terminal = await validation_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice_id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(status="reconciliation_required")
|
||||
# Reject a paid top-up whose target was pruned before redeeming its
|
||||
# single-use quote. The validation session is closed before mint I/O.
|
||||
if settlement.purpose == "topup":
|
||||
if not settlement.api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
async with create_session() as validation_session:
|
||||
target = await validation_session.get(
|
||||
ApiKey, settlement.api_key_hash
|
||||
)
|
||||
await validation_session.commit()
|
||||
if terminal.rowcount == 1:
|
||||
set_committed_value(
|
||||
invoice, "status", "reconciliation_required"
|
||||
)
|
||||
else:
|
||||
committed_invoice = await validation_session.get(
|
||||
LightningInvoice, invoice_id
|
||||
)
|
||||
if committed_invoice is not None:
|
||||
set_committed_value(
|
||||
invoice, "status", committed_invoice.status
|
||||
if target is None:
|
||||
terminal = await validation_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == settlement.id,
|
||||
col(LightningInvoice.status).in_(
|
||||
_RETRYABLE_INVOICE_STATUSES
|
||||
),
|
||||
)
|
||||
logger.critical(
|
||||
"Paid topup invoice target API key was not found; reconciliation required",
|
||||
extra={"invoice_id": invoice_id},
|
||||
)
|
||||
return
|
||||
.values(status="reconciliation_required")
|
||||
)
|
||||
await validation_session.commit()
|
||||
if terminal.rowcount == 1:
|
||||
_publish_invoice_value(
|
||||
invoice, "status", "reconciliation_required"
|
||||
)
|
||||
else:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
logger.critical(
|
||||
"Paid topup invoice target API key was not found; reconciliation required",
|
||||
extra={"invoice_id": settlement.id},
|
||||
)
|
||||
return False
|
||||
|
||||
# The mint enforces single-use quotes, so a concurrent checker that
|
||||
# races us here fails inside wallet.mint rather than double-minting.
|
||||
await wallet.mint(invoice_amount_sats, quote_id=invoice_payment_hash)
|
||||
minted = True
|
||||
# Quote-linked proof verification makes an ambiguous mint response
|
||||
# retryable without crediting unrelated wallet balance growth.
|
||||
await _mint_invoice_quote(wallet, settlement)
|
||||
minted = True
|
||||
|
||||
# Paid finalization owns a fresh session. The API/watcher session is
|
||||
# never rolled back by this function, so its invoice and sibling ORM
|
||||
# objects remain usable after a DB failure or lost CAS race.
|
||||
async with create_session() as finalization_session:
|
||||
if invoice_purpose == "create":
|
||||
api_key = await _create_api_key_record(invoice, finalization_session)
|
||||
finalized_api_key_hash = api_key.hashed_key
|
||||
elif invoice_purpose == "topup":
|
||||
await _credit_topup_record(invoice, finalization_session)
|
||||
|
||||
# Conditional transition guards against double-credit: the credit
|
||||
# above and this status flip commit atomically, and a lost race
|
||||
# rolls both back in the owned finalization session.
|
||||
paid_at = int(time.time())
|
||||
finalized = await finalization_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice_id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(
|
||||
status="paid",
|
||||
paid_at=paid_at,
|
||||
api_key_hash=finalized_api_key_hash,
|
||||
async with create_session() as finalization_session:
|
||||
settled, api_key_hash = await _finalize_invoice_settlement(
|
||||
settlement, finalization_session, paid_at
|
||||
)
|
||||
if not settled:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
return False
|
||||
|
||||
_publish_invoice_value(invoice, "status", "paid")
|
||||
_publish_invoice_value(invoice, "paid_at", paid_at)
|
||||
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
|
||||
logger.info(
|
||||
"Lightning invoice paid",
|
||||
extra={
|
||||
"invoice_id": settlement.id,
|
||||
"amount_sats": settlement.amount_sats,
|
||||
"purpose": settlement.purpose,
|
||||
"api_key_hash": api_key_hash[:8] + "..."
|
||||
if api_key_hash
|
||||
else None,
|
||||
},
|
||||
)
|
||||
if finalized.rowcount != 1:
|
||||
await finalization_session.rollback()
|
||||
committed_invoice = await finalization_session.get(
|
||||
LightningInvoice, invoice_id
|
||||
)
|
||||
await finalization_session.commit()
|
||||
if committed_invoice is not None:
|
||||
# A concurrent finalizer won the CAS. Publish only the
|
||||
# state observed from the database after ending the owned
|
||||
# read transaction; never refresh the caller's session.
|
||||
set_committed_value(
|
||||
invoice, "api_key_hash", committed_invoice.api_key_hash
|
||||
return False
|
||||
except BaseException as error:
|
||||
# Never roll back the caller-owned session: doing so expires invoice
|
||||
# and sibling ORM objects. Owned sessions roll themselves back.
|
||||
if payment_confirmed and invoice.status != "settlement_pending":
|
||||
try:
|
||||
async with create_session() as state_session:
|
||||
pending = await state_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice.id,
|
||||
col(LightningInvoice.status).in_(
|
||||
_RETRYABLE_INVOICE_STATUSES
|
||||
),
|
||||
)
|
||||
.values(status="settlement_pending")
|
||||
)
|
||||
await state_session.commit()
|
||||
if pending.rowcount == 1:
|
||||
_publish_invoice_value(
|
||||
invoice, "status", "settlement_pending"
|
||||
)
|
||||
except Exception as state_error:
|
||||
logger.critical(
|
||||
"Paid invoice reconciliation state could not be persisted",
|
||||
extra={"invoice_id": invoice.id, "error": str(state_error)},
|
||||
)
|
||||
set_committed_value(invoice, "status", committed_invoice.status)
|
||||
set_committed_value(invoice, "paid_at", committed_invoice.paid_at)
|
||||
return
|
||||
await finalization_session.commit()
|
||||
if minted:
|
||||
logger.critical(
|
||||
"Invoice mint succeeded but DB finalization failed; reconciliation required",
|
||||
extra={"invoice_id": invoice.id, "purpose": invoice.purpose},
|
||||
)
|
||||
try:
|
||||
await _reload_invoice_view(invoice, session)
|
||||
except Exception:
|
||||
pass
|
||||
if not isinstance(error, Exception):
|
||||
raise
|
||||
logger.error(f"Failed to check invoice payment: {error}")
|
||||
return False
|
||||
|
||||
# Only publish finalized values to the caller-owned object after the
|
||||
# owned transaction has committed successfully.
|
||||
set_committed_value(invoice, "api_key_hash", finalized_api_key_hash)
|
||||
set_committed_value(invoice, "status", "paid")
|
||||
set_committed_value(invoice, "paid_at", paid_at)
|
||||
|
||||
logger.info(
|
||||
"Lightning invoice paid",
|
||||
extra={
|
||||
"invoice_id": invoice_id,
|
||||
"amount_sats": invoice_amount_sats,
|
||||
"purpose": invoice_purpose,
|
||||
"api_key_hash": finalized_api_key_hash[:8] + "..."
|
||||
if finalized_api_key_hash
|
||||
else None,
|
||||
},
|
||||
def _is_outputs_already_signed(error: BaseException) -> bool:
|
||||
message = str(error)
|
||||
return bool(
|
||||
re.search(
|
||||
r"\boutputs?\s+(?:have\s+)?already\s+(?:been\s+)?signed(?:\s+before)?\b",
|
||||
message,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
except BaseException as e:
|
||||
# BaseException so task cancellation (e.g. client disconnect) after a
|
||||
# successful mint still triggers the reconciliation alert. Any rollback
|
||||
# belongs to create_session(), never to the caller-owned session.
|
||||
if minted:
|
||||
logger.critical(
|
||||
"Invoice mint succeeded but DB finalization failed; reconciliation required",
|
||||
extra={"invoice_id": invoice_id, "purpose": invoice_purpose},
|
||||
)
|
||||
if not isinstance(e, Exception):
|
||||
and re.search(r"\bcode\s*:\s*11003\b", message, re.IGNORECASE)
|
||||
)
|
||||
|
||||
|
||||
def _invoice_quote_proof_amount(wallet: Any, quote_id: str) -> int:
|
||||
"""Return spendable wallet value minted by one Lightning quote."""
|
||||
return sum(
|
||||
proof.amount
|
||||
for proof in wallet.proofs
|
||||
if proof.mint_id == quote_id and not proof.reserved
|
||||
)
|
||||
|
||||
|
||||
async def _mint_invoice_quote(
|
||||
wallet: Any, invoice: LightningInvoice | _InvoiceSettlement
|
||||
) -> None:
|
||||
"""Mint a paid quote, proving quote-linked outputs before DB credit."""
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
await wallet.load_proofs(reload=True)
|
||||
if _invoice_quote_proof_amount(wallet, invoice.payment_hash) >= invoice.amount_sats:
|
||||
return
|
||||
|
||||
try:
|
||||
await run_mint_operation(
|
||||
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
|
||||
op_name=f"invoice_mint_{invoice.purpose}",
|
||||
mint_url=mint_url,
|
||||
retry_timeouts=False,
|
||||
)
|
||||
except Exception as error:
|
||||
if not _is_outputs_already_signed(error):
|
||||
raise
|
||||
logger.error(f"Failed to check invoice payment: {e}")
|
||||
|
||||
for keyset_id in wallet.keysets:
|
||||
await wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
|
||||
await wallet.load_proofs(reload=True)
|
||||
recovered = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
|
||||
if recovered < invoice.amount_sats:
|
||||
raise RuntimeError(
|
||||
"Invoice outputs were already signed but quote-linked recovery returned "
|
||||
f"{recovered} sats; expected at least {invoice.amount_sats}"
|
||||
) from error
|
||||
else:
|
||||
await wallet.load_proofs(reload=True)
|
||||
minted_amount = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
|
||||
if minted_amount < invoice.amount_sats:
|
||||
raise RuntimeError(
|
||||
"Invoice mint succeeded but quote-linked proofs total "
|
||||
f"{minted_amount} sats; expected at least {invoice.amount_sats}"
|
||||
)
|
||||
|
||||
|
||||
def _invoice_api_key_hash(invoice: LightningInvoice | _InvoiceSettlement) -> str:
|
||||
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
|
||||
return hashlib.sha256(dummy_token.encode()).hexdigest()
|
||||
|
||||
|
||||
async def _create_api_key_record(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> ApiKey:
|
||||
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
|
||||
hashed_key = hashlib.sha256(dummy_token.encode()).hexdigest()
|
||||
mint_url = invoice.mint_url or settings.primary_mint
|
||||
api_key = ApiKey(
|
||||
hashed_key=hashed_key,
|
||||
hashed_key=_invoice_api_key_hash(invoice),
|
||||
balance=invoice.amount_sats * 1000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url=settings.primary_mint,
|
||||
refund_mint_url=mint_url,
|
||||
balance_limit=invoice.balance_limit,
|
||||
balance_limit_reset=invoice.balance_limit_reset,
|
||||
validity_date=invoice.validity_date,
|
||||
@@ -384,50 +658,142 @@ async def _create_api_key_record(
|
||||
return api_key
|
||||
|
||||
|
||||
async def _credit_topup_record(
|
||||
invoice: LightningInvoice, session: AsyncSession
|
||||
async def _topup_api_key_record(
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> None:
|
||||
if not invoice.api_key_hash:
|
||||
raise ValueError("No API key associated with topup invoice")
|
||||
credited = await session.exec( # type: ignore[call-overload]
|
||||
result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == invoice.api_key_hash)
|
||||
.values(balance=col(ApiKey.balance) + invoice.amount_sats * 1000)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if credited.rowcount != 1:
|
||||
if result.rowcount != 1:
|
||||
raise ValueError("Associated API key not found")
|
||||
|
||||
|
||||
INVOICE_WATCH_INTERVAL_SECONDS = 5
|
||||
async def _finalize_invoice_settlement(
|
||||
invoice: _InvoiceSettlement, session: AsyncSession, paid_at: int
|
||||
) -> tuple[bool, str | None]:
|
||||
"""Atomically fence and apply one invoice credit in the provided owned session."""
|
||||
api_key_hash = (
|
||||
_invoice_api_key_hash(invoice)
|
||||
if invoice.purpose == "create"
|
||||
else invoice.api_key_hash
|
||||
)
|
||||
claim = await session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(col(LightningInvoice.id) == invoice.id)
|
||||
.where(
|
||||
col(LightningInvoice.status).in_(_RETRYABLE_INVOICE_STATUSES)
|
||||
)
|
||||
.values(status="paid", paid_at=paid_at, api_key_hash=api_key_hash)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if claim.rowcount != 1:
|
||||
await session.rollback()
|
||||
return False, None
|
||||
|
||||
if invoice.purpose == "create":
|
||||
await _create_api_key_record(invoice, session)
|
||||
elif invoice.purpose == "topup":
|
||||
await _topup_api_key_record(invoice, session)
|
||||
else:
|
||||
raise ValueError(f"Unsupported invoice purpose: {invoice.purpose}")
|
||||
await session.commit()
|
||||
return True, api_key_hash
|
||||
|
||||
|
||||
async def _reload_invoice_view(
|
||||
invoice: LightningInvoice, _caller_session: AsyncSession
|
||||
) -> None:
|
||||
"""Publish committed invoice state without touching the caller transaction."""
|
||||
async with create_session() as reload_session:
|
||||
stored = await reload_session.get(LightningInvoice, invoice.id)
|
||||
if stored is None:
|
||||
return
|
||||
status = stored.status
|
||||
paid_at = stored.paid_at
|
||||
api_key_hash = stored.api_key_hash
|
||||
await reload_session.commit()
|
||||
_publish_invoice_value(invoice, "status", status)
|
||||
_publish_invoice_value(invoice, "paid_at", paid_at)
|
||||
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
|
||||
|
||||
|
||||
async def _expire_invoice_if_authoritatively_unpaid(
|
||||
invoice: LightningInvoice,
|
||||
caller_session: AsyncSession,
|
||||
definitively_unpaid: bool,
|
||||
) -> bool:
|
||||
"""Expire one overdue unpaid invoice without overwriting concurrent settlement."""
|
||||
if (
|
||||
not definitively_unpaid
|
||||
or invoice.status != "pending"
|
||||
or int(time.time()) <= invoice.expires_at
|
||||
):
|
||||
return False
|
||||
|
||||
async with create_session() as expiry_session:
|
||||
expired = await expiry_session.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.id) == invoice.id,
|
||||
col(LightningInvoice.status) == "pending",
|
||||
)
|
||||
.values(status="expired")
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
await expiry_session.commit()
|
||||
|
||||
if expired.rowcount == 1:
|
||||
_publish_invoice_value(invoice, "status", "expired")
|
||||
return True
|
||||
|
||||
await _reload_invoice_view(invoice, caller_session)
|
||||
return False
|
||||
|
||||
|
||||
async def _credit_topup_record(
|
||||
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
|
||||
) -> None:
|
||||
await _topup_api_key_record(invoice, session)
|
||||
|
||||
|
||||
# Nutshell mints throttle Lightning backend lookups to once per 10s per
|
||||
# quote, so polling faster just burns the global request budget for nothing.
|
||||
INVOICE_WATCH_INTERVAL_SECONDS = 10
|
||||
INVOICE_WATCH_BATCH_LIMIT = 100
|
||||
|
||||
|
||||
async def periodic_invoice_watcher() -> None:
|
||||
"""Background task: detect paid Lightning invoices and credit balances.
|
||||
async def _process_invoice_watch_batch(session: AsyncSession) -> None:
|
||||
result = await session.exec(
|
||||
select(LightningInvoice)
|
||||
.where(
|
||||
col(LightningInvoice.status).in_(_RETRYABLE_INVOICE_STATUSES)
|
||||
)
|
||||
.limit(INVOICE_WATCH_BATCH_LIMIT)
|
||||
)
|
||||
for invoice in result.all():
|
||||
try:
|
||||
definitively_unpaid = await check_invoice_payment(invoice, session)
|
||||
await _expire_invoice_if_authoritatively_unpaid(
|
||||
invoice, session, definitively_unpaid
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Invoice watcher failed for invoice",
|
||||
extra={"invoice_id": invoice.id, "error": str(e)},
|
||||
)
|
||||
|
||||
Removes the need for clients to poll the status endpoint after paying.
|
||||
"""
|
||||
|
||||
async def periodic_invoice_watcher() -> None:
|
||||
"""Background task: detect paid Lightning invoices and credit balances."""
|
||||
while True:
|
||||
try:
|
||||
async with create_session() as session:
|
||||
now = int(time.time())
|
||||
result = await session.exec(
|
||||
select(LightningInvoice)
|
||||
.where(
|
||||
LightningInvoice.status == "pending",
|
||||
col(LightningInvoice.expires_at) > now,
|
||||
)
|
||||
.limit(INVOICE_WATCH_BATCH_LIMIT)
|
||||
)
|
||||
pending = result.all()
|
||||
for invoice in pending:
|
||||
try:
|
||||
await check_invoice_payment(invoice, session)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Invoice watcher failed for invoice",
|
||||
extra={"invoice_id": invoice.id, "error": str(e)},
|
||||
)
|
||||
await _process_invoice_watch_batch(session)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
||||
+343
@@ -0,0 +1,343 @@
|
||||
"""Shared policy for bounded, rate-aware Cashu mint API operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import socket
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Any, AsyncGenerator, Awaitable, Callable
|
||||
|
||||
import httpx
|
||||
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
MINT_TRANSPORT_EXCEPTIONS: tuple[type[BaseException], ...] = (
|
||||
httpx.NetworkError,
|
||||
httpx.TimeoutException,
|
||||
ConnectionError,
|
||||
socket.gaierror,
|
||||
asyncio.TimeoutError,
|
||||
)
|
||||
|
||||
MINT_TRANSPORT_COOLDOWN_SECONDS = 30.0
|
||||
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS = 60.0
|
||||
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS = 7 * 60 * 60
|
||||
|
||||
_fail_fast_depth: ContextVar[int] = ContextVar("mint_fail_fast_depth", default=0)
|
||||
|
||||
|
||||
class MintRateLimitedError(httpx.HTTPStatusError):
|
||||
"""Typed boundary error preserving a Cashu mint's HTTP 429 response."""
|
||||
|
||||
|
||||
class MintCooldownError(Exception):
|
||||
"""A mint is cooling down and this operation must not wait."""
|
||||
|
||||
def __init__(self, mint_url: str, retry_after_seconds: float):
|
||||
self.mint_url = mint_url
|
||||
self.retry_after_seconds = max(0.0, retry_after_seconds)
|
||||
super().__init__(
|
||||
f"Mint {mint_url} is cooling down; retry after "
|
||||
f"{self.retry_after_seconds:.2f}s"
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def fail_fast_mint_operations() -> AsyncGenerator[None, None]:
|
||||
"""Make mint cooldown/probe waits fail fast in the current task.
|
||||
|
||||
Wallet mutation code holds a process-wide file lock. It enters this scope so
|
||||
an existing mint cooldown can never turn that lock into a multi-hour wait.
|
||||
"""
|
||||
|
||||
token = _fail_fast_depth.set(_fail_fast_depth.get() + 1)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_fail_fast_depth.reset(token)
|
||||
|
||||
|
||||
class MintRateGuard:
|
||||
"""Limit concurrency and remember per-mint cooldown/probe state."""
|
||||
|
||||
_guards: dict[str, "MintRateGuard"] = {}
|
||||
|
||||
@classmethod
|
||||
def get(cls, mint_url: str) -> "MintRateGuard":
|
||||
concurrency = settings.mint_max_concurrency
|
||||
guard = cls._guards.get(mint_url)
|
||||
if guard is None or guard._max_concurrency != concurrency:
|
||||
previous = guard
|
||||
guard = cls(mint_url, concurrency)
|
||||
if previous is not None:
|
||||
# Concurrency changed at runtime: keep the live cooldown/backoff
|
||||
# state so an active 429 cooldown is not silently discarded.
|
||||
guard._cooldown_until = previous._cooldown_until
|
||||
guard._cooldown_reason = previous._cooldown_reason
|
||||
guard._consecutive_rate_limits = previous._consecutive_rate_limits
|
||||
guard._needs_probe = previous._needs_probe
|
||||
cls._guards[mint_url] = guard
|
||||
return guard
|
||||
|
||||
def __init__(self, mint_url: str, max_concurrency: int):
|
||||
self._mint_url = mint_url
|
||||
self._max_concurrency = max_concurrency
|
||||
self._semaphore = (
|
||||
asyncio.Semaphore(max_concurrency) if max_concurrency > 0 else None
|
||||
)
|
||||
self._cooldown_until = 0.0
|
||||
self._cooldown_reason: str | None = None
|
||||
self._consecutive_rate_limits = 0
|
||||
self._needs_probe = False
|
||||
self._probe_lock = asyncio.Lock()
|
||||
|
||||
def apply_cooldown(self, delay: float, *, reason: str | None = None) -> None:
|
||||
deadline = time.monotonic() + max(0.0, delay)
|
||||
if deadline >= self._cooldown_until:
|
||||
self._cooldown_until = deadline
|
||||
if reason is not None:
|
||||
self._cooldown_reason = reason
|
||||
elif self._cooldown_reason is None and reason is not None:
|
||||
self._cooldown_reason = reason
|
||||
self._needs_probe = True
|
||||
|
||||
def apply_rate_limit_cooldown(self, retry_after: float | None = None) -> float:
|
||||
remaining = self.cooldown_remaining()
|
||||
if remaining > 0 and self._cooldown_reason == "rate_limited":
|
||||
minimum = min(
|
||||
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS,
|
||||
max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0),
|
||||
)
|
||||
if minimum > remaining:
|
||||
self.apply_cooldown(minimum, reason="rate_limited")
|
||||
return minimum
|
||||
return remaining
|
||||
|
||||
self._consecutive_rate_limits += 1
|
||||
base = max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0)
|
||||
multiplier = 2 ** min(self._consecutive_rate_limits - 1, 10)
|
||||
delay = min(_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS, base * multiplier)
|
||||
self.apply_cooldown(delay, reason="rate_limited")
|
||||
return delay
|
||||
|
||||
def cooldown_remaining(self) -> float:
|
||||
return max(0.0, self._cooldown_until - time.monotonic())
|
||||
|
||||
def cooldown_reason(self) -> str | None:
|
||||
return self._cooldown_reason if self.cooldown_remaining() > 0 else None
|
||||
|
||||
def _raise_if_wait_forbidden(self) -> None:
|
||||
remaining = self.cooldown_remaining()
|
||||
if _fail_fast_depth.get() and remaining > 0:
|
||||
raise MintCooldownError(self._mint_url, remaining)
|
||||
|
||||
async def _wait_for_cooldown(self) -> None:
|
||||
while True:
|
||||
self._raise_if_wait_forbidden()
|
||||
deadline = self._cooldown_until
|
||||
wait = max(0.0, deadline - time.monotonic())
|
||||
if wait <= 0:
|
||||
return
|
||||
logger.debug(
|
||||
"Mint rate guard: cooling down",
|
||||
extra={"mint_url": self._mint_url, "wait_seconds": round(wait, 2)},
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
if self._cooldown_until <= deadline:
|
||||
return
|
||||
|
||||
async def _run_probe(self, factory: Callable[[], Awaitable[Any]]) -> Any:
|
||||
await self._wait_for_cooldown()
|
||||
logger.info(
|
||||
"Mint cooldown ended; sending one probe request",
|
||||
extra={"event": "mint_cooldown_probe_started", "mint_url": self._mint_url},
|
||||
)
|
||||
try:
|
||||
result = await factory()
|
||||
except Exception as error:
|
||||
if is_mint_rate_limited(error):
|
||||
retry_after = None
|
||||
if isinstance(error, httpx.HTTPStatusError):
|
||||
retry_after = parse_retry_after(error.response.headers)
|
||||
self.apply_rate_limit_cooldown(retry_after)
|
||||
else:
|
||||
self.apply_cooldown(1.0)
|
||||
logger.warning(
|
||||
"Mint cooldown probe failed",
|
||||
extra={
|
||||
"event": "mint_cooldown_probe_failed",
|
||||
"mint_url": self._mint_url,
|
||||
"error": str(error),
|
||||
"error_type": type(error).__name__,
|
||||
"cooldown_seconds": round(self.cooldown_remaining(), 2),
|
||||
"consecutive_rate_limits": self._consecutive_rate_limits,
|
||||
},
|
||||
)
|
||||
raise
|
||||
|
||||
self._needs_probe = False
|
||||
self._cooldown_until = 0.0
|
||||
self._cooldown_reason = None
|
||||
self._consecutive_rate_limits = 0
|
||||
logger.info(
|
||||
"Mint cooldown probe succeeded; restoring normal concurrency",
|
||||
extra={
|
||||
"event": "mint_cooldown_probe_succeeded",
|
||||
"mint_url": self._mint_url,
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
async def run(self, factory: Callable[[], Awaitable[Any]]) -> Any:
|
||||
while True:
|
||||
self._raise_if_wait_forbidden()
|
||||
if self._needs_probe or self.cooldown_remaining() > 0:
|
||||
if _fail_fast_depth.get() and self._probe_lock.locked():
|
||||
raise MintCooldownError(self._mint_url, self.cooldown_remaining())
|
||||
async with self._probe_lock:
|
||||
self._raise_if_wait_forbidden()
|
||||
if self.cooldown_remaining() > 0:
|
||||
self._needs_probe = True
|
||||
if self._needs_probe:
|
||||
return await self._run_probe(factory)
|
||||
continue
|
||||
|
||||
if self._semaphore is None:
|
||||
return await factory()
|
||||
async with self._semaphore:
|
||||
self._raise_if_wait_forbidden()
|
||||
if self._needs_probe:
|
||||
continue
|
||||
return await factory()
|
||||
|
||||
|
||||
def mint_cooldown_remaining(mint_url: str) -> float:
|
||||
return MintRateGuard.get(mint_url).cooldown_remaining()
|
||||
|
||||
|
||||
def mint_cooldown_reason(mint_url: str) -> str | None:
|
||||
return MintRateGuard.get(mint_url).cooldown_reason()
|
||||
|
||||
|
||||
def is_mint_rate_limited(error: BaseException) -> bool:
|
||||
"""Return whether an exception chain represents HTTP 429/cooldown."""
|
||||
|
||||
current: BaseException | None = error
|
||||
seen: set[int] = set()
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
if isinstance(current, MintCooldownError):
|
||||
return True
|
||||
if isinstance(current, httpx.HTTPStatusError):
|
||||
if current.response.status_code == 429:
|
||||
return True
|
||||
current = current.__cause__ or current.__context__
|
||||
return False
|
||||
|
||||
|
||||
def parse_retry_after(headers: Any) -> float | None:
|
||||
raw = headers.get("retry-after") or headers.get("Retry-After")
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return float(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
async def run_mint_operation(
|
||||
factory: Callable[[], Awaitable[Any]],
|
||||
*,
|
||||
op_name: str = "mint_operation",
|
||||
mint_url: str = "",
|
||||
retry_timeouts: bool = True,
|
||||
retry_on_rate_limit: bool = True,
|
||||
) -> Any:
|
||||
"""Run one mint operation with bounded concurrency and adaptive cooldown."""
|
||||
|
||||
guard = MintRateGuard.get(mint_url) if mint_url else None
|
||||
timeout = settings.mint_operation_timeout_seconds
|
||||
max_attempts = settings.mint_retry_max_attempts + 1
|
||||
|
||||
async def timed_factory() -> Any:
|
||||
if timeout > 0:
|
||||
return await asyncio.wait_for(factory(), timeout=timeout)
|
||||
return await factory()
|
||||
|
||||
async def invoke() -> Any:
|
||||
if guard is not None:
|
||||
return await guard.run(timed_factory)
|
||||
return await timed_factory()
|
||||
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
return await invoke()
|
||||
except MintCooldownError:
|
||||
raise
|
||||
except (asyncio.TimeoutError, httpx.TimeoutException) as exc:
|
||||
if retry_timeouts and attempt < max_attempts - 1:
|
||||
backoff = (2**attempt) + (time.monotonic() % 1.0)
|
||||
logger.warning(
|
||||
"Mint operation timed out, retrying",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"attempt": attempt + 1,
|
||||
"backoff_seconds": round(backoff, 2),
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
continue
|
||||
raise httpx.TimeoutException(
|
||||
f"{op_name} timed out (attempts: {attempt + 1})"
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
if not is_mint_rate_limited(exc):
|
||||
raise
|
||||
|
||||
backoff = (2**attempt) + (time.monotonic() % 1.0)
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
retry_after = parse_retry_after(exc.response.headers)
|
||||
if retry_after is not None:
|
||||
backoff = max(retry_after, backoff)
|
||||
cooldown = backoff
|
||||
if guard is not None:
|
||||
cooldown = guard.apply_rate_limit_cooldown(backoff)
|
||||
|
||||
if not retry_on_rate_limit:
|
||||
logger.warning(
|
||||
"Mint rate-limited, skipping retries for fallback",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"cooldown_seconds": round(cooldown, 2),
|
||||
"consecutive_rate_limits": guard._consecutive_rate_limits
|
||||
if guard is not None
|
||||
else attempt + 1,
|
||||
},
|
||||
)
|
||||
raise
|
||||
|
||||
if attempt >= max_attempts - 1:
|
||||
raise
|
||||
logger.warning(
|
||||
"Mint rate-limited, applying cooldown",
|
||||
extra={
|
||||
"op_name": op_name,
|
||||
"mint_url": mint_url,
|
||||
"attempt": attempt + 1,
|
||||
"cooldown_seconds": round(cooldown, 2),
|
||||
"consecutive_rate_limits": guard._consecutive_rate_limits
|
||||
if guard is not None
|
||||
else attempt + 1,
|
||||
},
|
||||
)
|
||||
if guard is None:
|
||||
await asyncio.sleep(cooldown)
|
||||
|
||||
raise RuntimeError(f"{op_name}: exhausted retries unexpectedly")
|
||||
@@ -18,7 +18,6 @@ from ..wallet import deserialize_token_from_string
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> None:
|
||||
if x_cashu := headers.get("x-cashu", None):
|
||||
cashu_token = x_cashu
|
||||
@@ -243,7 +242,7 @@ async def calculate_discounted_max_cost(
|
||||
},
|
||||
)
|
||||
|
||||
return max(0, adjusted)
|
||||
return max(settings.min_request_msat, adjusted)
|
||||
|
||||
|
||||
def estimate_tokens(messages: list) -> int:
|
||||
|
||||
+63
-14
@@ -1,16 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
from typing import TypedDict
|
||||
|
||||
import httpx
|
||||
from cashu.core.base import MeltQuoteState
|
||||
from cashu.wallet.wallet import Proof, Wallet
|
||||
|
||||
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
|
||||
# very slow mint can block a melt (and any caller, e.g. the payout loop)
|
||||
# indefinitely. Bound it here so callers fail instead of hanging forever.
|
||||
MELT_TIMEOUT_SECONDS = 60
|
||||
from ..mint import (
|
||||
MINT_TRANSPORT_EXCEPTIONS,
|
||||
is_mint_rate_limited,
|
||||
run_mint_operation,
|
||||
)
|
||||
|
||||
try:
|
||||
from bech32 import bech32_decode, convertbits # type: ignore
|
||||
@@ -31,6 +32,15 @@ class LNURLError(Exception):
|
||||
"""LNURL related errors."""
|
||||
|
||||
|
||||
class MeltOutcomeAmbiguousError(LNURLError):
|
||||
"""A melt was dispatched but its final outcome could not be confirmed.
|
||||
|
||||
Callers must NOT treat this as a clean failure: the payment may still
|
||||
settle, so debits backing it must be kept until reconciliation confirms
|
||||
the true outcome.
|
||||
"""
|
||||
|
||||
|
||||
async def decode_lnurl(lnurl: str) -> str:
|
||||
"""Decode LNURL to get the actual URL.
|
||||
|
||||
@@ -221,23 +231,62 @@ async def raw_send_to_lnurl(
|
||||
lnurl_data["callback_url"], final_amount
|
||||
)
|
||||
|
||||
melt_quote_resp = await wallet.melt_quote(invoice=bolt11_invoice)
|
||||
melt_quote_resp = await run_mint_operation(
|
||||
lambda: wallet.melt_quote(invoice=bolt11_invoice),
|
||||
op_name="lnurl_melt_quote",
|
||||
mint_url=str(wallet.url),
|
||||
)
|
||||
|
||||
if amount:
|
||||
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
|
||||
|
||||
try:
|
||||
_ = await asyncio.wait_for(
|
||||
wallet.melt(
|
||||
melt_response = await run_mint_operation(
|
||||
lambda: wallet.melt(
|
||||
proofs=proofs,
|
||||
invoice=bolt11_invoice,
|
||||
fee_reserve_sat=melt_quote_resp.fee_reserve,
|
||||
quote_id=melt_quote_resp.quote,
|
||||
),
|
||||
timeout=MELT_TIMEOUT_SECONDS,
|
||||
op_name="lnurl_melt",
|
||||
mint_url=str(wallet.url),
|
||||
retry_timeouts=False,
|
||||
)
|
||||
except asyncio.TimeoutError as e:
|
||||
raise LNURLError(
|
||||
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
|
||||
) from e
|
||||
return final_amount
|
||||
except Exception as error:
|
||||
if is_mint_rate_limited(error):
|
||||
# Cooldown failures happen before dispatch, and HTTP 429 means the
|
||||
# mint rejected the request. Neither outcome may keep proofs
|
||||
# reserved as though a Lightning payment could still settle.
|
||||
await wallet.set_reserved_for_send(proofs, reserved=False)
|
||||
raise
|
||||
if not isinstance(error, MINT_TRANSPORT_EXCEPTIONS):
|
||||
raise
|
||||
melt_response = None
|
||||
melt_error: BaseException | None = error
|
||||
else:
|
||||
melt_error = None
|
||||
|
||||
if getattr(melt_response, "state", None) == MeltQuoteState.paid:
|
||||
return final_amount
|
||||
|
||||
try:
|
||||
quote = await run_mint_operation(
|
||||
lambda: wallet.get_melt_quote(melt_quote_resp.quote),
|
||||
op_name="reconcile_lnurl_melt_quote",
|
||||
mint_url=str(wallet.url),
|
||||
retry_timeouts=False,
|
||||
)
|
||||
except Exception as reconciliation_error:
|
||||
raise MeltOutcomeAmbiguousError(
|
||||
"Melt outcome is ambiguous; quote reconciliation failed and proofs "
|
||||
"must not be retried"
|
||||
) from reconciliation_error
|
||||
|
||||
if quote is not None and quote.state == MeltQuoteState.paid:
|
||||
return final_amount
|
||||
|
||||
state = getattr(getattr(quote, "state", None), "value", "unknown")
|
||||
raise MeltOutcomeAmbiguousError(
|
||||
"Melt outcome is ambiguous; proofs must not be retried "
|
||||
f"(quote_state={state})"
|
||||
) from melt_error
|
||||
|
||||
+20
-12
@@ -26,7 +26,6 @@ from .core.db import (
|
||||
)
|
||||
from .core.exceptions import UpstreamError
|
||||
from .core.not_found import build_not_found_response
|
||||
from .core.settings import settings
|
||||
from .payment.helpers import (
|
||||
calculate_discounted_max_cost,
|
||||
check_token_balance,
|
||||
@@ -367,8 +366,6 @@ async def _proxy(
|
||||
max_cost_for_model = await calculate_discounted_max_cost(
|
||||
_max_cost_for_model, request_body_dict, model_obj=model_obj
|
||||
)
|
||||
# Ensure max_cost_for_model is at least the minimum allowed request cost
|
||||
max_cost_for_model = max(max_cost_for_model, settings.min_request_msat)
|
||||
|
||||
check_token_balance(headers, request_body_dict, max_cost_for_model)
|
||||
|
||||
@@ -507,7 +504,6 @@ async def _proxy(
|
||||
candidate_max = await calculate_discounted_max_cost(
|
||||
candidate_max, request_body_dict, model_obj=model_obj
|
||||
)
|
||||
candidate_max = max(candidate_max, settings.min_request_msat)
|
||||
if candidate_max > max_cost_for_model:
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
@@ -807,17 +803,29 @@ async def get_bearer_token_key(
|
||||
},
|
||||
)
|
||||
return key
|
||||
except Exception as e:
|
||||
key_preview = bearer_key[:20] + "..." if len(bearer_key) > 20 else bearer_key
|
||||
logger.error(
|
||||
f"Bearer token validation failed: {type(e).__name__}: {e} path={path} model={model_id!r} min_cost={min_cost} key={key_preview!r}",
|
||||
except HTTPException as error:
|
||||
detail: dict[str, Any] = error.detail if isinstance(error.detail, dict) else {}
|
||||
raw_error = detail.get("error")
|
||||
error_info = raw_error if isinstance(raw_error, dict) else {}
|
||||
logger.warning(
|
||||
"Bearer token rejected",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
"status_code": error.status_code,
|
||||
"error_code": error_info.get("code"),
|
||||
"path": path,
|
||||
"model_id": model_id,
|
||||
"min_cost_msat": min_cost,
|
||||
"bearer_key_preview": key_preview,
|
||||
"required_msat": min_cost,
|
||||
},
|
||||
)
|
||||
raise
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
"Bearer token validation failed",
|
||||
extra={
|
||||
"error_type": type(error).__name__,
|
||||
"path": path,
|
||||
"model_id": model_id,
|
||||
"required_msat": min_cost,
|
||||
},
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -12,7 +12,7 @@ from ..core.db import (
|
||||
from ..core.db import (
|
||||
store_cashu_transaction_with_retry as store_cashu_transaction,
|
||||
)
|
||||
from ..wallet import send_token
|
||||
from ..wallet import release_token_reservation, send_token, token_mint_url
|
||||
from .routstr import RoutstrUpstreamProvider
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -144,12 +144,13 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
|
||||
)
|
||||
return
|
||||
|
||||
actual_mint_url = token_mint_url(token, mint_url)
|
||||
try:
|
||||
await store_cashu_transaction(
|
||||
token=token,
|
||||
amount=amount,
|
||||
unit="sat",
|
||||
mint_url=mint_url,
|
||||
mint_url=actual_mint_url,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="auto_topup",
|
||||
@@ -157,8 +158,24 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
|
||||
except Exception:
|
||||
logger.critical(
|
||||
"Aborting auto top-up because its cashu token could not be persisted",
|
||||
extra={"provider_id": row.id, "mint_url": mint_url},
|
||||
extra={"provider_id": row.id, "mint_url": actual_mint_url},
|
||||
)
|
||||
try:
|
||||
await release_token_reservation(token)
|
||||
except Exception as error:
|
||||
logger.critical(
|
||||
"Failed to release untracked auto-topup token",
|
||||
extra={
|
||||
"provider_id": row.id,
|
||||
"mint_url": actual_mint_url,
|
||||
"error": str(error),
|
||||
},
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Auto-topup token was released after persistence failed",
|
||||
extra={"provider_id": row.id, "mint_url": actual_mint_url},
|
||||
)
|
||||
return
|
||||
|
||||
result = await provider.topup(token)
|
||||
|
||||
@@ -53,6 +53,7 @@ from ..wallet import (
|
||||
classify_redemption_error,
|
||||
recieve_token,
|
||||
send_token,
|
||||
token_mint_url,
|
||||
)
|
||||
from . import messages_dispatch
|
||||
from .cache_breakpoints import (
|
||||
@@ -3520,7 +3521,7 @@ class BaseUpstreamProvider:
|
||||
token=refund_token,
|
||||
amount=amount,
|
||||
unit=unit,
|
||||
mint_url=mint,
|
||||
mint_url=token_mint_url(refund_token, mint),
|
||||
typ="out",
|
||||
request_id=request_id,
|
||||
)
|
||||
@@ -3873,7 +3874,7 @@ class BaseUpstreamProvider:
|
||||
token=refund_token,
|
||||
amount=emergency_refund,
|
||||
unit=unit,
|
||||
mint_url=mint,
|
||||
mint_url=token_mint_url(refund_token, mint),
|
||||
typ="out",
|
||||
request_id=request_id,
|
||||
)
|
||||
@@ -4843,7 +4844,7 @@ class BaseUpstreamProvider:
|
||||
token=refund_token,
|
||||
amount=emergency_refund,
|
||||
unit=unit,
|
||||
mint_url=mint,
|
||||
mint_url=token_mint_url(refund_token, mint),
|
||||
typ="out",
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
+1081
-228
File diff suppressed because it is too large
Load Diff
@@ -203,8 +203,13 @@ class TestmintWallet:
|
||||
token_base64 = base64.urlsafe_b64encode(token_json.encode()).decode()
|
||||
return f"cashuA{token_base64}"
|
||||
|
||||
async def redeem_token(self, token: str) -> Tuple[int, str, str]:
|
||||
"""Redeem a Cashu token - compatible with wallet.recieve_token"""
|
||||
async def redeem_token(
|
||||
self,
|
||||
token: str,
|
||||
destination_mint: str | None = None,
|
||||
destination_unit: str | None = None,
|
||||
) -> Tuple[int, str, str]:
|
||||
"""Redeem a Cashu token - compatible with wallet.recieve_token."""
|
||||
if not self.wallet:
|
||||
await self.init()
|
||||
|
||||
|
||||
@@ -207,8 +207,35 @@ async def test_pay_for_request_succeeds_when_balance_equals_cost(
|
||||
assert key.balance == model_cost # balance unchanged, only reserved goes up
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_model_maximum_is_required_and_reserved(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
from routstr.auth import pay_for_request, validate_bearer_key
|
||||
|
||||
short_key = _key(balance=95_000)
|
||||
exact_key = _key(balance=100_000)
|
||||
integration_session.add(short_key)
|
||||
integration_session.add(exact_key)
|
||||
await integration_session.commit()
|
||||
|
||||
with pytest.raises(HTTPException) as insufficient:
|
||||
await validate_bearer_key(
|
||||
f"sk-{short_key.hashed_key}", integration_session, min_cost=100_000
|
||||
)
|
||||
assert insufficient.value.status_code == 402
|
||||
|
||||
validated = await validate_bearer_key(
|
||||
f"sk-{exact_key.hashed_key}", integration_session, min_cost=100_000
|
||||
)
|
||||
await pay_for_request(validated, 100_000, integration_session)
|
||||
|
||||
await integration_session.refresh(exact_key)
|
||||
assert exact_key.reserved_balance == 100_000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 6 — HTTP layer returns 402 JSON with the right shape
|
||||
# HTTP layer returns 402 JSON with the right shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -266,8 +293,8 @@ async def test_http_402_response_shape_on_insufficient_balance(
|
||||
error = body["detail"]["error"]
|
||||
assert error["code"] == "insufficient_balance"
|
||||
assert error["type"] == "insufficient_quota"
|
||||
assert str(model_cost) in error["message"]
|
||||
assert str(user_balance) in error["message"]
|
||||
assert "622.888 sats (622888 msats) required" in error["message"]
|
||||
assert "20.32 sats (20320 msats) available" in error["message"]
|
||||
|
||||
# Balance must be completely untouched
|
||||
await integration_session.refresh(key)
|
||||
|
||||
@@ -14,6 +14,7 @@ import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import Proof
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
@@ -22,6 +23,12 @@ from routstr.core.db import ApiKey, LightningInvoice
|
||||
from routstr.lightning import _create_api_key_record
|
||||
|
||||
|
||||
def _configure_quote_proof_wallet(wallet: MagicMock) -> None:
|
||||
wallet.proofs = []
|
||||
wallet.keysets = {}
|
||||
wallet.load_proofs = AsyncMock()
|
||||
|
||||
|
||||
def _make_invoice(**kwargs: object) -> LightningInvoice:
|
||||
base = dict(
|
||||
id="inv_test_001",
|
||||
@@ -42,7 +49,15 @@ def _make_invoice(**kwargs: object) -> LightningInvoice:
|
||||
def mock_wallet_mint() -> object:
|
||||
with patch("routstr.lightning.get_wallet") as mock_get_wallet:
|
||||
wallet = AsyncMock()
|
||||
wallet.mint = AsyncMock(return_value=[])
|
||||
wallet.proofs = []
|
||||
wallet.load_proofs = AsyncMock()
|
||||
|
||||
async def mint(amount: int, quote_id: str) -> list[Proof]:
|
||||
proofs = [Proof(amount=amount, mint_id=quote_id)]
|
||||
wallet.proofs.extend(proofs)
|
||||
return proofs
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=mint)
|
||||
mock_get_wallet.return_value = wallet
|
||||
yield mock_get_wallet
|
||||
|
||||
@@ -183,6 +198,7 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
|
||||
mint_calls = 0
|
||||
@@ -196,7 +212,9 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
|
||||
await asyncio.sleep(0.05)
|
||||
if call_number > 1:
|
||||
raise Exception("quote already issued")
|
||||
return []
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=single_use_mint)
|
||||
|
||||
@@ -228,7 +246,7 @@ async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_mint_keeps_invoice_pending_for_retry(
|
||||
async def test_failed_mint_marks_invoice_for_settlement_retry(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
@@ -238,6 +256,7 @@ async def test_failed_mint_keeps_invoice_pending_for_retry(
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
wallet.mint = AsyncMock(side_effect=TimeoutError("mint unavailable"))
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
@@ -251,7 +270,7 @@ async def test_failed_mint_keeps_invoice_pending_for_retry(
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "pending"
|
||||
assert stored.status == "settlement_pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -349,8 +368,15 @@ async def test_post_mint_db_failure_keeps_invoice_pending_for_reconciliation(
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
wallet.mint = AsyncMock(return_value=[])
|
||||
|
||||
async def successful_mint(*args: object, **kwargs: object) -> list[Proof]:
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=successful_mint)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
|
||||
stored = await session.get(LightningInvoice, invoice.id)
|
||||
stored_sibling = await session.get(LightningInvoice, sibling.id)
|
||||
@@ -373,14 +399,14 @@ async def test_post_mint_db_failure_keeps_invoice_pending_for_reconciliation(
|
||||
assert sibling_state is not None
|
||||
assert stored_state.expired is False
|
||||
assert sibling_state.expired is False
|
||||
assert stored.status == "pending"
|
||||
assert stored.status == "settlement_pending"
|
||||
assert stored_sibling.id == sibling.id
|
||||
|
||||
assert wallet.mint.await_count == 1
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "pending"
|
||||
assert stored.status == "settlement_pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -428,11 +454,14 @@ async def test_db_guard_credits_once_when_both_mints_succeed(
|
||||
await setup.commit()
|
||||
|
||||
wallet = MagicMock()
|
||||
_configure_quote_proof_wallet(wallet)
|
||||
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
|
||||
|
||||
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[object]:
|
||||
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[Proof]:
|
||||
await asyncio.sleep(0.05)
|
||||
return []
|
||||
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
|
||||
wallet.proofs.append(proof)
|
||||
return [proof]
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=always_succeeding_mint)
|
||||
|
||||
@@ -472,7 +501,7 @@ async def test_db_guard_credits_once_when_both_mints_succeed(
|
||||
assert first_invoice not in first.dirty
|
||||
assert second_invoice not in second.dirty
|
||||
|
||||
assert wallet.mint.await_count == 2
|
||||
assert wallet.mint.await_count == 1
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_invoice = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored_invoice is not None
|
||||
|
||||
@@ -26,11 +26,17 @@ async def patch_invoice_generation() -> Any:
|
||||
"""Stub out `generate_lightning_invoice` so no mint round-trip is needed."""
|
||||
counter = {"n": 0}
|
||||
|
||||
async def fake_generate(amount_sats: int, description: str) -> tuple[str, str]:
|
||||
async def fake_generate(
|
||||
amount_sats: int,
|
||||
description: str,
|
||||
*,
|
||||
allowed_mints: list[str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
counter["n"] += 1
|
||||
return (
|
||||
f"lnbc{amount_sats}n1pfakeinvoice{counter['n']}",
|
||||
f"payment_hash_{counter['n']}",
|
||||
"http://localhost:3338",
|
||||
)
|
||||
|
||||
with patch(
|
||||
@@ -95,6 +101,8 @@ async def test_topup_with_authorization_header(
|
||||
body = resp.json()
|
||||
assert body["amount_sats"] == 500
|
||||
assert body["bolt11"].startswith("lnbc")
|
||||
allowed_mints = patch_invoice_generation.call_args.kwargs["allowed_mints"]
|
||||
assert allowed_mints == ["http://localhost:3338"]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import Proof
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlmodel import col, update
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey, LightningInvoice
|
||||
from routstr.lightning import (
|
||||
_expire_invoice_if_authoritatively_unpaid,
|
||||
_finalize_invoice_settlement,
|
||||
_InvoiceSettlement,
|
||||
check_invoice_payment,
|
||||
)
|
||||
|
||||
|
||||
def _lightning_invoice(**overrides: object) -> LightningInvoice:
|
||||
suffix = uuid.uuid4().hex
|
||||
values = {
|
||||
"id": f"invoice-{suffix}",
|
||||
"bolt11": f"lnbc-{suffix}",
|
||||
"amount_sats": 100,
|
||||
"description": "settlement test",
|
||||
"payment_hash": f"quote-{suffix}",
|
||||
"status": "pending",
|
||||
"purpose": "create",
|
||||
"mint_url": "http://mint:3338",
|
||||
"expires_at": int(time.time()) + 3600,
|
||||
}
|
||||
values.update(overrides)
|
||||
return LightningInvoice(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_read_transaction_closes_before_external_mint_io(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
invoice = _lightning_invoice()
|
||||
integration_session.add(invoice)
|
||||
await integration_session.commit()
|
||||
stored = await integration_session.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=False)))
|
||||
|
||||
async def get_wallet_without_open_db_transaction(
|
||||
*args: object, **kwargs: object
|
||||
) -> Mock:
|
||||
assert not integration_session.in_transaction()
|
||||
return wallet
|
||||
|
||||
with patch(
|
||||
"routstr.lightning.get_wallet", side_effect=get_wallet_without_open_db_transaction
|
||||
):
|
||||
await check_invoice_payment(stored, integration_session)
|
||||
|
||||
assert not integration_session.in_transaction()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_separate_sessions_cas_topup_credit_exactly_once(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup",
|
||||
api_key_hash=key_hash,
|
||||
amount_sats=100,
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
snapshot_a = _InvoiceSettlement.from_invoice(invoice)
|
||||
snapshot_b = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with (
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as session_a,
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as session_b,
|
||||
):
|
||||
results = await asyncio.gather(
|
||||
_finalize_invoice_settlement(snapshot_a, session_a, 1_700_000_000),
|
||||
_finalize_invoice_settlement(snapshot_b, session_b, 1_700_000_001),
|
||||
)
|
||||
|
||||
assert sorted(settled for settled, _ in results) == [False, True]
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_invoice = await verify.get(LightningInvoice, invoice.id)
|
||||
stored_key = await verify.get(ApiKey, key_hash)
|
||||
assert stored_invoice is not None
|
||||
assert stored_invoice.status == "paid"
|
||||
assert stored_key is not None
|
||||
assert stored_key.balance == 200_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_atomic_increment_preserves_concurrent_balance_mutation(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup", api_key_hash=key_hash, amount_sats=100
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
async def debit_balance(session: AsyncSession) -> None:
|
||||
result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key_hash)
|
||||
.values(balance=col(ApiKey.balance) - 10_000)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
assert result.rowcount == 1
|
||||
await session.commit()
|
||||
|
||||
snapshot = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with (
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as settlement,
|
||||
AsyncSession(integration_engine, expire_on_commit=False) as debit,
|
||||
):
|
||||
settlement_result, _ = await asyncio.gather(
|
||||
_finalize_invoice_settlement(snapshot, settlement, 1_700_000_000),
|
||||
debit_balance(debit),
|
||||
)
|
||||
|
||||
assert settlement_result[0]
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored_key = await verify.get(ApiKey, key_hash)
|
||||
assert stored_key is not None
|
||||
assert stored_key.balance == 190_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_final_commit_rolls_back_claim_and_credit_for_retry(
|
||||
integration_engine: AsyncEngine,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup",
|
||||
api_key_hash=key_hash,
|
||||
amount_sats=100,
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
snapshot = _InvoiceSettlement.from_invoice(invoice)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
|
||||
with patch.object(
|
||||
failed, "commit", AsyncMock(side_effect=Exception("db unavailable"))
|
||||
):
|
||||
with pytest.raises(Exception, match="db unavailable"):
|
||||
await _finalize_invoice_settlement(snapshot, failed, 1_700_000_000)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
pending = await verify.get(LightningInvoice, invoice.id)
|
||||
unchanged = await verify.get(ApiKey, key_hash)
|
||||
assert pending is not None
|
||||
assert pending.status == "pending"
|
||||
assert unchanged is not None
|
||||
assert unchanged.balance == 100_000
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
|
||||
settled, _ = await _finalize_invoice_settlement(
|
||||
snapshot, retry, 1_700_000_001
|
||||
)
|
||||
assert settled
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
paid = await verify.get(LightningInvoice, invoice.id)
|
||||
credited = await verify.get(ApiKey, key_hash)
|
||||
assert paid is not None
|
||||
assert paid.status == "paid"
|
||||
assert credited is not None
|
||||
assert credited.balance == 200_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_invoice_payment_retries_after_mint_success_and_db_failure(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
key_hash = uuid.uuid4().hex
|
||||
invoice = _lightning_invoice(
|
||||
purpose="topup", api_key_hash=key_hash, amount_sats=100
|
||||
)
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=100_000,
|
||||
refund_currency="sat",
|
||||
refund_mint_url="http://mint:3338",
|
||||
)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(key)
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
wallet = Mock(
|
||||
proofs=[],
|
||||
keysets={"keyset-1": Mock()},
|
||||
load_proofs=AsyncMock(),
|
||||
get_mint_quote=AsyncMock(return_value=Mock(paid=True)),
|
||||
restore_tokens_for_keyset=AsyncMock(),
|
||||
)
|
||||
|
||||
async def mint(amount: int, quote_id: str) -> list[Proof]:
|
||||
proofs = [Proof(amount=amount, mint_id=quote_id)]
|
||||
wallet.proofs.extend(proofs)
|
||||
return proofs
|
||||
|
||||
wallet.mint = AsyncMock(side_effect=mint)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
|
||||
stored = await failed.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch(
|
||||
"routstr.lightning._finalize_invoice_settlement",
|
||||
AsyncMock(side_effect=Exception("db unavailable")),
|
||||
),
|
||||
):
|
||||
await check_invoice_payment(stored, failed)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
pending = await verify.get(LightningInvoice, invoice.id)
|
||||
unchanged = await verify.get(ApiKey, key_hash)
|
||||
assert pending is not None
|
||||
assert pending.status == "settlement_pending"
|
||||
assert unchanged is not None
|
||||
assert unchanged.balance == 100_000
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
|
||||
stored = await retry.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
await check_invoice_payment(stored, retry)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
paid = await verify.get(LightningInvoice, invoice.id)
|
||||
credited = await verify.get(ApiKey, key_hash)
|
||||
assert paid is not None
|
||||
assert paid.status == "paid"
|
||||
assert credited is not None
|
||||
assert credited.balance == 200_000
|
||||
|
||||
wallet.mint.assert_awaited_once_with(100, quote_id=invoice.payment_hash)
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expiry_cas_cannot_overwrite_concurrent_paid_invoice(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _lightning_invoice(expires_at=0)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as caller:
|
||||
stale = await caller.get(LightningInvoice, invoice.id)
|
||||
assert stale is not None
|
||||
await caller.commit()
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as paid:
|
||||
result = await paid.exec( # type: ignore[call-overload]
|
||||
update(LightningInvoice)
|
||||
.where(col(LightningInvoice.id) == invoice.id)
|
||||
.values(status="paid", paid_at=123)
|
||||
)
|
||||
assert result.rowcount == 1
|
||||
await paid.commit()
|
||||
|
||||
expired = await _expire_invoice_if_authoritatively_unpaid(
|
||||
stale, caller, True
|
||||
)
|
||||
|
||||
assert expired is False
|
||||
assert stale.status == "paid"
|
||||
assert stale.paid_at == 123
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "paid"
|
||||
assert stored.paid_at == 123
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_quote_worker_does_not_mint_after_expiry_claim_wins(
|
||||
integration_engine: AsyncEngine,
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
invoice = _lightning_invoice(expires_at=0)
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
|
||||
seed.add(invoice)
|
||||
await seed.commit()
|
||||
|
||||
quote_started = asyncio.Event()
|
||||
release_quote = asyncio.Event()
|
||||
|
||||
async def paid_quote_after_expiry(*_args: object, **_kwargs: object) -> Mock:
|
||||
quote_started.set()
|
||||
await release_quote.wait()
|
||||
return Mock(paid=True)
|
||||
|
||||
wallet = Mock(
|
||||
get_mint_quote=AsyncMock(side_effect=paid_quote_after_expiry),
|
||||
mint=AsyncMock(),
|
||||
)
|
||||
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as worker:
|
||||
observed_pending = await worker.get(LightningInvoice, invoice.id)
|
||||
assert observed_pending is not None
|
||||
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
|
||||
settlement_task = asyncio.create_task(
|
||||
check_invoice_payment(observed_pending, worker)
|
||||
)
|
||||
await quote_started.wait()
|
||||
|
||||
async with AsyncSession(
|
||||
integration_engine, expire_on_commit=False
|
||||
) as expirer:
|
||||
expiry_view = await expirer.get(LightningInvoice, invoice.id)
|
||||
assert expiry_view is not None
|
||||
await expirer.commit()
|
||||
assert await _expire_invoice_if_authoritatively_unpaid(
|
||||
expiry_view, expirer, True
|
||||
)
|
||||
|
||||
release_quote.set()
|
||||
assert await settlement_task is False
|
||||
|
||||
wallet.mint.assert_not_awaited()
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
|
||||
stored = await verify.get(LightningInvoice, invoice.id)
|
||||
assert stored is not None
|
||||
assert stored.status == "expired"
|
||||
@@ -110,7 +110,11 @@ async def test_payout_does_not_send_proofs_whose_liability_commit_is_in_flight(
|
||||
finish_redemption = asyncio.Event()
|
||||
liability_read = asyncio.Event()
|
||||
|
||||
async def redeem_token(token: str) -> tuple[int, str, str]:
|
||||
async def redeem_token(
|
||||
token: str,
|
||||
destination_mint: str | None = None,
|
||||
destination_unit: str | None = None,
|
||||
) -> tuple[int, str, str]:
|
||||
proofs.append(MagicMock(amount=200))
|
||||
proof_visible.set()
|
||||
await finish_redemption.wait()
|
||||
|
||||
@@ -126,8 +126,11 @@ async def test_parent_and_child_keys_are_not_pruned(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
|
||||
"""A key referenced by a pending topup invoice is never pruned mid-topup."""
|
||||
@pytest.mark.parametrize("status", ["pending", "settlement_pending"])
|
||||
async def test_retryable_invoice_protects_key(
|
||||
patched_db_engine: None, status: str
|
||||
) -> None:
|
||||
"""A key referenced by a retryable topup invoice is never pruned mid-topup."""
|
||||
key = _dead_key(LONG_AGO)
|
||||
invoice = LightningInvoice(
|
||||
id=f"inv_{uuid.uuid4().hex}",
|
||||
@@ -135,7 +138,7 @@ async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
|
||||
amount_sats=10,
|
||||
description="topup",
|
||||
payment_hash=uuid.uuid4().hex,
|
||||
status="pending",
|
||||
status=status,
|
||||
api_key_hash=key.hashed_key,
|
||||
purpose="topup",
|
||||
expires_at=NOW + 10_000,
|
||||
|
||||
@@ -20,6 +20,7 @@ from collections.abc import Callable
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import MeltQuoteState
|
||||
from httpx import AsyncClient, Response
|
||||
|
||||
from routstr.core.settings import settings
|
||||
@@ -28,7 +29,9 @@ from routstr.core.settings import settings
|
||||
# with the testmint stub that bypasses swapping (see conftest.py).
|
||||
from routstr.wallet import recieve_token as _real_recieve_token
|
||||
|
||||
PRIMARY_MINT = "http://primary:3338"
|
||||
# Match the authenticated fixture's persisted refund mint: existing-key topups
|
||||
# are intentionally constrained to that mint for collateral provenance.
|
||||
PRIMARY_MINT = "http://localhost:3338"
|
||||
|
||||
|
||||
def _make_swap_mocks(
|
||||
@@ -81,7 +84,9 @@ def _make_swap_mocks(
|
||||
quote=f"melt_quote_{invoice}", amount=invoice, fee_reserve=_next_fee()
|
||||
)
|
||||
)
|
||||
mock_token_wallet.melt = AsyncMock(return_value=Mock())
|
||||
mock_token_wallet.melt = AsyncMock(
|
||||
return_value=Mock(state=MeltQuoteState.paid)
|
||||
)
|
||||
|
||||
return mock_token, mock_token_wallet, mock_primary_wallet
|
||||
|
||||
@@ -89,7 +94,12 @@ def _make_swap_mocks(
|
||||
def _wallet_router(primary_wallet: Mock, token_wallet: Mock) -> Callable[..., Mock]:
|
||||
"""Route get_wallet calls to the primary or foreign wallet mock by URL."""
|
||||
|
||||
def fake_get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Mock:
|
||||
def fake_get_wallet(
|
||||
mint_url: str,
|
||||
unit: str = "sat",
|
||||
load: bool = True,
|
||||
**kwargs: object,
|
||||
) -> Mock:
|
||||
return primary_wallet if mint_url == PRIMARY_MINT else token_wallet
|
||||
|
||||
return fake_get_wallet
|
||||
@@ -139,7 +149,7 @@ async def test_topup_retries_when_melt_demands_more_than_quoted(
|
||||
"Mint Error: not enough inputs provided for melt. "
|
||||
"Provided: 179, needed: 180 (Code: 11000)"
|
||||
),
|
||||
Mock(),
|
||||
Mock(state=MeltQuoteState.paid),
|
||||
]
|
||||
|
||||
response = await _post_topup(
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import base64
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
import routstr.wallet as wallet_module
|
||||
from routstr.core import admin
|
||||
|
||||
|
||||
@@ -13,20 +17,12 @@ async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
|
||||
) -> None:
|
||||
primary_mint = "https://primary.example"
|
||||
effective_mint = requested_mint or primary_mint
|
||||
wallet = object()
|
||||
proofs = [SimpleNamespace(amount=40), SimpleNamespace(amount=60)]
|
||||
token = "cashuBoutgoing"
|
||||
|
||||
get_wallet = AsyncMock(return_value=wallet)
|
||||
get_proofs = Mock(return_value=proofs)
|
||||
filter_proofs = AsyncMock(return_value=proofs)
|
||||
send_token = AsyncMock(return_value=token)
|
||||
store_transaction = AsyncMock(return_value=True)
|
||||
|
||||
monkeypatch.setattr(admin, "get_wallet", get_wallet)
|
||||
monkeypatch.setattr(admin, "get_proofs_per_mint_and_unit", get_proofs)
|
||||
monkeypatch.setattr(admin, "slow_filter_spend_proofs", filter_proofs)
|
||||
monkeypatch.setattr(admin, "send_token", send_token)
|
||||
monkeypatch.setattr(admin, "token_mint_url", Mock(return_value=effective_mint))
|
||||
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
|
||||
monkeypatch.setattr(admin.settings, "primary_mint", primary_mint)
|
||||
|
||||
@@ -35,10 +31,7 @@ async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
|
||||
admin.WithdrawRequest(amount=75, mint_url=requested_mint, unit="sat"),
|
||||
)
|
||||
|
||||
assert result == {"token": token}
|
||||
get_wallet.assert_awaited_once_with(effective_mint, "sat")
|
||||
get_proofs.assert_called_once_with(wallet, effective_mint, "sat", not_reserved=True)
|
||||
filter_proofs.assert_awaited_once_with(proofs, wallet)
|
||||
assert result == {"token": token, "mint_url": effective_mint}
|
||||
send_token.assert_awaited_once_with(75, "sat", effective_mint)
|
||||
store_transaction.assert_awaited_once_with(
|
||||
token=token,
|
||||
@@ -56,17 +49,10 @@ async def test_withdraw_returns_issued_token_when_audit_storage_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mint = "https://primary.example"
|
||||
proofs = [SimpleNamespace(amount=100)]
|
||||
token = "cashuBrecoverable"
|
||||
|
||||
monkeypatch.setattr(admin, "get_wallet", AsyncMock(return_value=object()))
|
||||
monkeypatch.setattr(
|
||||
admin, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
admin, "slow_filter_spend_proofs", AsyncMock(return_value=proofs)
|
||||
)
|
||||
monkeypatch.setattr(admin, "send_token", AsyncMock(return_value=token))
|
||||
monkeypatch.setattr(admin, "token_mint_url", Mock(return_value=mint))
|
||||
monkeypatch.setattr(
|
||||
admin,
|
||||
"store_cashu_transaction",
|
||||
@@ -78,5 +64,89 @@ async def test_withdraw_returns_issued_token_when_audit_storage_fails(
|
||||
|
||||
result = await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
|
||||
|
||||
assert result == {"token": token}
|
||||
assert result == {"token": token, "mint_url": mint}
|
||||
critical.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_withdraw_falls_back_from_insufficient_preferred_mint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
requested_mint = "https://primary.example"
|
||||
actual_mint = "https://secondary.example"
|
||||
proofs = [SimpleNamespace(amount=100, reserved=False, id="00")]
|
||||
token_payload = {
|
||||
"token": [
|
||||
{
|
||||
"mint": actual_mint,
|
||||
"proofs": [
|
||||
{
|
||||
"id": "00",
|
||||
"amount": 75,
|
||||
"secret": "secret",
|
||||
"C": "02" + "00" * 32,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"unit": "sat",
|
||||
}
|
||||
token = "cashuA" + base64.urlsafe_b64encode(
|
||||
json.dumps(token_payload).encode()
|
||||
).decode()
|
||||
wallet = SimpleNamespace(
|
||||
keysets={},
|
||||
proofs=proofs,
|
||||
select_to_send=AsyncMock(return_value=(proofs, 0)),
|
||||
serialize_proofs=AsyncMock(return_value=token),
|
||||
set_reserved_for_send=AsyncMock(),
|
||||
)
|
||||
find_funded = AsyncMock(return_value=actual_mint)
|
||||
store_transaction = AsyncMock(return_value=True)
|
||||
|
||||
monkeypatch.setattr(wallet_module, "find_trusted_mint_with_funds", find_funded)
|
||||
monkeypatch.setattr(wallet_module, "get_wallet", AsyncMock(return_value=wallet))
|
||||
monkeypatch.setattr(
|
||||
wallet_module, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
|
||||
)
|
||||
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
|
||||
|
||||
result = await admin.withdraw(
|
||||
Mock(), admin.WithdrawRequest(amount=75, mint_url=requested_mint)
|
||||
)
|
||||
|
||||
assert result == {"token": token, "mint_url": actual_mint}
|
||||
find_funded.assert_awaited_once_with(
|
||||
75, "sat", requested_mint, force_reload=True
|
||||
)
|
||||
wallet.select_to_send.assert_awaited_once()
|
||||
store_transaction.assert_awaited_once_with(
|
||||
token=token,
|
||||
amount=75,
|
||||
unit="sat",
|
||||
mint_url=actual_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="admin",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_withdraw_maps_true_aggregate_insufficient_funds_to_400(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
admin,
|
||||
"send_token",
|
||||
AsyncMock(
|
||||
side_effect=ValueError(
|
||||
"No trusted mint has 75 sat available; balances={'mint': 0}"
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.detail == "Insufficient wallet balance"
|
||||
|
||||
@@ -270,6 +270,54 @@ async def test_internal_error_with_invalid_keyword_does_not_masquerade(
|
||||
assert await session.get(ApiKey, hashed_key) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_primary_msat_token_sets_provenance_without_cashu_mint_duplicate(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
token = "cashuAprimary_msat_token"
|
||||
token_obj = SimpleNamespace(mint="http://primary:3338", unit="msat")
|
||||
credit = AsyncMock(return_value=1_000)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", token_obj.mint),
|
||||
patch.object(settings, "primary_mint_unit", "msat"),
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch("routstr.auth.deserialize_token_from_string", return_value=token_obj),
|
||||
patch("routstr.auth.credit_balance", new=credit),
|
||||
):
|
||||
key = await validate_bearer_key(token, session)
|
||||
|
||||
assert key.refund_mint_url == token_obj.mint
|
||||
assert key.refund_currency == "msat"
|
||||
credit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_primary_token_unit_mismatch_is_rejected_before_redemption(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
token = "cashuAprimary_wrong_unit"
|
||||
token_obj = SimpleNamespace(mint="http://primary:3338", unit="sat")
|
||||
credit = AsyncMock(return_value=1_000)
|
||||
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", token_obj.mint),
|
||||
patch.object(settings, "primary_mint_unit", "msat"),
|
||||
patch.object(settings, "cashu_mints", []),
|
||||
patch("routstr.auth.deserialize_token_from_string", return_value=token_obj),
|
||||
patch("routstr.auth.credit_balance", new=credit),
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await validate_bearer_key(token, session)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
credit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_cashu_token_returns_400_invalid_token(
|
||||
session: AsyncSession,
|
||||
|
||||
@@ -66,6 +66,10 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(return_value=True),
|
||||
) as store,
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.token_mint_url",
|
||||
return_value="https://fallback-mint.test",
|
||||
),
|
||||
patch("routstr.upstream.auto_topup.create_session", return_value=session),
|
||||
):
|
||||
await _check_and_topup(_row())
|
||||
@@ -74,7 +78,7 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
|
||||
token="cashu-token",
|
||||
amount=50,
|
||||
unit="sat",
|
||||
mint_url="https://mint.test",
|
||||
mint_url="https://fallback-mint.test",
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="auto_topup",
|
||||
@@ -138,6 +142,12 @@ async def test_auto_topup_does_not_send_untracked_token() -> None:
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(side_effect=RuntimeError("database unavailable")),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.release_token_reservation",
|
||||
AsyncMock(),
|
||||
) as reclaim,
|
||||
):
|
||||
await _check_and_topup(_row())
|
||||
|
||||
reclaim.assert_awaited_once_with("cashu-token")
|
||||
provider.topup.assert_not_awaited()
|
||||
|
||||
@@ -606,6 +606,29 @@ async def test_topup_mint_unreachable_returns_503(error: Exception) -> None:
|
||||
assert exc_info.value.detail == "Cashu mint is unreachable"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_unreachable_source_mint_explains_why_fallback_is_impossible() -> None:
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.wallet import SourceMintConnectionError
|
||||
|
||||
key = _make_api_key(balance=1000)
|
||||
session = MagicMock()
|
||||
error = SourceMintConnectionError("Issuing Cashu mint is unreachable")
|
||||
|
||||
with (
|
||||
patch("routstr.balance.get_billing_key", AsyncMock(return_value=key)),
|
||||
patch("routstr.balance.credit_balance", AsyncMock(side_effect=error)),
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await topup_wallet_endpoint(
|
||||
cashu_token="cashuAtoken", key=key, session=session
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 503
|
||||
assert "cannot be redeemed at another mint" in exc_info.value.detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topup_already_spent_still_returns_400() -> None:
|
||||
"""Regression: the mint-unreachable short-circuit must not swallow the
|
||||
@@ -758,3 +781,68 @@ async def test_topup_unexpected_non_valueerror_returns_500() -> None:
|
||||
|
||||
assert exc_info.value.status_code == 500
|
||||
assert exc_info.value.detail == "Internal server error"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apikey_refund_ambiguous_melt_does_not_restore_balance() -> None:
|
||||
"""An ambiguous LNURL melt may still settle: the debit must be kept."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.payment.lnurl import MeltOutcomeAmbiguousError
|
||||
|
||||
key = _make_api_key(balance=5000, refund_address="user@ln.example.com")
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.exec = AsyncMock(return_value=MagicMock(rowcount=1))
|
||||
session.commit = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
|
||||
patch("routstr.balance._refund_cache_set", AsyncMock()),
|
||||
patch(
|
||||
"routstr.balance.send_to_lnurl",
|
||||
AsyncMock(side_effect=MeltOutcomeAmbiguousError("outcome is ambiguous")),
|
||||
),
|
||||
patch("routstr.balance._restore_balance", AsyncMock()) as mock_restore,
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await refund_wallet_endpoint(
|
||||
authorization="Bearer sk-testhash",
|
||||
x_cashu=None,
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 502
|
||||
mock_restore.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apikey_refund_clean_failure_still_restores_balance() -> None:
|
||||
"""A definitively failed melt must keep restoring the debited balance."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
key = _make_api_key(balance=5000, refund_address="user@ln.example.com")
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.exec = AsyncMock(return_value=MagicMock(rowcount=1))
|
||||
session.commit = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
|
||||
patch("routstr.balance._refund_cache_set", AsyncMock()),
|
||||
patch(
|
||||
"routstr.balance.send_to_lnurl",
|
||||
AsyncMock(side_effect=RuntimeError("mint rejected melt")),
|
||||
),
|
||||
patch("routstr.balance._restore_balance", AsyncMock()) as mock_restore,
|
||||
):
|
||||
with pytest.raises(HTTPException):
|
||||
await refund_wallet_endpoint(
|
||||
authorization="Bearer sk-testhash",
|
||||
x_cashu=None,
|
||||
session=session,
|
||||
)
|
||||
|
||||
mock_restore.assert_awaited_once()
|
||||
|
||||
@@ -4,7 +4,7 @@ Tests admin endpoints that are testable without full app setup:
|
||||
withdraw validation, authentication guards, and slug validation.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException, Request
|
||||
@@ -46,22 +46,19 @@ async def test_withdraw_rejects_insufficient_balance() -> None:
|
||||
|
||||
request = Request(scope={"type": "http", "method": "POST"})
|
||||
|
||||
with patch("routstr.core.admin.get_wallet") as mock_wallet, \
|
||||
patch("routstr.core.admin.get_proofs_per_mint_and_unit") as mock_proofs, \
|
||||
patch("routstr.core.admin.slow_filter_spend_proofs") as mock_filter:
|
||||
|
||||
mock_w = Mock()
|
||||
mock_w.keysets = {}
|
||||
mock_w.proofs = []
|
||||
mock_wallet.return_value = mock_w
|
||||
mock_proofs.return_value = []
|
||||
mock_filter.return_value = []
|
||||
|
||||
with patch(
|
||||
"routstr.core.admin.send_token",
|
||||
new=AsyncMock(
|
||||
side_effect=ValueError(
|
||||
"No trusted mint has 1000000 sat available; balances={}"
|
||||
)
|
||||
),
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await withdraw(request, WithdrawRequest(amount=1000000, unit="sat"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "Insufficient" in str(exc_info.value.detail)
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "Insufficient" in str(exc_info.value.detail)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
|
||||
@@ -67,7 +67,7 @@ async def test_fee_payout_prepares_wallet_then_checkpoints_before_sending() -> N
|
||||
payout_wallet = Mock()
|
||||
events: list[str] = []
|
||||
|
||||
async def prepare(*_args: object) -> Mock:
|
||||
async def prepare(*_args: object, **_kwargs: object) -> Mock:
|
||||
events.append("prepare")
|
||||
return payout_wallet
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import AsyncGenerator, Generator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlmodel import SQLModel
|
||||
@@ -12,6 +13,21 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
from routstr.wallet import fetch_all_balances
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_balance_fetch_state() -> Generator[None, None, None]:
|
||||
from routstr import wallet
|
||||
|
||||
wallet._balance_fetch_failures.clear()
|
||||
wallet._balance_fetch_locks.clear()
|
||||
wallet._mint_supported_units.clear()
|
||||
wallet._MintRateGuard._guards.clear()
|
||||
yield
|
||||
wallet._balance_fetch_failures.clear()
|
||||
wallet._balance_fetch_locks.clear()
|
||||
wallet._mint_supported_units.clear()
|
||||
wallet._MintRateGuard._guards.clear()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _fake_session(): # type: ignore[no-untyped-def]
|
||||
yield MagicMock()
|
||||
@@ -29,7 +45,7 @@ def _patches( # type: ignore[no-untyped-def]
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
AsyncMock(side_effect=lambda proofs, wallet, **kwargs: proofs),
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.balances_by_mint_and_unit",
|
||||
@@ -63,6 +79,161 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
|
||||
assert total_wallet == 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_uses_units_advertised_by_mint() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch(
|
||||
"routstr.wallet._get_supported_mint_units",
|
||||
AsyncMock(return_value=["sat"]),
|
||||
) as supported_units,
|
||||
):
|
||||
for p in _patches(proof_amount=1000):
|
||||
p.start()
|
||||
try:
|
||||
details, *_ = await fetch_all_balances()
|
||||
finally:
|
||||
patch.stopall()
|
||||
|
||||
supported_units.assert_awaited_once_with("http://mint:3338")
|
||||
assert [detail["unit"] for detail in details] == ["sat"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unit_discovery_failure_returns_structured_balance_error() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
get_wallet = AsyncMock()
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch(
|
||||
"routstr.wallet._get_supported_mint_units",
|
||||
AsyncMock(side_effect=httpx.ConnectError("mint unavailable")),
|
||||
),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
):
|
||||
details, *_ = await fetch_all_balances()
|
||||
|
||||
assert details[0]["unit"] == settings.primary_mint_unit
|
||||
assert details[0]["error_code"] == "unreachable"
|
||||
assert details[0]["retry_after_seconds"] > 0
|
||||
get_wallet.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_supported_mint_units_come_from_active_keysets() -> None:
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _get_supported_mint_units
|
||||
|
||||
# Cashu versions/mints may deserialize keyset units as either strings or
|
||||
# Unit enum-like objects. Both representations must be accepted.
|
||||
sat = MagicMock(active=True, unit="sat")
|
||||
msat = MagicMock(active=False, unit="msat")
|
||||
usd = MagicMock(active=True)
|
||||
usd.unit.name = "usd"
|
||||
wallet = MagicMock()
|
||||
wallet._get_keysets = AsyncMock(return_value=[usd, msat, sat])
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint_unit", "sat"),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)),
|
||||
):
|
||||
units = await _get_supported_mint_units("http://mint:3338")
|
||||
cached_units = await _get_supported_mint_units("http://mint:3338")
|
||||
|
||||
assert units == ["sat", "usd"]
|
||||
assert cached_units == units
|
||||
wallet._get_keysets.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_backs_off_after_connection_failure() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.mint.time.monotonic", return_value=10),
|
||||
patch("routstr.wallet.logger.warning") as warning,
|
||||
):
|
||||
first = await fetch_all_balances(units=["sat"])
|
||||
second = await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert first[0][0]["error"] == "mint unavailable"
|
||||
assert first[0][0]["error_code"] == "unreachable"
|
||||
assert first[0][0]["retry_after_seconds"] == 60
|
||||
assert second[0][0]["error"] == "mint unavailable"
|
||||
assert second[0][0]["error_code"] == "unreachable"
|
||||
assert get_wallet.await_count == 1
|
||||
warning.assert_called_once()
|
||||
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.mint.time.monotonic", return_value=71),
|
||||
patch("routstr.wallet.logger.warning"),
|
||||
):
|
||||
await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert get_wallet.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_reports_rate_limit_status() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
request = httpx.Request("GET", "http://mint:3338/v1/keysets")
|
||||
response = httpx.Response(429, request=request, headers={"Retry-After": "45"})
|
||||
error = httpx.HTTPStatusError("rate limited", request=request, response=response)
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
|
||||
patch.object(settings, "primary_mint", "http://mint:3338"),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(side_effect=error)),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat"])
|
||||
|
||||
assert details[0]["error_code"] == "rate_limited"
|
||||
assert details[0]["retry_after_seconds"] == 60
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_balance_failure_applies_mint_cooldown_to_other_units() -> None:
|
||||
from routstr.core.settings import settings
|
||||
from routstr.wallet import _mint_cooldown_remaining
|
||||
|
||||
mint = "http://mint:3338"
|
||||
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
|
||||
with (
|
||||
patch.object(settings, "cashu_mints", [mint]),
|
||||
patch.object(settings, "primary_mint", mint),
|
||||
patch("routstr.wallet.get_wallet", get_wallet),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
patch("routstr.mint.time.monotonic", return_value=10),
|
||||
patch("routstr.wallet.logger.warning") as warning,
|
||||
):
|
||||
details, *_ = await fetch_all_balances(units=["sat", "msat"])
|
||||
cooldown = _mint_cooldown_remaining(mint)
|
||||
|
||||
assert get_wallet.await_count == 1
|
||||
assert warning.call_count == 1
|
||||
assert cooldown == 60
|
||||
assert details[0]["error"] == "mint unavailable"
|
||||
assert details[0]["error_code"] == "unreachable"
|
||||
assert details[1]["error"] == "Mint is unreachable"
|
||||
assert details[1]["error_code"] == "unreachable"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_closes_db_session_before_concurrent_mint_io() -> None:
|
||||
"""Slow mint checks must never run while the balance DB session is open."""
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from cashu.core.base import MintQuoteState, Proof
|
||||
|
||||
from routstr.lightning import (
|
||||
InvoiceRecoverRequest,
|
||||
_invoice_settlement_locks,
|
||||
_is_outputs_already_signed,
|
||||
_mint_invoice_quote,
|
||||
check_invoice_payment,
|
||||
get_invoice_status,
|
||||
recover_invoice,
|
||||
)
|
||||
from routstr.wallet import Wallet
|
||||
|
||||
|
||||
def _invoice(**overrides: object) -> SimpleNamespace:
|
||||
values = {
|
||||
"id": "invoice-1",
|
||||
"payment_hash": "quote-1",
|
||||
"amount_sats": 100,
|
||||
"purpose": "create",
|
||||
"status": "pending",
|
||||
"paid_at": None,
|
||||
"api_key_hash": None,
|
||||
"mint_url": "http://mint:3338",
|
||||
"balance_limit": None,
|
||||
"balance_limit_reset": None,
|
||||
"validity_date": None,
|
||||
"created_at": 1,
|
||||
"expires_at": 2,
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def _proof(amount: int, mint_id: str, *, reserved: bool = False) -> Proof:
|
||||
return Proof(amount=amount, mint_id=mint_id, reserved=reserved)
|
||||
|
||||
|
||||
def _recovery_wallet(
|
||||
error: Exception,
|
||||
*,
|
||||
proofs_before: list[Proof] | None = None,
|
||||
proofs_after: list[Proof] | None = None,
|
||||
) -> Mock:
|
||||
async def load_proofs(*, reload: bool) -> None:
|
||||
if wallet.load_proofs.await_count >= 2 and proofs_after is not None:
|
||||
wallet.proofs = list(proofs_after)
|
||||
|
||||
wallet = Mock(
|
||||
mint=AsyncMock(side_effect=error),
|
||||
keysets={"keyset-1": Mock()},
|
||||
restore_tokens_for_keyset=AsyncMock(),
|
||||
load_proofs=AsyncMock(side_effect=load_proofs),
|
||||
proofs=list(proofs_before or []),
|
||||
)
|
||||
return wallet
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_recovers_quote_linked_outputs_already_signed() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs have already been signed before (Code: 11003)"),
|
||||
proofs_after=[_proof(100, "quote-1")],
|
||||
)
|
||||
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
wallet.restore_tokens_for_keyset.assert_awaited_once_with(
|
||||
"keyset-1", to=1, batch=25
|
||||
)
|
||||
assert wallet.load_proofs.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_accepts_preloaded_quote_linked_proofs() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("must not mint"),
|
||||
proofs_before=[_proof(64, "quote-1"), _proof(36, "quote-1")],
|
||||
)
|
||||
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
wallet.mint.assert_not_awaited()
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_does_not_accept_unrelated_11003_text() -> None:
|
||||
invoice = _invoice()
|
||||
error = Exception("backend request 11003 failed")
|
||||
wallet = _recovery_wallet(error)
|
||||
|
||||
with pytest.raises(Exception) as caught:
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
assert caught.value is error
|
||||
wallet.restore_tokens_for_keyset.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_installed_cashu_error_shape_recognizes_realistic_11003_phrase() -> None:
|
||||
request = httpx.Request("POST", "http://mint:3338/v1/mint/bolt11")
|
||||
response = httpx.Response(
|
||||
400,
|
||||
request=request,
|
||||
json={"detail": "outputs have already been signed before", "code": 11003},
|
||||
)
|
||||
|
||||
with pytest.raises(Exception) as caught:
|
||||
Wallet.raise_on_error_request(response)
|
||||
|
||||
assert _is_outputs_already_signed(caught.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("recovered", [0, 99])
|
||||
async def test_invoice_mint_rejects_empty_or_short_quote_recovery(
|
||||
recovered: int,
|
||||
) -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs already signed (Code: 11003)"),
|
||||
proofs_after=[_proof(recovered, "quote-1")] if recovered else [],
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="expected at least 100"):
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoice_mint_rejects_unrelated_concurrent_balance_growth() -> None:
|
||||
invoice = _invoice()
|
||||
wallet = _recovery_wallet(
|
||||
Exception("Mint Error: outputs already signed (Code: 11003)"),
|
||||
proofs_after=[_proof(10_000, "different-quote")],
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="quote-linked recovery returned 0"):
|
||||
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_pending_invoice_is_not_minted() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(status="expired")
|
||||
session = AsyncMock()
|
||||
|
||||
with patch("routstr.lightning.get_wallet", AsyncMock()) as get_wallet:
|
||||
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
get_wallet.assert_not_awaited()
|
||||
session.commit.assert_awaited_once()
|
||||
assert _invoice_settlement_locks == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ambiguous_invoice_mint_timeout_remains_recoverable() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice()
|
||||
session = AsyncMock()
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
|
||||
state_session = AsyncMock()
|
||||
state_session.exec.return_value.rowcount = 1
|
||||
|
||||
@asynccontextmanager
|
||||
async def owned_session() -> AsyncIterator[AsyncMock]:
|
||||
yield state_session
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning.create_session", owned_session),
|
||||
patch(
|
||||
"routstr.lightning._mint_invoice_quote",
|
||||
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
|
||||
),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert invoice.status == "settlement_pending"
|
||||
state_session.commit.assert_awaited_once()
|
||||
session.rollback.assert_not_awaited()
|
||||
# One commit closes the initial read transaction before external I/O.
|
||||
session.commit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quote_lookup_timeout_is_not_definitively_unpaid() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(expires_at=0)
|
||||
session = AsyncMock()
|
||||
wallet = Mock(
|
||||
get_mint_quote=AsyncMock(side_effect=httpx.TimeoutException("quote timeout"))
|
||||
)
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overdue_invoice_does_not_expire_after_ambiguous_quote_lookup() -> None:
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
session.get.return_value = invoice
|
||||
check = AsyncMock(return_value=False)
|
||||
|
||||
with patch("routstr.lightning.check_invoice_payment", check):
|
||||
response = await get_invoice_status(invoice.id, session) # type: ignore[arg-type]
|
||||
|
||||
assert response.status == "pending"
|
||||
session.commit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overdue_invoice_expires_only_after_definitive_unpaid_quote() -> None:
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
session.get.return_value = invoice
|
||||
check = AsyncMock(return_value=True)
|
||||
|
||||
async def expire(
|
||||
candidate: SimpleNamespace, _session: AsyncMock, definitive: bool
|
||||
) -> bool:
|
||||
assert definitive is True
|
||||
candidate.status = "expired"
|
||||
return True
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.check_invoice_payment", check),
|
||||
patch(
|
||||
"routstr.lightning._expire_invoice_if_authoritatively_unpaid",
|
||||
side_effect=expire,
|
||||
) as expire_invoice,
|
||||
):
|
||||
response = await get_invoice_status(invoice.id, session) # type: ignore[arg-type]
|
||||
|
||||
assert response.status == "expired"
|
||||
expire_invoice.assert_awaited_once_with(invoice, session, True)
|
||||
session.commit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_applies_authoritative_expiry_helper() -> None:
|
||||
invoice = _invoice(status="pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
result = Mock()
|
||||
result.first.return_value = invoice
|
||||
session.exec.return_value = result
|
||||
check = AsyncMock(return_value=True)
|
||||
|
||||
async def expire(
|
||||
candidate: SimpleNamespace, _session: AsyncMock, definitive: bool
|
||||
) -> bool:
|
||||
assert definitive is True
|
||||
candidate.status = "expired"
|
||||
return True
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.check_invoice_payment", check),
|
||||
patch(
|
||||
"routstr.lightning._expire_invoice_if_authoritatively_unpaid",
|
||||
side_effect=expire,
|
||||
) as expire_invoice,
|
||||
):
|
||||
response = await recover_invoice(
|
||||
InvoiceRecoverRequest(bolt11="lnbc-test"), session # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert response.status == "expired"
|
||||
expire_invoice.assert_awaited_once_with(invoice, session, True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_state_write_failure_still_reports_non_expirable_outcome() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice(expires_at=0)
|
||||
session = AsyncMock()
|
||||
wallet = Mock(
|
||||
get_mint_quote=AsyncMock(
|
||||
return_value=Mock(paid=True, state=MintQuoteState.paid)
|
||||
)
|
||||
)
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch(
|
||||
"routstr.lightning._mint_invoice_quote",
|
||||
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
|
||||
),
|
||||
patch(
|
||||
"routstr.lightning.create_session",
|
||||
side_effect=RuntimeError("database unavailable"),
|
||||
),
|
||||
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
|
||||
):
|
||||
definitively_unpaid = await check_invoice_payment(
|
||||
invoice, session # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert definitively_unpaid is False
|
||||
assert invoice.status == "pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_settlement_pending_invoice_does_not_expire() -> None:
|
||||
invoice = _invoice(status="settlement_pending", expires_at=0)
|
||||
session = AsyncMock()
|
||||
session.get.return_value = invoice
|
||||
check = AsyncMock()
|
||||
|
||||
with patch("routstr.lightning.check_invoice_payment", check):
|
||||
response = await get_invoice_status(
|
||||
invoice.id, session # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
check.assert_awaited_once_with(invoice, session)
|
||||
assert response.status == "settlement_pending"
|
||||
session.commit.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_invoice_checks_finalize_once_in_process() -> None:
|
||||
_invoice_settlement_locks.clear()
|
||||
invoice = _invoice()
|
||||
session = AsyncMock()
|
||||
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
|
||||
|
||||
async def refresh(obj: SimpleNamespace) -> None:
|
||||
return None
|
||||
|
||||
session.refresh = AsyncMock(side_effect=refresh)
|
||||
|
||||
@asynccontextmanager
|
||||
async def owned_session() -> AsyncIterator[AsyncMock]:
|
||||
owned = AsyncMock()
|
||||
owned.exec.return_value.rowcount = 1
|
||||
yield owned
|
||||
|
||||
with (
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.lightning.create_session", owned_session),
|
||||
patch("routstr.lightning._mint_invoice_quote", AsyncMock()),
|
||||
patch(
|
||||
"routstr.lightning._finalize_invoice_settlement",
|
||||
AsyncMock(return_value=(True, "b" * 64)),
|
||||
) as finalize,
|
||||
):
|
||||
await asyncio.gather(
|
||||
check_invoice_payment(invoice, session), # type: ignore[arg-type]
|
||||
check_invoice_payment(invoice, session), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert invoice.status == "paid"
|
||||
finalize.assert_awaited_once()
|
||||
assert _invoice_settlement_locks == {}
|
||||
@@ -1,70 +1,202 @@
|
||||
"""raw_send_to_lnurl() must not hang forever on an unresponsive mint.
|
||||
|
||||
The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung
|
||||
mint would block the melt (and the payout loop) indefinitely. raw_send_to_lnurl
|
||||
now wraps wallet.melt() in asyncio.wait_for(MELT_TIMEOUT_SECONDS) and surfaces a
|
||||
timeout as LNURLError instead of hanging.
|
||||
"""
|
||||
"""LNURL melt attempts must not misclassify ambiguous payment outcomes."""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from cashu.core.base import MeltQuoteState
|
||||
|
||||
from routstr.payment import lnurl
|
||||
from routstr.payment.lnurl import LNURLError, raw_send_to_lnurl
|
||||
from routstr.core.settings import settings
|
||||
from routstr.mint import MintCooldownError, MintRateGuard
|
||||
from routstr.payment.lnurl import (
|
||||
MeltOutcomeAmbiguousError,
|
||||
raw_send_to_lnurl,
|
||||
)
|
||||
|
||||
LNURL_DATA = {
|
||||
"callback_url": "https://ln.tld/cb",
|
||||
"min_sendable": 1_000,
|
||||
"max_sendable": 100_000_000,
|
||||
}
|
||||
|
||||
|
||||
def _wallet() -> tuple[MagicMock, list[MagicMock]]:
|
||||
proofs = [MagicMock(amount=1000)]
|
||||
wallet = MagicMock(url="https://mint.test")
|
||||
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
|
||||
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
|
||||
return wallet, proofs
|
||||
|
||||
|
||||
def _lnurl_patches() -> tuple[Any, Any]:
|
||||
return (
|
||||
patch(
|
||||
"routstr.payment.lnurl.get_lnurl_data",
|
||||
AsyncMock(return_value=LNURL_DATA),
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.lnurl.get_lnurl_invoice",
|
||||
AsyncMock(return_value=("lnbc1...", {})),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_times_out_on_hung_melt() -> None:
|
||||
proofs = [MagicMock(amount=1000)]
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
|
||||
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
|
||||
async def test_raw_send_to_lnurl_timeout_keeps_unpaid_outcome_ambiguous() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
|
||||
async def _hang(**kwargs: object) -> None:
|
||||
await asyncio.sleep(5) # far longer than the patched timeout
|
||||
await asyncio.sleep(5)
|
||||
|
||||
wallet.melt = AsyncMock(side_effect=_hang)
|
||||
wallet.get_melt_quote = AsyncMock(
|
||||
return_value=MagicMock(state=MeltQuoteState.unpaid)
|
||||
)
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
lnurl_data = {
|
||||
"callback_url": "https://ln.tld/cb",
|
||||
"min_sendable": 1_000,
|
||||
"max_sendable": 100_000_000,
|
||||
}
|
||||
|
||||
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 0.05), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
|
||||
), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_invoice",
|
||||
AsyncMock(return_value=("lnbc1...", {})),
|
||||
with (
|
||||
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
|
||||
patch.object(settings, "mint_retry_max_attempts", 0),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
pytest.raises(MeltOutcomeAmbiguousError, match="outcome is ambiguous"),
|
||||
):
|
||||
with pytest.raises(LNURLError, match="Melt timed out"):
|
||||
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
|
||||
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
|
||||
|
||||
wallet.get_melt_quote.assert_awaited_once_with("q")
|
||||
wallet.set_reserved_for_melt.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
|
||||
"""A prompt melt still returns the net amount, unaffected by the guard."""
|
||||
proofs = [MagicMock(amount=1000)]
|
||||
async def test_raw_send_to_lnurl_timeout_reconciled_paid_is_success() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
|
||||
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
|
||||
wallet.melt = AsyncMock(return_value=MagicMock())
|
||||
async def _hang(**kwargs: object) -> None:
|
||||
await asyncio.sleep(5)
|
||||
|
||||
lnurl_data = {
|
||||
"callback_url": "https://ln.tld/cb",
|
||||
"min_sendable": 1_000,
|
||||
"max_sendable": 100_000_000,
|
||||
}
|
||||
wallet.melt = AsyncMock(side_effect=_hang)
|
||||
wallet.get_melt_quote = AsyncMock(
|
||||
return_value=MagicMock(state=MeltQuoteState.paid)
|
||||
)
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 5), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
|
||||
), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_invoice",
|
||||
AsyncMock(return_value=("lnbc1...", {})),
|
||||
with (
|
||||
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
|
||||
patch.object(settings, "mint_retry_max_attempts", 0),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
):
|
||||
paid = await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
)
|
||||
|
||||
assert paid > 0
|
||||
wallet.get_melt_quote.assert_awaited_once_with("q")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_pending_response_stays_ambiguous() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.pending))
|
||||
wallet.get_melt_quote = AsyncMock(
|
||||
return_value=MagicMock(state=MeltQuoteState.pending)
|
||||
)
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
with (
|
||||
patch.object(settings, "mint_operation_timeout_seconds", 5),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
pytest.raises(MeltOutcomeAmbiguousError, match="outcome is ambiguous"),
|
||||
):
|
||||
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
|
||||
|
||||
wallet.get_melt_quote.assert_awaited_once_with("q")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("rate_error", ["cooldown", "http_429"])
|
||||
async def test_raw_send_to_lnurl_rate_rejection_unreserves_proofs(
|
||||
rate_error: str,
|
||||
) -> None:
|
||||
wallet, proofs = _wallet()
|
||||
wallet.melt = AsyncMock()
|
||||
wallet.set_reserved_for_send = AsyncMock()
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
async def run_operation(factory: Any, *, op_name: str, **_: object) -> Any:
|
||||
if op_name == "lnurl_melt":
|
||||
if rate_error == "cooldown":
|
||||
raise MintCooldownError(str(wallet.url), 60)
|
||||
request = httpx.Request("POST", f"{wallet.url}/v1/melt/bolt11")
|
||||
response = httpx.Response(429, request=request)
|
||||
raise httpx.HTTPStatusError(
|
||||
"rate limited", request=request, response=response
|
||||
)
|
||||
return await factory()
|
||||
|
||||
with (
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
patch(
|
||||
"routstr.payment.lnurl.run_mint_operation",
|
||||
side_effect=run_operation,
|
||||
),
|
||||
pytest.raises((MintCooldownError, httpx.HTTPStatusError)),
|
||||
):
|
||||
await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
)
|
||||
|
||||
wallet.melt.assert_not_awaited()
|
||||
wallet.set_reserved_for_send.assert_awaited_once_with(
|
||||
proofs, reserved=False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_mint_wrapper_http_429_unreserves_proofs() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
request = httpx.Request("POST", f"{wallet.url}/v1/melt/bolt11")
|
||||
response = httpx.Response(429, request=request)
|
||||
wallet.melt = AsyncMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
"rate limited", request=request, response=response
|
||||
)
|
||||
)
|
||||
wallet.set_reserved_for_send = AsyncMock()
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
with (
|
||||
patch.object(settings, "mint_retry_max_attempts", 0),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
pytest.raises(httpx.HTTPStatusError),
|
||||
):
|
||||
await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
)
|
||||
|
||||
wallet.melt.assert_awaited_once()
|
||||
wallet.set_reserved_for_send.assert_awaited_once_with(
|
||||
proofs, reserved=False
|
||||
)
|
||||
MintRateGuard._guards.pop(str(wallet.url), None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_succeeds_on_explicit_paid_response() -> None:
|
||||
wallet, proofs = _wallet()
|
||||
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.paid))
|
||||
wallet.get_melt_quote = AsyncMock()
|
||||
data_patch, invoice_patch = _lnurl_patches()
|
||||
|
||||
with (
|
||||
patch.object(settings, "mint_operation_timeout_seconds", 5),
|
||||
data_patch,
|
||||
invoice_patch,
|
||||
):
|
||||
paid = await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
@@ -72,3 +204,4 @@ async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
|
||||
|
||||
assert paid > 0
|
||||
wallet.melt.assert_awaited_once()
|
||||
wallet.get_melt_quote.assert_not_awaited()
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
from cashu.core.base import MeltQuoteState, ProofSpentState
|
||||
|
||||
from routstr.wallet import (
|
||||
TokenConsumedError,
|
||||
_confirm_melt_paid,
|
||||
_reconcile_ambiguous_melt,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_quote_is_authoritative_when_proof_lookup_would_fail() -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.paid)),
|
||||
check_proof_state=AsyncMock(side_effect=RuntimeError("proof API unavailable")),
|
||||
)
|
||||
|
||||
assert await _reconcile_ambiguous_melt(wallet, "quote-1", [Mock()]) is True
|
||||
wallet.check_proof_state.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_timeout_snapshot_unpaid_unspent_remains_non_retryable() -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.unpaid)),
|
||||
check_proof_state=AsyncMock(
|
||||
return_value=Mock(states=[Mock(state=ProofSpentState.unspent)])
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(TokenConsumedError, match="ambiguous"):
|
||||
await _reconcile_ambiguous_melt(wallet, "quote-2", [Mock()])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_pending_melt_response_requires_reconciliation() -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.pending)),
|
||||
check_proof_state=AsyncMock(
|
||||
return_value=Mock(states=[Mock(state=ProofSpentState.pending)])
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(TokenConsumedError, match="ambiguous"):
|
||||
await _confirm_melt_paid(
|
||||
wallet,
|
||||
"quote-pending",
|
||||
[Mock()],
|
||||
Mock(state=MeltQuoteState.pending),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("quote_state", "proof_state"),
|
||||
[
|
||||
(MeltQuoteState.pending, ProofSpentState.pending),
|
||||
(MeltQuoteState.unpaid, ProofSpentState.spent),
|
||||
(MeltQuoteState.unpaid, ProofSpentState.pending),
|
||||
],
|
||||
)
|
||||
async def test_ambiguous_or_consumed_melt_is_never_reported_unspent(
|
||||
quote_state: MeltQuoteState, proof_state: ProofSpentState
|
||||
) -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(return_value=Mock(state=quote_state)),
|
||||
check_proof_state=AsyncMock(
|
||||
return_value=Mock(states=[Mock(state=proof_state)])
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(TokenConsumedError, match="reconciliation required"):
|
||||
await _reconcile_ambiguous_melt(wallet, "quote-3", [Mock()])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_melt_reconciliation_is_non_retryable() -> None:
|
||||
wallet = Mock(
|
||||
url="http://source-mint:3338",
|
||||
get_melt_quote=AsyncMock(side_effect=RuntimeError("mint unavailable")),
|
||||
check_proof_state=AsyncMock(),
|
||||
)
|
||||
|
||||
with pytest.raises(TokenConsumedError, match="outcome is unknown"):
|
||||
await _reconcile_ambiguous_melt(wallet, "quote-4", [Mock()])
|
||||
@@ -0,0 +1,121 @@
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from cashu.core.base import Unit
|
||||
|
||||
from routstr.mint import (
|
||||
MintCooldownError,
|
||||
MintRateGuard,
|
||||
MintRateLimitedError,
|
||||
fail_fast_mint_operations,
|
||||
)
|
||||
from routstr.wallet import Wallet
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cooldown_fails_fast_while_wallet_mutation_scope_is_held() -> None:
|
||||
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
|
||||
guard.apply_cooldown(3600, reason="rate_limited")
|
||||
operation = AsyncMock(return_value="should not run")
|
||||
|
||||
with (
|
||||
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
|
||||
pytest.raises(MintCooldownError) as caught,
|
||||
):
|
||||
async with fail_fast_mint_operations():
|
||||
await guard.run(operation)
|
||||
|
||||
assert caught.value.retry_after_seconds > 0
|
||||
operation.assert_not_awaited()
|
||||
sleep.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expired_cooldown_allows_probe_in_wallet_mutation_scope() -> None:
|
||||
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
|
||||
guard.apply_cooldown(0, reason="rate_limited")
|
||||
operation = AsyncMock(return_value="recovered")
|
||||
|
||||
async with fail_fast_mint_operations():
|
||||
result = await guard.run(operation)
|
||||
|
||||
assert result == "recovered"
|
||||
operation.assert_awaited_once()
|
||||
assert guard._needs_probe is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_fast_does_not_wait_behind_existing_probe() -> None:
|
||||
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
|
||||
guard.apply_cooldown(0, reason="rate_limited")
|
||||
probe_started = asyncio.Event()
|
||||
release_probe = asyncio.Event()
|
||||
|
||||
async def probe() -> str:
|
||||
probe_started.set()
|
||||
await release_probe.wait()
|
||||
return "recovered"
|
||||
|
||||
first = asyncio.create_task(guard.run(probe))
|
||||
await probe_started.wait()
|
||||
try:
|
||||
async with fail_fast_mint_operations():
|
||||
with pytest.raises(MintCooldownError):
|
||||
await asyncio.wait_for(guard.run(AsyncMock()), timeout=0.05)
|
||||
finally:
|
||||
release_probe.set()
|
||||
assert await first == "recovered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cashu_429_dispatches_through_wallet_override() -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
429,
|
||||
request=request,
|
||||
json={"detail": "too many requests", "code": 42900},
|
||||
)
|
||||
|
||||
wallet = object.__new__(Wallet)
|
||||
wallet.url = "http://mint:3338"
|
||||
wallet.db = Mock()
|
||||
wallet.keysets = {"loaded": Mock()}
|
||||
wallet.mint_info = Mock()
|
||||
wallet.mint_info.requires_blind_auth_path.return_value = False
|
||||
wallet.mint_info.requires_clear_auth_path.return_value = False
|
||||
wallet.auth_db = None
|
||||
wallet.auth_keyset_id = None
|
||||
|
||||
real_client = httpx.AsyncClient
|
||||
|
||||
def client_factory(*args: object, **kwargs: object) -> httpx.AsyncClient:
|
||||
return real_client(
|
||||
transport=httpx.MockTransport(handler),
|
||||
base_url=str(kwargs["base_url"]),
|
||||
)
|
||||
|
||||
with (
|
||||
patch("cashu.wallet.v1_api.httpx.AsyncClient", side_effect=client_factory),
|
||||
pytest.raises(MintRateLimitedError),
|
||||
):
|
||||
await wallet.mint_quote(1, Unit.sat)
|
||||
|
||||
|
||||
async def test_guard_concurrency_change_preserves_cooldown_state() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
mint_url = "https://mint.test-concurrency-carryover"
|
||||
with patch.object(settings, "mint_max_concurrency", 2):
|
||||
guard = MintRateGuard.get(mint_url)
|
||||
guard.apply_cooldown(120.0, reason="rate_limited")
|
||||
guard._consecutive_rate_limits = 3
|
||||
|
||||
with patch.object(settings, "mint_max_concurrency", 5):
|
||||
rebuilt = MintRateGuard.get(mint_url)
|
||||
|
||||
assert rebuilt is not guard
|
||||
assert rebuilt.cooldown_remaining() > 0
|
||||
assert rebuilt._cooldown_reason == "rate_limited"
|
||||
assert rebuilt._consecutive_rate_limits == 3
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Persisted mint preferences must not bypass the configured trusted set."""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.core.settings import settings
|
||||
from routstr.lightning import _request_mint_with_fallback
|
||||
|
||||
TRUSTED = "https://good-mint.example.com"
|
||||
UNTRUSTED = "https://removed-mint.example.com"
|
||||
|
||||
|
||||
async def test_untrusted_allowed_mints_fall_back_to_trusted_set() -> None:
|
||||
attempted: list[str] = []
|
||||
|
||||
async def fake_get_wallet(mint_url: str, unit: str, **kwargs: object) -> None:
|
||||
attempted.append(mint_url)
|
||||
raise ConnectionError("unreachable in test")
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", TRUSTED),
|
||||
patch.object(settings, "cashu_mints", [TRUSTED]),
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(side_effect=fake_get_wallet)),
|
||||
patch("routstr.lightning.mint_cooldown_remaining", return_value=0.0),
|
||||
):
|
||||
with pytest.raises(Exception):
|
||||
await _request_mint_with_fallback(10, allowed_mints=[UNTRUSTED])
|
||||
|
||||
assert UNTRUSTED not in attempted
|
||||
assert attempted == [TRUSTED]
|
||||
|
||||
|
||||
async def test_trusted_allowed_mints_are_used_verbatim() -> None:
|
||||
attempted: list[str] = []
|
||||
|
||||
async def fake_get_wallet(mint_url: str, unit: str, **kwargs: object) -> None:
|
||||
attempted.append(mint_url)
|
||||
raise ConnectionError("unreachable in test")
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", TRUSTED),
|
||||
patch.object(settings, "cashu_mints", [TRUSTED, "https://other.example.com"]),
|
||||
patch("routstr.lightning.get_wallet", AsyncMock(side_effect=fake_get_wallet)),
|
||||
patch("routstr.lightning.mint_cooldown_remaining", return_value=0.0),
|
||||
):
|
||||
with pytest.raises(Exception):
|
||||
await _request_mint_with_fallback(10, allowed_mints=[TRUSTED])
|
||||
|
||||
assert attempted == [TRUSTED]
|
||||
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _run_alembic(root: Path, database_url: str, command: str, revision: str) -> None:
|
||||
env = os.environ.copy()
|
||||
env["DATABASE_URL"] = database_url
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", command, revision],
|
||||
cwd=root,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def _lightning_invoice_columns(database_path: Path) -> set[str]:
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
return {
|
||||
row[1]
|
||||
for row in connection.execute("PRAGMA table_info(lightning_invoices)")
|
||||
}
|
||||
|
||||
|
||||
def test_mint_url_migration_upgrades_and_downgrades_from_main_head(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
database_path = tmp_path / "mint-url-migration.db"
|
||||
database_url = f"sqlite+aiosqlite:///{database_path}"
|
||||
previous_head = "64ed5594df1f"
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", previous_head)
|
||||
assert "mint_url" not in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", "ecfa0d6e2a36")
|
||||
assert "mint_url" in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "downgrade", previous_head)
|
||||
assert "mint_url" not in _lightning_invoice_columns(database_path)
|
||||
|
||||
_run_alembic(root, database_url, "upgrade", "head")
|
||||
assert "mint_url" in _lightning_invoice_columns(database_path)
|
||||
@@ -125,3 +125,33 @@ async def test_get_max_cost_for_model_tolerance() -> None:
|
||||
"gpt-4", session=mock_session, model_obj=mock_model
|
||||
)
|
||||
assert cost == 450000 # 500 sats * 1000 * 0.9 = 450000
|
||||
|
||||
|
||||
async def test_discounted_max_cost_floors_at_min_request_msat() -> None:
|
||||
from routstr.payment.helpers import calculate_discounted_max_cost
|
||||
|
||||
pricing = Mock()
|
||||
pricing.prompt = 0.001
|
||||
pricing.completion = 0.001
|
||||
pricing.max_prompt_cost = 100.0
|
||||
pricing.max_completion_cost = 100.0
|
||||
|
||||
model_obj = Mock()
|
||||
model_obj.sats_pricing = pricing
|
||||
model_obj.top_provider = None
|
||||
model_obj.context_length = None
|
||||
|
||||
body = {
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 1,
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(settings, "fixed_pricing", False),
|
||||
patch.object(settings, "tolerance_percentage", 0),
|
||||
patch.object(settings, "min_request_msat", 1000),
|
||||
):
|
||||
cost = await calculate_discounted_max_cost(150_000, body, model_obj)
|
||||
|
||||
assert cost == 1000
|
||||
|
||||
@@ -147,7 +147,9 @@ async def test_periodic_payout_isolates_failing_mint() -> None:
|
||||
"""A failing mint does not prevent payout for the other mints."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
async def _get_wallet(mint_url: str, unit: str) -> MagicMock:
|
||||
async def _get_wallet(
|
||||
mint_url: str, unit: str, force_reload: bool = False
|
||||
) -> MagicMock:
|
||||
if mint_url == "http://bad:3338":
|
||||
raise RuntimeError("mint unreachable")
|
||||
return MagicMock()
|
||||
|
||||
@@ -71,7 +71,9 @@ async def test_pay_for_request_sets_reserved_at(session: AsyncSession) -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pay_for_request_sets_reserved_at_on_child_key(session: AsyncSession) -> None:
|
||||
async def test_pay_for_request_sets_reserved_at_on_child_key(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
parent = ApiKey(hashed_key="parentkey", balance=10_000)
|
||||
child = ApiKey(hashed_key="childkey", balance=0, parent_key_hash="parentkey")
|
||||
session.add(parent)
|
||||
@@ -204,7 +206,9 @@ async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) ->
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncSession) -> None:
|
||||
async def test_release_stale_reservations_skips_null_reserved_at(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
# Reservations without a timestamp may belong to instances running older
|
||||
# code (rolling deploy) — the background sweeper must not touch them.
|
||||
key = ApiKey(
|
||||
@@ -224,7 +228,9 @@ async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_all_reserved_balances_clears_reserved_at(session: AsyncSession) -> None:
|
||||
async def test_reset_all_reserved_balances_clears_reserved_at(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
key = ApiKey(
|
||||
hashed_key="resetkey",
|
||||
balance=5_000,
|
||||
@@ -404,9 +410,7 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
|
||||
AsyncMock(return_value=1_000),
|
||||
),
|
||||
patch.object(proxy_module, "check_token_balance", MagicMock()),
|
||||
patch.object(
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
@@ -418,6 +422,4 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await proxy_module.proxy(request, "v1/chat/completions", session=session)
|
||||
|
||||
revert_mock.assert_awaited_once_with(
|
||||
key, session, 1_000, reservation_snapshot
|
||||
)
|
||||
revert_mock.assert_awaited_once_with(key, session, 1000, reservation_snapshot)
|
||||
|
||||
@@ -383,9 +383,7 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
AsyncMock(return_value=1_000),
|
||||
),
|
||||
patch.object(proxy_module, "check_token_balance", MagicMock()),
|
||||
patch.object(
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
@@ -408,4 +406,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
# Single upstream failed -> reservation reverted exactly once (no double-charge).
|
||||
revert_mock.assert_awaited_once_with(key, session, 1_000, reservation)
|
||||
revert_mock.assert_awaited_once_with(key, session, 1000, reservation)
|
||||
|
||||
+1338
-47
File diff suppressed because it is too large
Load Diff
@@ -105,6 +105,23 @@ export function DetailedWalletBalance({
|
||||
const formatMintLabel = (detail: BalanceDetail) =>
|
||||
`${detail.mint_url.replace('https://', '').replace('http://', '')} • ${detail.unit.toUpperCase()}`;
|
||||
|
||||
const formatBalanceError = (detail: BalanceDetail) => {
|
||||
const labels: Record<string, string> = {
|
||||
rate_limited: 'rate limited',
|
||||
unreachable: 'unreachable',
|
||||
cooldown: 'cooling down',
|
||||
mint_error: 'mint error',
|
||||
};
|
||||
const label =
|
||||
(detail.error_code ? labels[detail.error_code] : undefined) ??
|
||||
detail.error ??
|
||||
'error';
|
||||
const retryAfter = detail.retry_after_seconds;
|
||||
return retryAfter && retryAfter > 0
|
||||
? `${label} (retry in ${Math.ceil(retryAfter)}s)`
|
||||
: label;
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Card>
|
||||
@@ -262,9 +279,12 @@ export function DetailedWalletBalance({
|
||||
<TableCell className='max-w-md font-mono text-xs break-all whitespace-normal'>
|
||||
{formatMintLabel(detail)}
|
||||
</TableCell>
|
||||
<TableCell className='text-right font-mono'>
|
||||
<TableCell
|
||||
className='text-right font-mono'
|
||||
title={detail.error}
|
||||
>
|
||||
{detail.error
|
||||
? 'error'
|
||||
? formatBalanceError(detail)
|
||||
: formatAmount(walletMsat)}
|
||||
</TableCell>
|
||||
<TableCell className='text-right font-mono'>
|
||||
@@ -306,9 +326,12 @@ export function DetailedWalletBalance({
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Wallet
|
||||
</p>
|
||||
<p className='font-mono text-sm'>
|
||||
<p
|
||||
className='font-mono text-sm'
|
||||
title={detail.error}
|
||||
>
|
||||
{detail.error
|
||||
? 'error'
|
||||
? formatBalanceError(detail)
|
||||
: formatAmount(walletMsat)}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -36,10 +36,13 @@ export interface BalanceDetail {
|
||||
user_balance: number;
|
||||
owner_balance: number;
|
||||
error?: string;
|
||||
error_code?: 'rate_limited' | 'unreachable' | 'cooldown' | 'mint_error';
|
||||
retry_after_seconds?: number;
|
||||
}
|
||||
|
||||
export interface WithdrawResponse {
|
||||
token: string;
|
||||
mint_url: string;
|
||||
}
|
||||
|
||||
export interface CreateChildKeyResponse {
|
||||
|
||||
Reference in New Issue
Block a user